1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2016 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
34 #include "alloc-pool.h"
35 #include "tree-pass.h"
41 #include "pretty-print.h"
42 #include "diagnostic-core.h"
44 #include "fold-const.h"
45 #include "stor-layout.h"
47 #include "internal-fn.h"
48 #include "gimple-fold.h"
50 #include "gimple-iterator.h"
51 #include "gimplify-me.h"
52 #include "gimple-walk.h"
53 #include "tree-iterator.h"
54 #include "tree-inline.h"
55 #include "langhooks.h"
57 #include "tree-into-ssa.h"
68 #include "splay-tree.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "symbol-summary.h"
76 #include "tree-nested.h"
80 #include "lto-section-names.h"
81 #include "gomp-constants.h"
82 #include "gimple-pretty-print.h"
83 #include "symbol-summary.h"
87 /* Lowering of OMP parallel and workshare constructs proceeds in two
88 phases. The first phase scans the function looking for OMP statements
89 and then for variables that must be replaced to satisfy data sharing
90 clauses. The second phase expands code for the constructs, as well as
91 re-gimplifying things when variables have been replaced with complex
94 Final code generation is done by pass_expand_omp. The flowgraph is
95 scanned for regions which are then moved to a new
96 function, to be invoked by the thread library, or offloaded. */
98 /* OMP region information. Every parallel and workshare
99 directive is enclosed between two markers, the OMP_* directive
100 and a corresponding GIMPLE_OMP_RETURN statement. */
104 /* The enclosing region. */
105 struct omp_region *outer;
107 /* First child region. */
108 struct omp_region *inner;
110 /* Next peer region. */
111 struct omp_region *next;
113 /* Block containing the omp directive as its last stmt. */
116 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
119 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
122 /* If this is a combined parallel+workshare region, this is a list
123 of additional arguments needed by the combined parallel+workshare
125 vec<tree, va_gc> *ws_args;
127 /* The code for the omp directive of this region. */
128 enum gimple_code type;
130 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
131 enum omp_clause_schedule_kind sched_kind;
133 /* Schedule modifiers. */
134 unsigned char sched_modifiers;
136 /* True if this is a combined parallel+workshare region. */
137 bool is_combined_parallel;
139 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
141 gomp_ordered *ord_stmt;
144 /* Context structure. Used to store information about each parallel
145 directive in the code. */
149 /* This field must be at the beginning, as we do "inheritance": Some
150 callback functions for tree-inline.c (e.g., omp_copy_decl)
151 receive a copy_body_data pointer that is up-casted to an
152 omp_context pointer. */
155 /* The tree of contexts corresponding to the encountered constructs. */
156 struct omp_context *outer;
159 /* Map variables to fields in a structure that allows communication
160 between sending and receiving threads. */
161 splay_tree field_map;
166 /* These are used just by task contexts, if task firstprivate fn is
167 needed. srecord_type is used to communicate from the thread
168 that encountered the task construct to task firstprivate fn,
169 record_type is allocated by GOMP_task, initialized by task firstprivate
170 fn and passed to the task body fn. */
171 splay_tree sfield_map;
174 /* A chain of variables to add to the top-level block surrounding the
175 construct. In the case of a parallel, this is in the child function. */
178 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
179 barriers should jump to during omplower pass. */
182 /* What to do with variables with implicitly determined sharing
184 enum omp_clause_default_kind default_kind;
186 /* Nesting depth of this context. Used to beautify error messages re
187 invalid gotos. The outermost ctx is depth 1, with depth 0 being
188 reserved for the main body of the function. */
191 /* True if this parallel directive is nested within another. */
194 /* True if this construct can be cancelled. */
198 /* A structure holding the elements of:
199 for (V = N1; V cond N2; V += STEP) [...] */
201 struct omp_for_data_loop
203 tree v, n1, n2, step;
204 enum tree_code cond_code;
207 /* A structure describing the main elements of a parallel loop. */
211 struct omp_for_data_loop loop;
217 bool have_nowait, have_ordered, simd_schedule;
218 unsigned char sched_modifiers;
219 enum omp_clause_schedule_kind sched_kind;
220 struct omp_for_data_loop *loops;
223 /* Describe the OpenACC looping structure of a function. The entire
224 function is held in a 'NULL' loop. */
228 oacc_loop *parent; /* Containing loop. */
230 oacc_loop *child; /* First inner loop. */
232 oacc_loop *sibling; /* Next loop within same parent. */
234 location_t loc; /* Location of the loop start. */
236 gcall *marker; /* Initial head marker. */
238 gcall *heads[GOMP_DIM_MAX]; /* Head marker functions. */
239 gcall *tails[GOMP_DIM_MAX]; /* Tail marker functions. */
241 tree routine; /* Pseudo-loop enclosing a routine. */
243 unsigned mask; /* Partitioning mask. */
244 unsigned flags; /* Partitioning flags. */
245 unsigned ifns; /* Contained loop abstraction functions. */
246 tree chunk_size; /* Chunk size. */
247 gcall *head_end; /* Final marker of head sequence. */
250 /* Flags for an OpenACC loop. */
252 enum oacc_loop_flags {
253 OLF_SEQ = 1u << 0, /* Explicitly sequential */
254 OLF_AUTO = 1u << 1, /* Compiler chooses axes. */
255 OLF_INDEPENDENT = 1u << 2, /* Iterations are known independent. */
256 OLF_GANG_STATIC = 1u << 3, /* Gang partitioning is static (has op). */
258 /* Explicitly specified loop axes. */
260 OLF_DIM_GANG = 1u << (OLF_DIM_BASE + GOMP_DIM_GANG),
261 OLF_DIM_WORKER = 1u << (OLF_DIM_BASE + GOMP_DIM_WORKER),
262 OLF_DIM_VECTOR = 1u << (OLF_DIM_BASE + GOMP_DIM_VECTOR),
264 OLF_MAX = OLF_DIM_BASE + GOMP_DIM_MAX
268 static splay_tree all_contexts;
269 static int taskreg_nesting_level;
270 static int target_nesting_level;
271 static struct omp_region *root_omp_region;
272 static bitmap task_shared_vars;
273 static vec<omp_context *> taskreg_contexts;
274 static bool omp_any_child_fn_dumped;
276 static void scan_omp (gimple_seq *, omp_context *);
277 static tree scan_omp_1_op (tree *, int *, void *);
278 static gphi *find_phi_with_arg_on_edge (tree, edge);
280 #define WALK_SUBSTMTS \
284 case GIMPLE_EH_FILTER: \
285 case GIMPLE_TRANSACTION: \
286 /* The sub-statements for these should be walked. */ \
287 *handled_ops_p = false; \
290 /* Return true if CTX corresponds to an oacc parallel region. */
293 is_oacc_parallel (omp_context *ctx)
295 enum gimple_code outer_type = gimple_code (ctx->stmt);
296 return ((outer_type == GIMPLE_OMP_TARGET)
297 && (gimple_omp_target_kind (ctx->stmt)
298 == GF_OMP_TARGET_KIND_OACC_PARALLEL));
301 /* Return true if CTX corresponds to an oacc kernels region. */
304 is_oacc_kernels (omp_context *ctx)
306 enum gimple_code outer_type = gimple_code (ctx->stmt);
307 return ((outer_type == GIMPLE_OMP_TARGET)
308 && (gimple_omp_target_kind (ctx->stmt)
309 == GF_OMP_TARGET_KIND_OACC_KERNELS));
312 /* If DECL is the artificial dummy VAR_DECL created for non-static
313 data member privatization, return the underlying "this" parameter,
314 otherwise return NULL. */
317 omp_member_access_dummy_var (tree decl)
320 || !DECL_ARTIFICIAL (decl)
321 || !DECL_IGNORED_P (decl)
322 || !DECL_HAS_VALUE_EXPR_P (decl)
323 || !lang_hooks.decls.omp_disregard_value_expr (decl, false))
326 tree v = DECL_VALUE_EXPR (decl);
327 if (TREE_CODE (v) != COMPONENT_REF)
331 switch (TREE_CODE (v))
337 case POINTER_PLUS_EXPR:
338 v = TREE_OPERAND (v, 0);
341 if (DECL_CONTEXT (v) == current_function_decl
342 && DECL_ARTIFICIAL (v)
343 && TREE_CODE (TREE_TYPE (v)) == POINTER_TYPE)
351 /* Helper for unshare_and_remap, called through walk_tree. */
354 unshare_and_remap_1 (tree *tp, int *walk_subtrees, void *data)
356 tree *pair = (tree *) data;
359 *tp = unshare_expr (pair[1]);
362 else if (IS_TYPE_OR_DECL_P (*tp))
367 /* Return unshare_expr (X) with all occurrences of FROM
371 unshare_and_remap (tree x, tree from, tree to)
373 tree pair[2] = { from, to };
374 x = unshare_expr (x);
375 walk_tree (&x, unshare_and_remap_1, pair, NULL);
379 /* Holds offload tables with decls. */
380 vec<tree, va_gc> *offload_funcs, *offload_vars;
382 /* Convenience function for calling scan_omp_1_op on tree operands. */
385 scan_omp_op (tree *tp, omp_context *ctx)
387 struct walk_stmt_info wi;
389 memset (&wi, 0, sizeof (wi));
391 wi.want_locations = true;
393 return walk_tree (tp, scan_omp_1_op, &wi, NULL);
396 static void lower_omp (gimple_seq *, omp_context *);
397 static tree lookup_decl_in_outer_ctx (tree, omp_context *);
398 static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
400 /* Find an OMP clause of type KIND within CLAUSES. */
403 find_omp_clause (tree clauses, enum omp_clause_code kind)
405 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
406 if (OMP_CLAUSE_CODE (clauses) == kind)
412 /* Return true if CTX is for an omp parallel. */
415 is_parallel_ctx (omp_context *ctx)
417 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
421 /* Return true if CTX is for an omp task. */
424 is_task_ctx (omp_context *ctx)
426 return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
430 /* Return true if CTX is for an omp taskloop. */
433 is_taskloop_ctx (omp_context *ctx)
435 return gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
436 && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP;
440 /* Return true if CTX is for an omp parallel or omp task. */
443 is_taskreg_ctx (omp_context *ctx)
445 return is_parallel_ctx (ctx) || is_task_ctx (ctx);
449 /* Return true if REGION is a combined parallel+workshare region. */
452 is_combined_parallel (struct omp_region *region)
454 return region->is_combined_parallel;
457 /* Adjust *COND_CODE and *N2 so that the former is either LT_EXPR or
461 adjust_for_condition (location_t loc, enum tree_code *cond_code, tree *n2)
470 if (POINTER_TYPE_P (TREE_TYPE (*n2)))
471 *n2 = fold_build_pointer_plus_hwi_loc (loc, *n2, 1);
473 *n2 = fold_build2_loc (loc, PLUS_EXPR, TREE_TYPE (*n2), *n2,
474 build_int_cst (TREE_TYPE (*n2), 1));
475 *cond_code = LT_EXPR;
478 if (POINTER_TYPE_P (TREE_TYPE (*n2)))
479 *n2 = fold_build_pointer_plus_hwi_loc (loc, *n2, -1);
481 *n2 = fold_build2_loc (loc, MINUS_EXPR, TREE_TYPE (*n2), *n2,
482 build_int_cst (TREE_TYPE (*n2), 1));
483 *cond_code = GT_EXPR;
490 /* Return the looping step from INCR, extracted from the step of a gimple omp
494 get_omp_for_step_from_incr (location_t loc, tree incr)
497 switch (TREE_CODE (incr))
500 step = TREE_OPERAND (incr, 1);
502 case POINTER_PLUS_EXPR:
503 step = fold_convert (ssizetype, TREE_OPERAND (incr, 1));
506 step = TREE_OPERAND (incr, 1);
507 step = fold_build1_loc (loc, NEGATE_EXPR, TREE_TYPE (step), step);
515 /* Extract the header elements of parallel loop FOR_STMT and store
519 extract_omp_for_data (gomp_for *for_stmt, struct omp_for_data *fd,
520 struct omp_for_data_loop *loops)
522 tree t, var, *collapse_iter, *collapse_count;
523 tree count = NULL_TREE, iter_type = long_integer_type_node;
524 struct omp_for_data_loop *loop;
526 struct omp_for_data_loop dummy_loop;
527 location_t loc = gimple_location (for_stmt);
528 bool simd = gimple_omp_for_kind (for_stmt) & GF_OMP_FOR_SIMD;
529 bool distribute = gimple_omp_for_kind (for_stmt)
530 == GF_OMP_FOR_KIND_DISTRIBUTE;
531 bool taskloop = gimple_omp_for_kind (for_stmt)
532 == GF_OMP_FOR_KIND_TASKLOOP;
535 fd->for_stmt = for_stmt;
537 if (gimple_omp_for_collapse (for_stmt) > 1)
540 fd->loops = &fd->loop;
542 fd->have_nowait = distribute || simd;
543 fd->have_ordered = false;
546 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
547 fd->sched_modifiers = 0;
548 fd->chunk_size = NULL_TREE;
549 fd->simd_schedule = false;
550 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_CILKFOR)
551 fd->sched_kind = OMP_CLAUSE_SCHEDULE_CILKFOR;
552 collapse_iter = NULL;
553 collapse_count = NULL;
555 for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t))
556 switch (OMP_CLAUSE_CODE (t))
558 case OMP_CLAUSE_NOWAIT:
559 fd->have_nowait = true;
561 case OMP_CLAUSE_ORDERED:
562 fd->have_ordered = true;
563 if (OMP_CLAUSE_ORDERED_EXPR (t))
564 fd->ordered = tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t));
566 case OMP_CLAUSE_SCHEDULE:
567 gcc_assert (!distribute && !taskloop);
569 = (enum omp_clause_schedule_kind)
570 (OMP_CLAUSE_SCHEDULE_KIND (t) & OMP_CLAUSE_SCHEDULE_MASK);
571 fd->sched_modifiers = (OMP_CLAUSE_SCHEDULE_KIND (t)
572 & ~OMP_CLAUSE_SCHEDULE_MASK);
573 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t);
574 fd->simd_schedule = OMP_CLAUSE_SCHEDULE_SIMD (t);
576 case OMP_CLAUSE_DIST_SCHEDULE:
577 gcc_assert (distribute);
578 fd->chunk_size = OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t);
580 case OMP_CLAUSE_COLLAPSE:
581 fd->collapse = tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t));
582 if (fd->collapse > 1)
584 collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t);
585 collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t);
591 if (fd->ordered && fd->collapse == 1 && loops != NULL)
596 collapse_iter = &iterv;
597 collapse_count = &countv;
600 /* FIXME: for now map schedule(auto) to schedule(static).
601 There should be analysis to determine whether all iterations
602 are approximately the same amount of work (then schedule(static)
603 is best) or if it varies (then schedule(dynamic,N) is better). */
604 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO)
606 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
607 gcc_assert (fd->chunk_size == NULL);
609 gcc_assert (fd->collapse == 1 || collapse_iter != NULL);
611 fd->sched_kind = OMP_CLAUSE_SCHEDULE_RUNTIME;
612 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
613 gcc_assert (fd->chunk_size == NULL);
614 else if (fd->chunk_size == NULL)
616 /* We only need to compute a default chunk size for ordered
617 static loops and dynamic loops. */
618 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
620 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
621 ? integer_zero_node : integer_one_node;
624 int cnt = fd->ordered ? fd->ordered : fd->collapse;
625 for (i = 0; i < cnt; i++)
627 if (i == 0 && fd->collapse == 1 && (fd->ordered == 0 || loops == NULL))
629 else if (loops != NULL)
634 loop->v = gimple_omp_for_index (for_stmt, i);
635 gcc_assert (SSA_VAR_P (loop->v));
636 gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
637 || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE);
638 var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v;
639 loop->n1 = gimple_omp_for_initial (for_stmt, i);
641 loop->cond_code = gimple_omp_for_cond (for_stmt, i);
642 loop->n2 = gimple_omp_for_final (for_stmt, i);
643 gcc_assert (loop->cond_code != NE_EXPR
644 || gimple_omp_for_kind (for_stmt) == GF_OMP_FOR_KIND_CILKSIMD
645 || gimple_omp_for_kind (for_stmt) == GF_OMP_FOR_KIND_CILKFOR);
646 adjust_for_condition (loc, &loop->cond_code, &loop->n2);
648 t = gimple_omp_for_incr (for_stmt, i);
649 gcc_assert (TREE_OPERAND (t, 0) == var);
650 loop->step = get_omp_for_step_from_incr (loc, t);
653 || (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
654 && !fd->have_ordered))
656 if (fd->collapse == 1)
657 iter_type = TREE_TYPE (loop->v);
659 || TYPE_PRECISION (iter_type)
660 < TYPE_PRECISION (TREE_TYPE (loop->v)))
662 = build_nonstandard_integer_type
663 (TYPE_PRECISION (TREE_TYPE (loop->v)), 1);
665 else if (iter_type != long_long_unsigned_type_node)
667 if (POINTER_TYPE_P (TREE_TYPE (loop->v)))
668 iter_type = long_long_unsigned_type_node;
669 else if (TYPE_UNSIGNED (TREE_TYPE (loop->v))
670 && TYPE_PRECISION (TREE_TYPE (loop->v))
671 >= TYPE_PRECISION (iter_type))
675 if (loop->cond_code == LT_EXPR)
676 n = fold_build2_loc (loc,
677 PLUS_EXPR, TREE_TYPE (loop->v),
678 loop->n2, loop->step);
681 if (TREE_CODE (n) != INTEGER_CST
682 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n))
683 iter_type = long_long_unsigned_type_node;
685 else if (TYPE_PRECISION (TREE_TYPE (loop->v))
686 > TYPE_PRECISION (iter_type))
690 if (loop->cond_code == LT_EXPR)
693 n2 = fold_build2_loc (loc,
694 PLUS_EXPR, TREE_TYPE (loop->v),
695 loop->n2, loop->step);
699 n1 = fold_build2_loc (loc,
700 MINUS_EXPR, TREE_TYPE (loop->v),
701 loop->n2, loop->step);
704 if (TREE_CODE (n1) != INTEGER_CST
705 || TREE_CODE (n2) != INTEGER_CST
706 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1)
707 || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type)))
708 iter_type = long_long_unsigned_type_node;
712 if (i >= fd->collapse)
715 if (collapse_count && *collapse_count == NULL)
717 t = fold_binary (loop->cond_code, boolean_type_node,
718 fold_convert (TREE_TYPE (loop->v), loop->n1),
719 fold_convert (TREE_TYPE (loop->v), loop->n2));
720 if (t && integer_zerop (t))
721 count = build_zero_cst (long_long_unsigned_type_node);
722 else if ((i == 0 || count != NULL_TREE)
723 && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
724 && TREE_CONSTANT (loop->n1)
725 && TREE_CONSTANT (loop->n2)
726 && TREE_CODE (loop->step) == INTEGER_CST)
728 tree itype = TREE_TYPE (loop->v);
730 if (POINTER_TYPE_P (itype))
731 itype = signed_type_for (itype);
732 t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1));
733 t = fold_build2_loc (loc,
735 fold_convert_loc (loc, itype, loop->step), t);
736 t = fold_build2_loc (loc, PLUS_EXPR, itype, t,
737 fold_convert_loc (loc, itype, loop->n2));
738 t = fold_build2_loc (loc, MINUS_EXPR, itype, t,
739 fold_convert_loc (loc, itype, loop->n1));
740 if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR)
741 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype,
742 fold_build1_loc (loc, NEGATE_EXPR, itype, t),
743 fold_build1_loc (loc, NEGATE_EXPR, itype,
744 fold_convert_loc (loc, itype,
747 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t,
748 fold_convert_loc (loc, itype, loop->step));
749 t = fold_convert_loc (loc, long_long_unsigned_type_node, t);
750 if (count != NULL_TREE)
751 count = fold_build2_loc (loc,
752 MULT_EXPR, long_long_unsigned_type_node,
756 if (TREE_CODE (count) != INTEGER_CST)
759 else if (count && !integer_zerop (count))
766 && (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
767 || fd->have_ordered))
769 if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node)))
770 iter_type = long_long_unsigned_type_node;
772 iter_type = long_integer_type_node;
774 else if (collapse_iter && *collapse_iter != NULL)
775 iter_type = TREE_TYPE (*collapse_iter);
776 fd->iter_type = iter_type;
777 if (collapse_iter && *collapse_iter == NULL)
778 *collapse_iter = create_tmp_var (iter_type, ".iter");
779 if (collapse_count && *collapse_count == NULL)
782 *collapse_count = fold_convert_loc (loc, iter_type, count);
784 *collapse_count = create_tmp_var (iter_type, ".count");
787 if (fd->collapse > 1 || (fd->ordered && loops))
789 fd->loop.v = *collapse_iter;
790 fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0);
791 fd->loop.n2 = *collapse_count;
792 fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1);
793 fd->loop.cond_code = LT_EXPR;
800 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
801 is the immediate dominator of PAR_ENTRY_BB, return true if there
802 are no data dependencies that would prevent expanding the parallel
803 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
805 When expanding a combined parallel+workshare region, the call to
806 the child function may need additional arguments in the case of
807 GIMPLE_OMP_FOR regions. In some cases, these arguments are
808 computed out of variables passed in from the parent to the child
809 via 'struct .omp_data_s'. For instance:
811 #pragma omp parallel for schedule (guided, i * 4)
816 # BLOCK 2 (PAR_ENTRY_BB)
818 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
820 # BLOCK 3 (WS_ENTRY_BB)
821 .omp_data_i = &.omp_data_o;
822 D.1667 = .omp_data_i->i;
824 #pragma omp for schedule (guided, D.1598)
826 When we outline the parallel region, the call to the child function
827 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
828 that value is computed *after* the call site. So, in principle we
829 cannot do the transformation.
831 To see whether the code in WS_ENTRY_BB blocks the combined
832 parallel+workshare call, we collect all the variables used in the
833 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
834 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
837 FIXME. If we had the SSA form built at this point, we could merely
838 hoist the code in block 3 into block 2 and be done with it. But at
839 this point we don't have dataflow information and though we could
840 hack something up here, it is really not worth the aggravation. */
843 workshare_safe_to_combine_p (basic_block ws_entry_bb)
845 struct omp_for_data fd;
846 gimple *ws_stmt = last_stmt (ws_entry_bb);
848 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
851 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
853 extract_omp_for_data (as_a <gomp_for *> (ws_stmt), &fd, NULL);
855 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
857 if (fd.iter_type != long_integer_type_node)
860 /* FIXME. We give up too easily here. If any of these arguments
861 are not constants, they will likely involve variables that have
862 been mapped into fields of .omp_data_s for sharing with the child
863 function. With appropriate data flow, it would be possible to
865 if (!is_gimple_min_invariant (fd.loop.n1)
866 || !is_gimple_min_invariant (fd.loop.n2)
867 || !is_gimple_min_invariant (fd.loop.step)
868 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
875 static int omp_max_vf (void);
877 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
878 presence (SIMD_SCHEDULE). */
881 omp_adjust_chunk_size (tree chunk_size, bool simd_schedule)
886 int vf = omp_max_vf ();
890 tree type = TREE_TYPE (chunk_size);
891 chunk_size = fold_build2 (PLUS_EXPR, type, chunk_size,
892 build_int_cst (type, vf - 1));
893 return fold_build2 (BIT_AND_EXPR, type, chunk_size,
894 build_int_cst (type, -vf));
898 /* Collect additional arguments needed to emit a combined
899 parallel+workshare call. WS_STMT is the workshare directive being
902 static vec<tree, va_gc> *
903 get_ws_args_for (gimple *par_stmt, gimple *ws_stmt)
906 location_t loc = gimple_location (ws_stmt);
907 vec<tree, va_gc> *ws_args;
909 if (gomp_for *for_stmt = dyn_cast <gomp_for *> (ws_stmt))
911 struct omp_for_data fd;
914 extract_omp_for_data (for_stmt, &fd, NULL);
918 if (gimple_omp_for_combined_into_p (for_stmt))
921 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt),
922 OMP_CLAUSE__LOOPTEMP_);
924 n1 = OMP_CLAUSE_DECL (innerc);
925 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
926 OMP_CLAUSE__LOOPTEMP_);
928 n2 = OMP_CLAUSE_DECL (innerc);
931 vec_alloc (ws_args, 3 + (fd.chunk_size != 0));
933 t = fold_convert_loc (loc, long_integer_type_node, n1);
934 ws_args->quick_push (t);
936 t = fold_convert_loc (loc, long_integer_type_node, n2);
937 ws_args->quick_push (t);
939 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step);
940 ws_args->quick_push (t);
944 t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size);
945 t = omp_adjust_chunk_size (t, fd.simd_schedule);
946 ws_args->quick_push (t);
951 else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
953 /* Number of sections is equal to the number of edges from the
954 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
955 the exit of the sections region. */
956 basic_block bb = single_succ (gimple_bb (ws_stmt));
957 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
958 vec_alloc (ws_args, 1);
959 ws_args->quick_push (t);
967 /* Discover whether REGION is a combined parallel+workshare region. */
970 determine_parallel_type (struct omp_region *region)
972 basic_block par_entry_bb, par_exit_bb;
973 basic_block ws_entry_bb, ws_exit_bb;
975 if (region == NULL || region->inner == NULL
976 || region->exit == NULL || region->inner->exit == NULL
977 || region->inner->cont == NULL)
980 /* We only support parallel+for and parallel+sections. */
981 if (region->type != GIMPLE_OMP_PARALLEL
982 || (region->inner->type != GIMPLE_OMP_FOR
983 && region->inner->type != GIMPLE_OMP_SECTIONS))
986 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
987 WS_EXIT_BB -> PAR_EXIT_BB. */
988 par_entry_bb = region->entry;
989 par_exit_bb = region->exit;
990 ws_entry_bb = region->inner->entry;
991 ws_exit_bb = region->inner->exit;
993 if (single_succ (par_entry_bb) == ws_entry_bb
994 && single_succ (ws_exit_bb) == par_exit_bb
995 && workshare_safe_to_combine_p (ws_entry_bb)
996 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb))
997 || (last_and_only_stmt (ws_entry_bb)
998 && last_and_only_stmt (par_exit_bb))))
1000 gimple *par_stmt = last_stmt (par_entry_bb);
1001 gimple *ws_stmt = last_stmt (ws_entry_bb);
1003 if (region->inner->type == GIMPLE_OMP_FOR)
1005 /* If this is a combined parallel loop, we need to determine
1006 whether or not to use the combined library calls. There
1007 are two cases where we do not apply the transformation:
1008 static loops and any kind of ordered loop. In the first
1009 case, we already open code the loop so there is no need
1010 to do anything else. In the latter case, the combined
1011 parallel loop call would still need extra synchronization
1012 to implement ordered semantics, so there would not be any
1013 gain in using the combined call. */
1014 tree clauses = gimple_omp_for_clauses (ws_stmt);
1015 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE);
1017 || ((OMP_CLAUSE_SCHEDULE_KIND (c) & OMP_CLAUSE_SCHEDULE_MASK)
1018 == OMP_CLAUSE_SCHEDULE_STATIC)
1019 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED))
1021 region->is_combined_parallel = false;
1022 region->inner->is_combined_parallel = false;
1027 region->is_combined_parallel = true;
1028 region->inner->is_combined_parallel = true;
1029 region->ws_args = get_ws_args_for (par_stmt, ws_stmt);
1034 /* Return true if EXPR is variable sized. */
1037 is_variable_sized (const_tree expr)
1039 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
1042 /* Return true if DECL is a reference type. */
1045 is_reference (tree decl)
1047 return lang_hooks.decls.omp_privatize_by_reference (decl);
1050 /* Return the type of a decl. If the decl is reference type,
1051 return its base type. */
1053 get_base_type (tree decl)
1055 tree type = TREE_TYPE (decl);
1056 if (is_reference (decl))
1057 type = TREE_TYPE (type);
1061 /* Lookup variables. The "maybe" form
1062 allows for the variable form to not have been entered, otherwise we
1063 assert that the variable must have been entered. */
1066 lookup_decl (tree var, omp_context *ctx)
1068 tree *n = ctx->cb.decl_map->get (var);
1073 maybe_lookup_decl (const_tree var, omp_context *ctx)
1075 tree *n = ctx->cb.decl_map->get (const_cast<tree> (var));
1076 return n ? *n : NULL_TREE;
1080 lookup_field (tree var, omp_context *ctx)
1083 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
1084 return (tree) n->value;
1088 lookup_sfield (splay_tree_key key, omp_context *ctx)
1091 n = splay_tree_lookup (ctx->sfield_map
1092 ? ctx->sfield_map : ctx->field_map, key);
1093 return (tree) n->value;
1097 lookup_sfield (tree var, omp_context *ctx)
1099 return lookup_sfield ((splay_tree_key) var, ctx);
1103 maybe_lookup_field (splay_tree_key key, omp_context *ctx)
1106 n = splay_tree_lookup (ctx->field_map, key);
1107 return n ? (tree) n->value : NULL_TREE;
1111 maybe_lookup_field (tree var, omp_context *ctx)
1113 return maybe_lookup_field ((splay_tree_key) var, ctx);
1116 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1117 the parallel context if DECL is to be shared. */
1120 use_pointer_for_field (tree decl, omp_context *shared_ctx)
1122 if (AGGREGATE_TYPE_P (TREE_TYPE (decl)))
1125 /* We can only use copy-in/copy-out semantics for shared variables
1126 when we know the value is not accessible from an outer scope. */
1129 gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt));
1131 /* ??? Trivially accessible from anywhere. But why would we even
1132 be passing an address in this case? Should we simply assert
1133 this to be false, or should we have a cleanup pass that removes
1134 these from the list of mappings? */
1135 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
1138 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1139 without analyzing the expression whether or not its location
1140 is accessible to anyone else. In the case of nested parallel
1141 regions it certainly may be. */
1142 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
1145 /* Do not use copy-in/copy-out for variables that have their
1147 if (TREE_ADDRESSABLE (decl))
1150 /* lower_send_shared_vars only uses copy-in, but not copy-out
1152 if (TREE_READONLY (decl)
1153 || ((TREE_CODE (decl) == RESULT_DECL
1154 || TREE_CODE (decl) == PARM_DECL)
1155 && DECL_BY_REFERENCE (decl)))
1158 /* Disallow copy-in/out in nested parallel if
1159 decl is shared in outer parallel, otherwise
1160 each thread could store the shared variable
1161 in its own copy-in location, making the
1162 variable no longer really shared. */
1163 if (shared_ctx->is_nested)
1167 for (up = shared_ctx->outer; up; up = up->outer)
1168 if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up))
1175 for (c = gimple_omp_taskreg_clauses (up->stmt);
1176 c; c = OMP_CLAUSE_CHAIN (c))
1177 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
1178 && OMP_CLAUSE_DECL (c) == decl)
1182 goto maybe_mark_addressable_and_ret;
1186 /* For tasks avoid using copy-in/out. As tasks can be
1187 deferred or executed in different thread, when GOMP_task
1188 returns, the task hasn't necessarily terminated. */
1189 if (is_task_ctx (shared_ctx))
1192 maybe_mark_addressable_and_ret:
1193 outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
1194 if (is_gimple_reg (outer) && !omp_member_access_dummy_var (outer))
1196 /* Taking address of OUTER in lower_send_shared_vars
1197 might need regimplification of everything that uses the
1199 if (!task_shared_vars)
1200 task_shared_vars = BITMAP_ALLOC (NULL);
1201 bitmap_set_bit (task_shared_vars, DECL_UID (outer));
1202 TREE_ADDRESSABLE (outer) = 1;
1211 /* Construct a new automatic decl similar to VAR. */
1214 omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
1216 tree copy = copy_var_decl (var, name, type);
1218 DECL_CONTEXT (copy) = current_function_decl;
1219 DECL_CHAIN (copy) = ctx->block_vars;
1220 /* If VAR is listed in task_shared_vars, it means it wasn't
1221 originally addressable and is just because task needs to take
1222 it's address. But we don't need to take address of privatizations
1224 if (TREE_ADDRESSABLE (var)
1226 && bitmap_bit_p (task_shared_vars, DECL_UID (var)))
1227 TREE_ADDRESSABLE (copy) = 0;
1228 ctx->block_vars = copy;
1234 omp_copy_decl_1 (tree var, omp_context *ctx)
1236 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
1239 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1242 omp_build_component_ref (tree obj, tree field)
1244 tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL);
1245 if (TREE_THIS_VOLATILE (field))
1246 TREE_THIS_VOLATILE (ret) |= 1;
1247 if (TREE_READONLY (field))
1248 TREE_READONLY (ret) |= 1;
1252 /* Build tree nodes to access the field for VAR on the receiver side. */
1255 build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
1257 tree x, field = lookup_field (var, ctx);
1259 /* If the receiver record type was remapped in the child function,
1260 remap the field into the new record type. */
1261 x = maybe_lookup_field (field, ctx);
1265 x = build_simple_mem_ref (ctx->receiver_decl);
1266 TREE_THIS_NOTRAP (x) = 1;
1267 x = omp_build_component_ref (x, field);
1270 x = build_simple_mem_ref (x);
1271 TREE_THIS_NOTRAP (x) = 1;
1277 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1278 of a parallel, this is a component reference; for workshare constructs
1279 this is some variable. */
1282 build_outer_var_ref (tree var, omp_context *ctx,
1283 enum omp_clause_code code = OMP_CLAUSE_ERROR)
1287 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
1289 else if (is_variable_sized (var))
1291 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
1292 x = build_outer_var_ref (x, ctx, code);
1293 x = build_simple_mem_ref (x);
1295 else if (is_taskreg_ctx (ctx))
1297 bool by_ref = use_pointer_for_field (var, NULL);
1298 x = build_receiver_ref (var, by_ref, ctx);
1300 else if ((gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1301 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
1302 || (code == OMP_CLAUSE_PRIVATE
1303 && (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1304 || gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS
1305 || gimple_code (ctx->stmt) == GIMPLE_OMP_SINGLE)))
1307 /* #pragma omp simd isn't a worksharing construct, and can reference
1308 even private vars in its linear etc. clauses.
1309 Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer
1310 to private vars in all worksharing constructs. */
1312 if (ctx->outer && is_taskreg_ctx (ctx))
1313 x = lookup_decl (var, ctx->outer);
1314 else if (ctx->outer)
1315 x = maybe_lookup_decl_in_outer_ctx (var, ctx);
1319 else if (code == OMP_CLAUSE_LASTPRIVATE && is_taskloop_ctx (ctx))
1321 gcc_assert (ctx->outer);
1323 = splay_tree_lookup (ctx->outer->field_map,
1324 (splay_tree_key) &DECL_UID (var));
1327 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx->outer)))
1330 x = lookup_decl (var, ctx->outer);
1334 tree field = (tree) n->value;
1335 /* If the receiver record type was remapped in the child function,
1336 remap the field into the new record type. */
1337 x = maybe_lookup_field (field, ctx->outer);
1341 x = build_simple_mem_ref (ctx->outer->receiver_decl);
1342 x = omp_build_component_ref (x, field);
1343 if (use_pointer_for_field (var, ctx->outer))
1344 x = build_simple_mem_ref (x);
1347 else if (ctx->outer)
1349 omp_context *outer = ctx->outer;
1350 if (gimple_code (outer->stmt) == GIMPLE_OMP_GRID_BODY)
1352 outer = outer->outer;
1354 && gimple_code (outer->stmt) != GIMPLE_OMP_GRID_BODY);
1356 x = lookup_decl (var, outer);
1358 else if (is_reference (var))
1359 /* This can happen with orphaned constructs. If var is reference, it is
1360 possible it is shared and as such valid. */
1362 else if (omp_member_access_dummy_var (var))
1369 tree t = omp_member_access_dummy_var (var);
1372 x = DECL_VALUE_EXPR (var);
1373 tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
1375 x = unshare_and_remap (x, t, o);
1377 x = unshare_expr (x);
1381 if (is_reference (var))
1382 x = build_simple_mem_ref (x);
1387 /* Build tree nodes to access the field for VAR on the sender side. */
1390 build_sender_ref (splay_tree_key key, omp_context *ctx)
1392 tree field = lookup_sfield (key, ctx);
1393 return omp_build_component_ref (ctx->sender_decl, field);
1397 build_sender_ref (tree var, omp_context *ctx)
1399 return build_sender_ref ((splay_tree_key) var, ctx);
1402 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. If
1403 BASE_POINTERS_RESTRICT, declare the field with restrict. */
1406 install_var_field (tree var, bool by_ref, int mask, omp_context *ctx,
1407 bool base_pointers_restrict = false)
1409 tree field, type, sfield = NULL_TREE;
1410 splay_tree_key key = (splay_tree_key) var;
1412 if ((mask & 8) != 0)
1414 key = (splay_tree_key) &DECL_UID (var);
1415 gcc_checking_assert (key != (splay_tree_key) var);
1417 gcc_assert ((mask & 1) == 0
1418 || !splay_tree_lookup (ctx->field_map, key));
1419 gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
1420 || !splay_tree_lookup (ctx->sfield_map, key));
1421 gcc_assert ((mask & 3) == 3
1422 || !is_gimple_omp_oacc (ctx->stmt));
1424 type = TREE_TYPE (var);
1425 /* Prevent redeclaring the var in the split-off function with a restrict
1426 pointer type. Note that we only clear type itself, restrict qualifiers in
1427 the pointed-to type will be ignored by points-to analysis. */
1428 if (POINTER_TYPE_P (type)
1429 && TYPE_RESTRICT (type))
1430 type = build_qualified_type (type, TYPE_QUALS (type) & ~TYPE_QUAL_RESTRICT);
1434 gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
1435 type = build_pointer_type (build_pointer_type (type));
1439 type = build_pointer_type (type);
1440 if (base_pointers_restrict)
1441 type = build_qualified_type (type, TYPE_QUAL_RESTRICT);
1443 else if ((mask & 3) == 1 && is_reference (var))
1444 type = TREE_TYPE (type);
1446 field = build_decl (DECL_SOURCE_LOCATION (var),
1447 FIELD_DECL, DECL_NAME (var), type);
1449 /* Remember what variable this field was created for. This does have a
1450 side effect of making dwarf2out ignore this member, so for helpful
1451 debugging we clear it later in delete_omp_context. */
1452 DECL_ABSTRACT_ORIGIN (field) = var;
1453 if (type == TREE_TYPE (var))
1455 SET_DECL_ALIGN (field, DECL_ALIGN (var));
1456 DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
1457 TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
1460 SET_DECL_ALIGN (field, TYPE_ALIGN (type));
1462 if ((mask & 3) == 3)
1464 insert_field_into_struct (ctx->record_type, field);
1465 if (ctx->srecord_type)
1467 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1468 FIELD_DECL, DECL_NAME (var), type);
1469 DECL_ABSTRACT_ORIGIN (sfield) = var;
1470 SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
1471 DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
1472 TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
1473 insert_field_into_struct (ctx->srecord_type, sfield);
1478 if (ctx->srecord_type == NULL_TREE)
1482 ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
1483 ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1484 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
1486 sfield = build_decl (DECL_SOURCE_LOCATION (t),
1487 FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
1488 DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
1489 insert_field_into_struct (ctx->srecord_type, sfield);
1490 splay_tree_insert (ctx->sfield_map,
1491 (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
1492 (splay_tree_value) sfield);
1496 insert_field_into_struct ((mask & 1) ? ctx->record_type
1497 : ctx->srecord_type, field);
1501 splay_tree_insert (ctx->field_map, key, (splay_tree_value) field);
1502 if ((mask & 2) && ctx->sfield_map)
1503 splay_tree_insert (ctx->sfield_map, key, (splay_tree_value) sfield);
1507 install_var_local (tree var, omp_context *ctx)
1509 tree new_var = omp_copy_decl_1 (var, ctx);
1510 insert_decl_map (&ctx->cb, var, new_var);
1514 /* Adjust the replacement for DECL in CTX for the new context. This means
1515 copying the DECL_VALUE_EXPR, and fixing up the type. */
1518 fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
1520 tree new_decl, size;
1522 new_decl = lookup_decl (decl, ctx);
1524 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
1526 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
1527 && DECL_HAS_VALUE_EXPR_P (decl))
1529 tree ve = DECL_VALUE_EXPR (decl);
1530 walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
1531 SET_DECL_VALUE_EXPR (new_decl, ve);
1532 DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
1535 if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
1537 size = remap_decl (DECL_SIZE (decl), &ctx->cb);
1538 if (size == error_mark_node)
1539 size = TYPE_SIZE (TREE_TYPE (new_decl));
1540 DECL_SIZE (new_decl) = size;
1542 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
1543 if (size == error_mark_node)
1544 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
1545 DECL_SIZE_UNIT (new_decl) = size;
1549 /* The callback for remap_decl. Search all containing contexts for a
1550 mapping of the variable; this avoids having to duplicate the splay
1551 tree ahead of time. We know a mapping doesn't already exist in the
1552 given context. Create new mappings to implement default semantics. */
1555 omp_copy_decl (tree var, copy_body_data *cb)
1557 omp_context *ctx = (omp_context *) cb;
1560 if (TREE_CODE (var) == LABEL_DECL)
1562 new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
1563 DECL_CONTEXT (new_var) = current_function_decl;
1564 insert_decl_map (&ctx->cb, var, new_var);
1568 while (!is_taskreg_ctx (ctx))
1573 new_var = maybe_lookup_decl (var, ctx);
1578 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
1581 return error_mark_node;
1585 /* Debugging dumps for parallel regions. */
1586 void dump_omp_region (FILE *, struct omp_region *, int);
1587 void debug_omp_region (struct omp_region *);
1588 void debug_all_omp_regions (void);
1590 /* Dump the parallel region tree rooted at REGION. */
1593 dump_omp_region (FILE *file, struct omp_region *region, int indent)
1595 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
1596 gimple_code_name[region->type]);
1599 dump_omp_region (file, region->inner, indent + 4);
1603 fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "",
1604 region->cont->index);
1608 fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "",
1609 region->exit->index);
1611 fprintf (file, "%*s[no exit marker]\n", indent, "");
1614 dump_omp_region (file, region->next, indent);
1618 debug_omp_region (struct omp_region *region)
1620 dump_omp_region (stderr, region, 0);
1624 debug_all_omp_regions (void)
1626 dump_omp_region (stderr, root_omp_region, 0);
1630 /* Create a new parallel region starting at STMT inside region PARENT. */
1632 static struct omp_region *
1633 new_omp_region (basic_block bb, enum gimple_code type,
1634 struct omp_region *parent)
1636 struct omp_region *region = XCNEW (struct omp_region);
1638 region->outer = parent;
1640 region->type = type;
1644 /* This is a nested region. Add it to the list of inner
1645 regions in PARENT. */
1646 region->next = parent->inner;
1647 parent->inner = region;
1651 /* This is a toplevel region. Add it to the list of toplevel
1652 regions in ROOT_OMP_REGION. */
1653 region->next = root_omp_region;
1654 root_omp_region = region;
1660 /* Release the memory associated with the region tree rooted at REGION. */
1663 free_omp_region_1 (struct omp_region *region)
1665 struct omp_region *i, *n;
1667 for (i = region->inner; i ; i = n)
1670 free_omp_region_1 (i);
1676 /* Release the memory for the entire omp region tree. */
1679 free_omp_regions (void)
1681 struct omp_region *r, *n;
1682 for (r = root_omp_region; r ; r = n)
1685 free_omp_region_1 (r);
1687 root_omp_region = NULL;
1691 /* Create a new context, with OUTER_CTX being the surrounding context. */
1693 static omp_context *
1694 new_omp_context (gimple *stmt, omp_context *outer_ctx)
1696 omp_context *ctx = XCNEW (omp_context);
1698 splay_tree_insert (all_contexts, (splay_tree_key) stmt,
1699 (splay_tree_value) ctx);
1704 ctx->outer = outer_ctx;
1705 ctx->cb = outer_ctx->cb;
1706 ctx->cb.block = NULL;
1707 ctx->depth = outer_ctx->depth + 1;
1711 ctx->cb.src_fn = current_function_decl;
1712 ctx->cb.dst_fn = current_function_decl;
1713 ctx->cb.src_node = cgraph_node::get (current_function_decl);
1714 gcc_checking_assert (ctx->cb.src_node);
1715 ctx->cb.dst_node = ctx->cb.src_node;
1716 ctx->cb.src_cfun = cfun;
1717 ctx->cb.copy_decl = omp_copy_decl;
1718 ctx->cb.eh_lp_nr = 0;
1719 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1723 ctx->cb.decl_map = new hash_map<tree, tree>;
1728 static gimple_seq maybe_catch_exception (gimple_seq);
1730 /* Finalize task copyfn. */
1733 finalize_task_copyfn (gomp_task *task_stmt)
1735 struct function *child_cfun;
1737 gimple_seq seq = NULL, new_seq;
1740 child_fn = gimple_omp_task_copy_fn (task_stmt);
1741 if (child_fn == NULL_TREE)
1744 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1745 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1747 push_cfun (child_cfun);
1748 bind = gimplify_body (child_fn, false);
1749 gimple_seq_add_stmt (&seq, bind);
1750 new_seq = maybe_catch_exception (seq);
1753 bind = gimple_build_bind (NULL, new_seq, NULL);
1755 gimple_seq_add_stmt (&seq, bind);
1757 gimple_set_body (child_fn, seq);
1760 /* Inform the callgraph about the new function. */
1761 cgraph_node *node = cgraph_node::get_create (child_fn);
1762 node->parallelized_function = 1;
1763 cgraph_node::add_new_function (child_fn, false);
1766 /* Destroy a omp_context data structures. Called through the splay tree
1767 value delete callback. */
1770 delete_omp_context (splay_tree_value value)
1772 omp_context *ctx = (omp_context *) value;
1774 delete ctx->cb.decl_map;
1777 splay_tree_delete (ctx->field_map);
1778 if (ctx->sfield_map)
1779 splay_tree_delete (ctx->sfield_map);
1781 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1782 it produces corrupt debug information. */
1783 if (ctx->record_type)
1786 for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1787 DECL_ABSTRACT_ORIGIN (t) = NULL;
1789 if (ctx->srecord_type)
1792 for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1793 DECL_ABSTRACT_ORIGIN (t) = NULL;
1796 if (is_task_ctx (ctx))
1797 finalize_task_copyfn (as_a <gomp_task *> (ctx->stmt));
1802 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1806 fixup_child_record_type (omp_context *ctx)
1808 tree f, type = ctx->record_type;
1810 if (!ctx->receiver_decl)
1812 /* ??? It isn't sufficient to just call remap_type here, because
1813 variably_modified_type_p doesn't work the way we expect for
1814 record types. Testing each field for whether it needs remapping
1815 and creating a new record by hand works, however. */
1816 for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1817 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1821 tree name, new_fields = NULL;
1823 type = lang_hooks.types.make_type (RECORD_TYPE);
1824 name = DECL_NAME (TYPE_NAME (ctx->record_type));
1825 name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1826 TYPE_DECL, name, type);
1827 TYPE_NAME (type) = name;
1829 for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1831 tree new_f = copy_node (f);
1832 DECL_CONTEXT (new_f) = type;
1833 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1834 DECL_CHAIN (new_f) = new_fields;
1835 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1836 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1838 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1842 /* Arrange to be able to look up the receiver field
1843 given the sender field. */
1844 splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1845 (splay_tree_value) new_f);
1847 TYPE_FIELDS (type) = nreverse (new_fields);
1851 /* In a target region we never modify any of the pointers in *.omp_data_i,
1852 so attempt to help the optimizers. */
1853 if (is_gimple_omp_offloaded (ctx->stmt))
1854 type = build_qualified_type (type, TYPE_QUAL_CONST);
1856 TREE_TYPE (ctx->receiver_decl)
1857 = build_qualified_type (build_reference_type (type), TYPE_QUAL_RESTRICT);
1860 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1861 specified by CLAUSES. If BASE_POINTERS_RESTRICT, install var field with
1865 scan_sharing_clauses (tree clauses, omp_context *ctx,
1866 bool base_pointers_restrict = false)
1869 bool scan_array_reductions = false;
1871 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1875 switch (OMP_CLAUSE_CODE (c))
1877 case OMP_CLAUSE_PRIVATE:
1878 decl = OMP_CLAUSE_DECL (c);
1879 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1881 else if (!is_variable_sized (decl))
1882 install_var_local (decl, ctx);
1885 case OMP_CLAUSE_SHARED:
1886 decl = OMP_CLAUSE_DECL (c);
1887 /* Ignore shared directives in teams construct. */
1888 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1890 /* Global variables don't need to be copied,
1891 the receiver side will use them directly. */
1892 tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx);
1893 if (is_global_var (odecl))
1895 insert_decl_map (&ctx->cb, decl, odecl);
1898 gcc_assert (is_taskreg_ctx (ctx));
1899 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1900 || !is_variable_sized (decl));
1901 /* Global variables don't need to be copied,
1902 the receiver side will use them directly. */
1903 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1905 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1907 use_pointer_for_field (decl, ctx);
1910 by_ref = use_pointer_for_field (decl, NULL);
1911 if ((! TREE_READONLY (decl) && !OMP_CLAUSE_SHARED_READONLY (c))
1912 || TREE_ADDRESSABLE (decl)
1914 || is_reference (decl))
1916 by_ref = use_pointer_for_field (decl, ctx);
1917 install_var_field (decl, by_ref, 3, ctx);
1918 install_var_local (decl, ctx);
1921 /* We don't need to copy const scalar vars back. */
1922 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1925 case OMP_CLAUSE_REDUCTION:
1926 decl = OMP_CLAUSE_DECL (c);
1927 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1928 && TREE_CODE (decl) == MEM_REF)
1930 tree t = TREE_OPERAND (decl, 0);
1931 if (TREE_CODE (t) == POINTER_PLUS_EXPR)
1932 t = TREE_OPERAND (t, 0);
1933 if (TREE_CODE (t) == INDIRECT_REF
1934 || TREE_CODE (t) == ADDR_EXPR)
1935 t = TREE_OPERAND (t, 0);
1936 install_var_local (t, ctx);
1937 if (is_taskreg_ctx (ctx)
1938 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t, ctx))
1939 && !is_variable_sized (t))
1941 by_ref = use_pointer_for_field (t, ctx);
1942 install_var_field (t, by_ref, 3, ctx);
1948 case OMP_CLAUSE_LASTPRIVATE:
1949 /* Let the corresponding firstprivate clause create
1951 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1955 case OMP_CLAUSE_FIRSTPRIVATE:
1956 case OMP_CLAUSE_LINEAR:
1957 decl = OMP_CLAUSE_DECL (c);
1959 if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1960 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR)
1961 && is_gimple_omp_offloaded (ctx->stmt))
1963 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
1964 install_var_field (decl, !is_reference (decl), 3, ctx);
1965 else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1966 install_var_field (decl, true, 3, ctx);
1968 install_var_field (decl, false, 3, ctx);
1970 if (is_variable_sized (decl))
1972 if (is_task_ctx (ctx))
1973 install_var_field (decl, false, 1, ctx);
1976 else if (is_taskreg_ctx (ctx))
1979 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1980 by_ref = use_pointer_for_field (decl, NULL);
1982 if (is_task_ctx (ctx)
1983 && (global || by_ref || is_reference (decl)))
1985 install_var_field (decl, false, 1, ctx);
1987 install_var_field (decl, by_ref, 2, ctx);
1990 install_var_field (decl, by_ref, 3, ctx);
1992 install_var_local (decl, ctx);
1995 case OMP_CLAUSE_USE_DEVICE_PTR:
1996 decl = OMP_CLAUSE_DECL (c);
1997 if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1998 install_var_field (decl, true, 3, ctx);
2000 install_var_field (decl, false, 3, ctx);
2001 if (DECL_SIZE (decl)
2002 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
2004 tree decl2 = DECL_VALUE_EXPR (decl);
2005 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
2006 decl2 = TREE_OPERAND (decl2, 0);
2007 gcc_assert (DECL_P (decl2));
2008 install_var_local (decl2, ctx);
2010 install_var_local (decl, ctx);
2013 case OMP_CLAUSE_IS_DEVICE_PTR:
2014 decl = OMP_CLAUSE_DECL (c);
2017 case OMP_CLAUSE__LOOPTEMP_:
2018 gcc_assert (is_taskreg_ctx (ctx));
2019 decl = OMP_CLAUSE_DECL (c);
2020 install_var_field (decl, false, 3, ctx);
2021 install_var_local (decl, ctx);
2024 case OMP_CLAUSE_COPYPRIVATE:
2025 case OMP_CLAUSE_COPYIN:
2026 decl = OMP_CLAUSE_DECL (c);
2027 by_ref = use_pointer_for_field (decl, NULL);
2028 install_var_field (decl, by_ref, 3, ctx);
2031 case OMP_CLAUSE_DEFAULT:
2032 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c);
2035 case OMP_CLAUSE_FINAL:
2037 case OMP_CLAUSE_NUM_THREADS:
2038 case OMP_CLAUSE_NUM_TEAMS:
2039 case OMP_CLAUSE_THREAD_LIMIT:
2040 case OMP_CLAUSE_DEVICE:
2041 case OMP_CLAUSE_SCHEDULE:
2042 case OMP_CLAUSE_DIST_SCHEDULE:
2043 case OMP_CLAUSE_DEPEND:
2044 case OMP_CLAUSE_PRIORITY:
2045 case OMP_CLAUSE_GRAINSIZE:
2046 case OMP_CLAUSE_NUM_TASKS:
2047 case OMP_CLAUSE__CILK_FOR_COUNT_:
2048 case OMP_CLAUSE_NUM_GANGS:
2049 case OMP_CLAUSE_NUM_WORKERS:
2050 case OMP_CLAUSE_VECTOR_LENGTH:
2052 scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
2056 case OMP_CLAUSE_FROM:
2057 case OMP_CLAUSE_MAP:
2059 scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
2060 decl = OMP_CLAUSE_DECL (c);
2061 /* Global variables with "omp declare target" attribute
2062 don't need to be copied, the receiver side will use them
2063 directly. However, global variables with "omp declare target link"
2064 attribute need to be copied. */
2065 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
2067 && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
2068 && (OMP_CLAUSE_MAP_KIND (c)
2069 != GOMP_MAP_FIRSTPRIVATE_REFERENCE))
2070 || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
2071 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
2072 && varpool_node::get_create (decl)->offloadable
2073 && !lookup_attribute ("omp declare target link",
2074 DECL_ATTRIBUTES (decl)))
2076 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
2077 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER)
2079 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2080 not offloaded; there is nothing to map for those. */
2081 if (!is_gimple_omp_offloaded (ctx->stmt)
2082 && !POINTER_TYPE_P (TREE_TYPE (decl))
2083 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
2086 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
2087 && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
2088 || (OMP_CLAUSE_MAP_KIND (c)
2089 == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
2091 if (TREE_CODE (decl) == COMPONENT_REF
2092 || (TREE_CODE (decl) == INDIRECT_REF
2093 && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
2094 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
2095 == REFERENCE_TYPE)))
2097 if (DECL_SIZE (decl)
2098 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
2100 tree decl2 = DECL_VALUE_EXPR (decl);
2101 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
2102 decl2 = TREE_OPERAND (decl2, 0);
2103 gcc_assert (DECL_P (decl2));
2104 install_var_local (decl2, ctx);
2106 install_var_local (decl, ctx);
2111 if (DECL_SIZE (decl)
2112 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
2114 tree decl2 = DECL_VALUE_EXPR (decl);
2115 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
2116 decl2 = TREE_OPERAND (decl2, 0);
2117 gcc_assert (DECL_P (decl2));
2118 install_var_field (decl2, true, 3, ctx);
2119 install_var_local (decl2, ctx);
2120 install_var_local (decl, ctx);
2124 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
2125 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
2126 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
2127 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
2128 install_var_field (decl, true, 7, ctx);
2130 install_var_field (decl, true, 3, ctx,
2131 base_pointers_restrict);
2132 if (is_gimple_omp_offloaded (ctx->stmt)
2133 && !OMP_CLAUSE_MAP_IN_REDUCTION (c))
2134 install_var_local (decl, ctx);
2139 tree base = get_base_address (decl);
2140 tree nc = OMP_CLAUSE_CHAIN (c);
2143 && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
2144 && OMP_CLAUSE_DECL (nc) == base
2145 && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER
2146 && integer_zerop (OMP_CLAUSE_SIZE (nc)))
2148 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
2149 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
2155 scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer);
2156 decl = OMP_CLAUSE_DECL (c);
2158 gcc_assert (!splay_tree_lookup (ctx->field_map,
2159 (splay_tree_key) decl));
2161 = build_decl (OMP_CLAUSE_LOCATION (c),
2162 FIELD_DECL, NULL_TREE, ptr_type_node);
2163 SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
2164 insert_field_into_struct (ctx->record_type, field);
2165 splay_tree_insert (ctx->field_map, (splay_tree_key) decl,
2166 (splay_tree_value) field);
2171 case OMP_CLAUSE__GRIDDIM_:
2174 scan_omp_op (&OMP_CLAUSE__GRIDDIM__SIZE (c), ctx->outer);
2175 scan_omp_op (&OMP_CLAUSE__GRIDDIM__GROUP (c), ctx->outer);
2179 case OMP_CLAUSE_NOWAIT:
2180 case OMP_CLAUSE_ORDERED:
2181 case OMP_CLAUSE_COLLAPSE:
2182 case OMP_CLAUSE_UNTIED:
2183 case OMP_CLAUSE_MERGEABLE:
2184 case OMP_CLAUSE_PROC_BIND:
2185 case OMP_CLAUSE_SAFELEN:
2186 case OMP_CLAUSE_SIMDLEN:
2187 case OMP_CLAUSE_THREADS:
2188 case OMP_CLAUSE_SIMD:
2189 case OMP_CLAUSE_NOGROUP:
2190 case OMP_CLAUSE_DEFAULTMAP:
2191 case OMP_CLAUSE_ASYNC:
2192 case OMP_CLAUSE_WAIT:
2193 case OMP_CLAUSE_GANG:
2194 case OMP_CLAUSE_WORKER:
2195 case OMP_CLAUSE_VECTOR:
2196 case OMP_CLAUSE_INDEPENDENT:
2197 case OMP_CLAUSE_AUTO:
2198 case OMP_CLAUSE_SEQ:
2201 case OMP_CLAUSE_ALIGNED:
2202 decl = OMP_CLAUSE_DECL (c);
2203 if (is_global_var (decl)
2204 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
2205 install_var_local (decl, ctx);
2208 case OMP_CLAUSE_DEVICE_RESIDENT:
2209 case OMP_CLAUSE_TILE:
2210 case OMP_CLAUSE__CACHE_:
2216 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2218 switch (OMP_CLAUSE_CODE (c))
2220 case OMP_CLAUSE_LASTPRIVATE:
2221 /* Let the corresponding firstprivate clause create
2223 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2224 scan_array_reductions = true;
2225 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
2229 case OMP_CLAUSE_FIRSTPRIVATE:
2230 case OMP_CLAUSE_PRIVATE:
2231 case OMP_CLAUSE_LINEAR:
2232 case OMP_CLAUSE_IS_DEVICE_PTR:
2233 decl = OMP_CLAUSE_DECL (c);
2234 if (is_variable_sized (decl))
2236 if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
2237 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR)
2238 && is_gimple_omp_offloaded (ctx->stmt))
2240 tree decl2 = DECL_VALUE_EXPR (decl);
2241 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
2242 decl2 = TREE_OPERAND (decl2, 0);
2243 gcc_assert (DECL_P (decl2));
2244 install_var_local (decl2, ctx);
2245 fixup_remapped_decl (decl2, ctx, false);
2247 install_var_local (decl, ctx);
2249 fixup_remapped_decl (decl, ctx,
2250 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
2251 && OMP_CLAUSE_PRIVATE_DEBUG (c));
2252 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2253 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2254 scan_array_reductions = true;
2257 case OMP_CLAUSE_REDUCTION:
2258 decl = OMP_CLAUSE_DECL (c);
2259 if (TREE_CODE (decl) != MEM_REF)
2261 if (is_variable_sized (decl))
2262 install_var_local (decl, ctx);
2263 fixup_remapped_decl (decl, ctx, false);
2265 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2266 scan_array_reductions = true;
2269 case OMP_CLAUSE_SHARED:
2270 /* Ignore shared directives in teams construct. */
2271 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2273 decl = OMP_CLAUSE_DECL (c);
2274 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2276 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
2278 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl,
2281 bool by_ref = use_pointer_for_field (decl, ctx);
2282 install_var_field (decl, by_ref, 11, ctx);
2285 fixup_remapped_decl (decl, ctx, false);
2288 case OMP_CLAUSE_MAP:
2289 if (!is_gimple_omp_offloaded (ctx->stmt))
2291 decl = OMP_CLAUSE_DECL (c);
2293 && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
2294 && (OMP_CLAUSE_MAP_KIND (c)
2295 != GOMP_MAP_FIRSTPRIVATE_REFERENCE))
2296 || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
2297 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
2298 && varpool_node::get_create (decl)->offloadable)
2302 if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
2303 || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER)
2304 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
2305 && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
2307 tree new_decl = lookup_decl (decl, ctx);
2308 TREE_TYPE (new_decl)
2309 = remap_type (TREE_TYPE (decl), &ctx->cb);
2311 else if (DECL_SIZE (decl)
2312 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
2314 tree decl2 = DECL_VALUE_EXPR (decl);
2315 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
2316 decl2 = TREE_OPERAND (decl2, 0);
2317 gcc_assert (DECL_P (decl2));
2318 fixup_remapped_decl (decl2, ctx, false);
2319 fixup_remapped_decl (decl, ctx, true);
2322 fixup_remapped_decl (decl, ctx, false);
2326 case OMP_CLAUSE_COPYPRIVATE:
2327 case OMP_CLAUSE_COPYIN:
2328 case OMP_CLAUSE_DEFAULT:
2330 case OMP_CLAUSE_NUM_THREADS:
2331 case OMP_CLAUSE_NUM_TEAMS:
2332 case OMP_CLAUSE_THREAD_LIMIT:
2333 case OMP_CLAUSE_DEVICE:
2334 case OMP_CLAUSE_SCHEDULE:
2335 case OMP_CLAUSE_DIST_SCHEDULE:
2336 case OMP_CLAUSE_NOWAIT:
2337 case OMP_CLAUSE_ORDERED:
2338 case OMP_CLAUSE_COLLAPSE:
2339 case OMP_CLAUSE_UNTIED:
2340 case OMP_CLAUSE_FINAL:
2341 case OMP_CLAUSE_MERGEABLE:
2342 case OMP_CLAUSE_PROC_BIND:
2343 case OMP_CLAUSE_SAFELEN:
2344 case OMP_CLAUSE_SIMDLEN:
2345 case OMP_CLAUSE_ALIGNED:
2346 case OMP_CLAUSE_DEPEND:
2347 case OMP_CLAUSE__LOOPTEMP_:
2349 case OMP_CLAUSE_FROM:
2350 case OMP_CLAUSE_PRIORITY:
2351 case OMP_CLAUSE_GRAINSIZE:
2352 case OMP_CLAUSE_NUM_TASKS:
2353 case OMP_CLAUSE_THREADS:
2354 case OMP_CLAUSE_SIMD:
2355 case OMP_CLAUSE_NOGROUP:
2356 case OMP_CLAUSE_DEFAULTMAP:
2357 case OMP_CLAUSE_USE_DEVICE_PTR:
2358 case OMP_CLAUSE__CILK_FOR_COUNT_:
2359 case OMP_CLAUSE_ASYNC:
2360 case OMP_CLAUSE_WAIT:
2361 case OMP_CLAUSE_NUM_GANGS:
2362 case OMP_CLAUSE_NUM_WORKERS:
2363 case OMP_CLAUSE_VECTOR_LENGTH:
2364 case OMP_CLAUSE_GANG:
2365 case OMP_CLAUSE_WORKER:
2366 case OMP_CLAUSE_VECTOR:
2367 case OMP_CLAUSE_INDEPENDENT:
2368 case OMP_CLAUSE_AUTO:
2369 case OMP_CLAUSE_SEQ:
2370 case OMP_CLAUSE__GRIDDIM_:
2373 case OMP_CLAUSE_DEVICE_RESIDENT:
2374 case OMP_CLAUSE_TILE:
2375 case OMP_CLAUSE__CACHE_:
2381 gcc_checking_assert (!scan_array_reductions
2382 || !is_gimple_omp_oacc (ctx->stmt));
2383 if (scan_array_reductions)
2385 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2386 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
2387 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2389 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx);
2390 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
2392 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
2393 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2394 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
2395 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2396 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2397 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
2401 /* Create a new name for omp child function. Returns an identifier. If
2402 IS_CILK_FOR is true then the suffix for the child function is
2406 create_omp_child_function_name (bool task_copy, bool is_cilk_for)
2409 return clone_function_name (current_function_decl, "_cilk_for_fn");
2410 return clone_function_name (current_function_decl,
2411 task_copy ? "_omp_cpyfn" : "_omp_fn");
2414 /* Returns the type of the induction variable for the child function for
2415 _Cilk_for and the types for _high and _low variables based on TYPE. */
2418 cilk_for_check_loop_diff_type (tree type)
2420 if (TYPE_PRECISION (type) <= TYPE_PRECISION (uint32_type_node))
2422 if (TYPE_UNSIGNED (type))
2423 return uint32_type_node;
2425 return integer_type_node;
2429 if (TYPE_UNSIGNED (type))
2430 return uint64_type_node;
2432 return long_long_integer_type_node;
2436 /* Build a decl for the omp child function. It'll not contain a body
2437 yet, just the bare decl. */
2440 create_omp_child_function (omp_context *ctx, bool task_copy)
2442 tree decl, type, name, t;
2445 = (flag_cilkplus && gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2446 ? find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
2447 OMP_CLAUSE__CILK_FOR_COUNT_) : NULL_TREE;
2448 tree cilk_var_type = NULL_TREE;
2450 name = create_omp_child_function_name (task_copy,
2451 cilk_for_count != NULL_TREE);
2453 type = build_function_type_list (void_type_node, ptr_type_node,
2454 ptr_type_node, NULL_TREE);
2455 else if (cilk_for_count)
2457 type = TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count, 0));
2458 cilk_var_type = cilk_for_check_loop_diff_type (type);
2459 type = build_function_type_list (void_type_node, ptr_type_node,
2460 cilk_var_type, cilk_var_type, NULL_TREE);
2463 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
2465 decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type);
2467 gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt)
2470 ctx->cb.dst_fn = decl;
2472 gimple_omp_task_set_copy_fn (ctx->stmt, decl);
2474 TREE_STATIC (decl) = 1;
2475 TREE_USED (decl) = 1;
2476 DECL_ARTIFICIAL (decl) = 1;
2477 DECL_IGNORED_P (decl) = 0;
2478 TREE_PUBLIC (decl) = 0;
2479 DECL_UNINLINABLE (decl) = 1;
2480 DECL_EXTERNAL (decl) = 0;
2481 DECL_CONTEXT (decl) = NULL_TREE;
2482 DECL_INITIAL (decl) = make_node (BLOCK);
2483 if (cgraph_node::get (current_function_decl)->offloadable)
2484 cgraph_node::get_create (decl)->offloadable = 1;
2488 for (octx = ctx; octx; octx = octx->outer)
2489 if (is_gimple_omp_offloaded (octx->stmt))
2491 cgraph_node::get_create (decl)->offloadable = 1;
2492 if (ENABLE_OFFLOADING)
2493 g->have_offload = true;
2499 if (cgraph_node::get_create (decl)->offloadable
2500 && !lookup_attribute ("omp declare target",
2501 DECL_ATTRIBUTES (current_function_decl)))
2502 DECL_ATTRIBUTES (decl)
2503 = tree_cons (get_identifier ("omp target entrypoint"),
2504 NULL_TREE, DECL_ATTRIBUTES (decl));
2506 t = build_decl (DECL_SOURCE_LOCATION (decl),
2507 RESULT_DECL, NULL_TREE, void_type_node);
2508 DECL_ARTIFICIAL (t) = 1;
2509 DECL_IGNORED_P (t) = 1;
2510 DECL_CONTEXT (t) = decl;
2511 DECL_RESULT (decl) = t;
2513 /* _Cilk_for's child function requires two extra parameters called
2514 __low and __high that are set the by Cilk runtime when it calls this
2518 t = build_decl (DECL_SOURCE_LOCATION (decl),
2519 PARM_DECL, get_identifier ("__high"), cilk_var_type);
2520 DECL_ARTIFICIAL (t) = 1;
2521 DECL_NAMELESS (t) = 1;
2522 DECL_ARG_TYPE (t) = ptr_type_node;
2523 DECL_CONTEXT (t) = current_function_decl;
2525 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2526 DECL_ARGUMENTS (decl) = t;
2528 t = build_decl (DECL_SOURCE_LOCATION (decl),
2529 PARM_DECL, get_identifier ("__low"), cilk_var_type);
2530 DECL_ARTIFICIAL (t) = 1;
2531 DECL_NAMELESS (t) = 1;
2532 DECL_ARG_TYPE (t) = ptr_type_node;
2533 DECL_CONTEXT (t) = current_function_decl;
2535 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2536 DECL_ARGUMENTS (decl) = t;
2539 tree data_name = get_identifier (".omp_data_i");
2540 t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name,
2542 DECL_ARTIFICIAL (t) = 1;
2543 DECL_NAMELESS (t) = 1;
2544 DECL_ARG_TYPE (t) = ptr_type_node;
2545 DECL_CONTEXT (t) = current_function_decl;
2547 TREE_READONLY (t) = 1;
2549 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2550 DECL_ARGUMENTS (decl) = t;
2552 ctx->receiver_decl = t;
2555 t = build_decl (DECL_SOURCE_LOCATION (decl),
2556 PARM_DECL, get_identifier (".omp_data_o"),
2558 DECL_ARTIFICIAL (t) = 1;
2559 DECL_NAMELESS (t) = 1;
2560 DECL_ARG_TYPE (t) = ptr_type_node;
2561 DECL_CONTEXT (t) = current_function_decl;
2563 TREE_ADDRESSABLE (t) = 1;
2564 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2565 DECL_ARGUMENTS (decl) = t;
2568 /* Allocate memory for the function structure. The call to
2569 allocate_struct_function clobbers CFUN, so we need to restore
2571 push_struct_function (decl);
2572 cfun->function_end_locus = gimple_location (ctx->stmt);
2576 /* Callback for walk_gimple_seq. Check if combined parallel
2577 contains gimple_omp_for_combined_into_p OMP_FOR. */
2580 find_combined_for (gimple_stmt_iterator *gsi_p,
2581 bool *handled_ops_p,
2582 struct walk_stmt_info *wi)
2584 gimple *stmt = gsi_stmt (*gsi_p);
2586 *handled_ops_p = true;
2587 switch (gimple_code (stmt))
2591 case GIMPLE_OMP_FOR:
2592 if (gimple_omp_for_combined_into_p (stmt)
2593 && gimple_omp_for_kind (stmt)
2594 == *(const enum gf_mask *) (wi->info))
2597 return integer_zero_node;
2606 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2609 add_taskreg_looptemp_clauses (enum gf_mask msk, gimple *stmt,
2610 omp_context *outer_ctx)
2612 struct walk_stmt_info wi;
2614 memset (&wi, 0, sizeof (wi));
2616 wi.info = (void *) &msk;
2617 walk_gimple_seq (gimple_omp_body (stmt), find_combined_for, NULL, &wi);
2618 if (wi.info != (void *) &msk)
2620 gomp_for *for_stmt = as_a <gomp_for *> ((gimple *) wi.info);
2621 struct omp_for_data fd;
2622 extract_omp_for_data (for_stmt, &fd, NULL);
2623 /* We need two temporaries with fd.loop.v type (istart/iend)
2624 and then (fd.collapse - 1) temporaries with the same
2625 type for count2 ... countN-1 vars if not constant. */
2626 size_t count = 2, i;
2627 tree type = fd.iter_type;
2629 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
2631 count += fd.collapse - 1;
2632 /* If there are lastprivate clauses on the inner
2633 GIMPLE_OMP_FOR, add one more temporaries for the total number
2634 of iterations (product of count1 ... countN-1). */
2635 if (find_omp_clause (gimple_omp_for_clauses (for_stmt),
2636 OMP_CLAUSE_LASTPRIVATE))
2638 else if (msk == GF_OMP_FOR_KIND_FOR
2639 && find_omp_clause (gimple_omp_parallel_clauses (stmt),
2640 OMP_CLAUSE_LASTPRIVATE))
2643 for (i = 0; i < count; i++)
2645 tree temp = create_tmp_var (type);
2646 tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
2647 insert_decl_map (&outer_ctx->cb, temp, temp);
2648 OMP_CLAUSE_DECL (c) = temp;
2649 OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2650 gimple_omp_taskreg_set_clauses (stmt, c);
2655 /* Scan an OpenMP parallel directive. */
2658 scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2662 gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi));
2664 /* Ignore parallel directives with empty bodies, unless there
2665 are copyin clauses. */
2667 && empty_body_p (gimple_omp_body (stmt))
2668 && find_omp_clause (gimple_omp_parallel_clauses (stmt),
2669 OMP_CLAUSE_COPYIN) == NULL)
2671 gsi_replace (gsi, gimple_build_nop (), false);
2675 if (gimple_omp_parallel_combined_p (stmt))
2676 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR, stmt, outer_ctx);
2678 ctx = new_omp_context (stmt, outer_ctx);
2679 taskreg_contexts.safe_push (ctx);
2680 if (taskreg_nesting_level > 1)
2681 ctx->is_nested = true;
2682 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2683 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2684 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2685 name = create_tmp_var_name (".omp_data_s");
2686 name = build_decl (gimple_location (stmt),
2687 TYPE_DECL, name, ctx->record_type);
2688 DECL_ARTIFICIAL (name) = 1;
2689 DECL_NAMELESS (name) = 1;
2690 TYPE_NAME (ctx->record_type) = name;
2691 TYPE_ARTIFICIAL (ctx->record_type) = 1;
2692 if (!gimple_omp_parallel_grid_phony (stmt))
2694 create_omp_child_function (ctx, false);
2695 gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2698 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2699 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2701 if (TYPE_FIELDS (ctx->record_type) == NULL)
2702 ctx->record_type = ctx->receiver_decl = NULL;
2705 /* Scan an OpenMP task directive. */
2708 scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2712 gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi));
2714 /* Ignore task directives with empty bodies. */
2716 && empty_body_p (gimple_omp_body (stmt)))
2718 gsi_replace (gsi, gimple_build_nop (), false);
2722 if (gimple_omp_task_taskloop_p (stmt))
2723 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP, stmt, outer_ctx);
2725 ctx = new_omp_context (stmt, outer_ctx);
2726 taskreg_contexts.safe_push (ctx);
2727 if (taskreg_nesting_level > 1)
2728 ctx->is_nested = true;
2729 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2730 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2731 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2732 name = create_tmp_var_name (".omp_data_s");
2733 name = build_decl (gimple_location (stmt),
2734 TYPE_DECL, name, ctx->record_type);
2735 DECL_ARTIFICIAL (name) = 1;
2736 DECL_NAMELESS (name) = 1;
2737 TYPE_NAME (ctx->record_type) = name;
2738 TYPE_ARTIFICIAL (ctx->record_type) = 1;
2739 create_omp_child_function (ctx, false);
2740 gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2742 scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2744 if (ctx->srecord_type)
2746 name = create_tmp_var_name (".omp_data_a");
2747 name = build_decl (gimple_location (stmt),
2748 TYPE_DECL, name, ctx->srecord_type);
2749 DECL_ARTIFICIAL (name) = 1;
2750 DECL_NAMELESS (name) = 1;
2751 TYPE_NAME (ctx->srecord_type) = name;
2752 TYPE_ARTIFICIAL (ctx->srecord_type) = 1;
2753 create_omp_child_function (ctx, true);
2756 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2758 if (TYPE_FIELDS (ctx->record_type) == NULL)
2760 ctx->record_type = ctx->receiver_decl = NULL;
2761 t = build_int_cst (long_integer_type_node, 0);
2762 gimple_omp_task_set_arg_size (stmt, t);
2763 t = build_int_cst (long_integer_type_node, 1);
2764 gimple_omp_task_set_arg_align (stmt, t);
2769 /* If any decls have been made addressable during scan_omp,
2770 adjust their fields if needed, and layout record types
2771 of parallel/task constructs. */
2774 finish_taskreg_scan (omp_context *ctx)
2776 if (ctx->record_type == NULL_TREE)
2779 /* If any task_shared_vars were needed, verify all
2780 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2781 statements if use_pointer_for_field hasn't changed
2782 because of that. If it did, update field types now. */
2783 if (task_shared_vars)
2787 for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2788 c; c = OMP_CLAUSE_CHAIN (c))
2789 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
2790 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
2792 tree decl = OMP_CLAUSE_DECL (c);
2794 /* Global variables don't need to be copied,
2795 the receiver side will use them directly. */
2796 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2798 if (!bitmap_bit_p (task_shared_vars, DECL_UID (decl))
2799 || !use_pointer_for_field (decl, ctx))
2801 tree field = lookup_field (decl, ctx);
2802 if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE
2803 && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl))
2805 TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl));
2806 TREE_THIS_VOLATILE (field) = 0;
2807 DECL_USER_ALIGN (field) = 0;
2808 SET_DECL_ALIGN (field, TYPE_ALIGN (TREE_TYPE (field)));
2809 if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field))
2810 SET_TYPE_ALIGN (ctx->record_type, DECL_ALIGN (field));
2811 if (ctx->srecord_type)
2813 tree sfield = lookup_sfield (decl, ctx);
2814 TREE_TYPE (sfield) = TREE_TYPE (field);
2815 TREE_THIS_VOLATILE (sfield) = 0;
2816 DECL_USER_ALIGN (sfield) = 0;
2817 SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
2818 if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield))
2819 SET_TYPE_ALIGN (ctx->srecord_type, DECL_ALIGN (sfield));
2824 if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2826 layout_type (ctx->record_type);
2827 fixup_child_record_type (ctx);
2831 location_t loc = gimple_location (ctx->stmt);
2832 tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2833 /* Move VLA fields to the end. */
2834 p = &TYPE_FIELDS (ctx->record_type);
2836 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2837 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2840 *p = TREE_CHAIN (*p);
2841 TREE_CHAIN (*q) = NULL_TREE;
2842 q = &TREE_CHAIN (*q);
2845 p = &DECL_CHAIN (*p);
2847 if (gimple_omp_task_taskloop_p (ctx->stmt))
2849 /* Move fields corresponding to first and second _looptemp_
2850 clause first. There are filled by GOMP_taskloop
2851 and thus need to be in specific positions. */
2852 tree c1 = gimple_omp_task_clauses (ctx->stmt);
2853 c1 = find_omp_clause (c1, OMP_CLAUSE__LOOPTEMP_);
2854 tree c2 = find_omp_clause (OMP_CLAUSE_CHAIN (c1),
2855 OMP_CLAUSE__LOOPTEMP_);
2856 tree f1 = lookup_field (OMP_CLAUSE_DECL (c1), ctx);
2857 tree f2 = lookup_field (OMP_CLAUSE_DECL (c2), ctx);
2858 p = &TYPE_FIELDS (ctx->record_type);
2860 if (*p == f1 || *p == f2)
2861 *p = DECL_CHAIN (*p);
2863 p = &DECL_CHAIN (*p);
2864 DECL_CHAIN (f1) = f2;
2865 DECL_CHAIN (f2) = TYPE_FIELDS (ctx->record_type);
2866 TYPE_FIELDS (ctx->record_type) = f1;
2867 if (ctx->srecord_type)
2869 f1 = lookup_sfield (OMP_CLAUSE_DECL (c1), ctx);
2870 f2 = lookup_sfield (OMP_CLAUSE_DECL (c2), ctx);
2871 p = &TYPE_FIELDS (ctx->srecord_type);
2873 if (*p == f1 || *p == f2)
2874 *p = DECL_CHAIN (*p);
2876 p = &DECL_CHAIN (*p);
2877 DECL_CHAIN (f1) = f2;
2878 DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2879 TYPE_FIELDS (ctx->srecord_type) = f1;
2882 layout_type (ctx->record_type);
2883 fixup_child_record_type (ctx);
2884 if (ctx->srecord_type)
2885 layout_type (ctx->srecord_type);
2886 tree t = fold_convert_loc (loc, long_integer_type_node,
2887 TYPE_SIZE_UNIT (ctx->record_type));
2888 gimple_omp_task_set_arg_size (ctx->stmt, t);
2889 t = build_int_cst (long_integer_type_node,
2890 TYPE_ALIGN_UNIT (ctx->record_type));
2891 gimple_omp_task_set_arg_align (ctx->stmt, t);
2895 /* Find the enclosing offload context. */
2897 static omp_context *
2898 enclosing_target_ctx (omp_context *ctx)
2900 for (; ctx; ctx = ctx->outer)
2901 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET)
2907 /* Return true if ctx is part of an oacc kernels region. */
2910 ctx_in_oacc_kernels_region (omp_context *ctx)
2912 for (;ctx != NULL; ctx = ctx->outer)
2914 gimple *stmt = ctx->stmt;
2915 if (gimple_code (stmt) == GIMPLE_OMP_TARGET
2916 && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS)
2923 /* Check the parallelism clauses inside a kernels regions.
2924 Until kernels handling moves to use the same loop indirection
2925 scheme as parallel, we need to do this checking early. */
2928 check_oacc_kernel_gwv (gomp_for *stmt, omp_context *ctx)
2930 bool checking = true;
2931 unsigned outer_mask = 0;
2932 unsigned this_mask = 0;
2933 bool has_seq = false, has_auto = false;
2936 outer_mask = check_oacc_kernel_gwv (NULL, ctx->outer);
2940 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR)
2942 stmt = as_a <gomp_for *> (ctx->stmt);
2945 for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2947 switch (OMP_CLAUSE_CODE (c))
2949 case OMP_CLAUSE_GANG:
2950 this_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
2952 case OMP_CLAUSE_WORKER:
2953 this_mask |= GOMP_DIM_MASK (GOMP_DIM_WORKER);
2955 case OMP_CLAUSE_VECTOR:
2956 this_mask |= GOMP_DIM_MASK (GOMP_DIM_VECTOR);
2958 case OMP_CLAUSE_SEQ:
2961 case OMP_CLAUSE_AUTO:
2971 if (has_seq && (this_mask || has_auto))
2972 error_at (gimple_location (stmt), "%<seq%> overrides other"
2973 " OpenACC loop specifiers");
2974 else if (has_auto && this_mask)
2975 error_at (gimple_location (stmt), "%<auto%> conflicts with other"
2976 " OpenACC loop specifiers");
2978 if (this_mask & outer_mask)
2979 error_at (gimple_location (stmt), "inner loop uses same"
2980 " OpenACC parallelism as containing loop");
2983 return outer_mask | this_mask;
2986 /* Scan a GIMPLE_OMP_FOR. */
2989 scan_omp_for (gomp_for *stmt, omp_context *outer_ctx)
2993 tree clauses = gimple_omp_for_clauses (stmt);
2995 ctx = new_omp_context (stmt, outer_ctx);
2997 if (is_gimple_omp_oacc (stmt))
2999 omp_context *tgt = enclosing_target_ctx (outer_ctx);
3001 if (!tgt || is_oacc_parallel (tgt))
3002 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
3004 char const *check = NULL;
3006 switch (OMP_CLAUSE_CODE (c))
3008 case OMP_CLAUSE_GANG:
3012 case OMP_CLAUSE_WORKER:
3016 case OMP_CLAUSE_VECTOR:
3024 if (check && OMP_CLAUSE_OPERAND (c, 0))
3025 error_at (gimple_location (stmt),
3026 "argument not permitted on %qs clause in"
3027 " OpenACC %<parallel%>", check);
3030 if (tgt && is_oacc_kernels (tgt))
3032 /* Strip out reductions, as they are not handled yet. */
3033 tree *prev_ptr = &clauses;
3035 while (tree probe = *prev_ptr)
3037 tree *next_ptr = &OMP_CLAUSE_CHAIN (probe);
3039 if (OMP_CLAUSE_CODE (probe) == OMP_CLAUSE_REDUCTION)
3040 *prev_ptr = *next_ptr;
3042 prev_ptr = next_ptr;
3045 gimple_omp_for_set_clauses (stmt, clauses);
3046 check_oacc_kernel_gwv (stmt, ctx);
3050 scan_sharing_clauses (clauses, ctx);
3052 scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
3053 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
3055 scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
3056 scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
3057 scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
3058 scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
3060 scan_omp (gimple_omp_body_ptr (stmt), ctx);
3063 /* Scan an OpenMP sections directive. */
3066 scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx)
3070 ctx = new_omp_context (stmt, outer_ctx);
3071 scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
3072 scan_omp (gimple_omp_body_ptr (stmt), ctx);
3075 /* Scan an OpenMP single directive. */
3078 scan_omp_single (gomp_single *stmt, omp_context *outer_ctx)
3083 ctx = new_omp_context (stmt, outer_ctx);
3084 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3085 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3086 name = create_tmp_var_name (".omp_copy_s");
3087 name = build_decl (gimple_location (stmt),
3088 TYPE_DECL, name, ctx->record_type);
3089 TYPE_NAME (ctx->record_type) = name;
3091 scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
3092 scan_omp (gimple_omp_body_ptr (stmt), ctx);
3094 if (TYPE_FIELDS (ctx->record_type) == NULL)
3095 ctx->record_type = NULL;
3097 layout_type (ctx->record_type);
3100 /* Return true if the CLAUSES of an omp target guarantee that the base pointers
3101 used in the corresponding offloaded function are restrict. */
3104 omp_target_base_pointers_restrict_p (tree clauses)
3106 /* The analysis relies on the GOMP_MAP_FORCE_* mapping kinds, which are only
3108 if (flag_openacc == 0)
3111 /* I. Basic example:
3115 unsigned int a[2], b[2];
3117 #pragma acc kernels \
3126 After gimplification, we have:
3128 #pragma omp target oacc_kernels \
3129 map(force_from:a [len: 8]) \
3130 map(force_from:b [len: 8])
3136 Because both mappings have the force prefix, we know that they will be
3137 allocated when calling the corresponding offloaded function, which means we
3138 can mark the base pointers for a and b in the offloaded function as
3142 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
3144 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP)
3147 switch (OMP_CLAUSE_MAP_KIND (c))
3149 case GOMP_MAP_FORCE_ALLOC:
3150 case GOMP_MAP_FORCE_TO:
3151 case GOMP_MAP_FORCE_FROM:
3152 case GOMP_MAP_FORCE_TOFROM:
3162 /* Scan a GIMPLE_OMP_TARGET. */
3165 scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
3169 bool offloaded = is_gimple_omp_offloaded (stmt);
3170 tree clauses = gimple_omp_target_clauses (stmt);
3172 ctx = new_omp_context (stmt, outer_ctx);
3173 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3174 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
3175 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3176 name = create_tmp_var_name (".omp_data_t");
3177 name = build_decl (gimple_location (stmt),
3178 TYPE_DECL, name, ctx->record_type);
3179 DECL_ARTIFICIAL (name) = 1;
3180 DECL_NAMELESS (name) = 1;
3181 TYPE_NAME (ctx->record_type) = name;
3182 TYPE_ARTIFICIAL (ctx->record_type) = 1;
3184 bool base_pointers_restrict = false;
3187 create_omp_child_function (ctx, false);
3188 gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
3190 base_pointers_restrict = omp_target_base_pointers_restrict_p (clauses);
3191 if (base_pointers_restrict
3192 && dump_file && (dump_flags & TDF_DETAILS))
3194 "Base pointers in offloaded function are restrict\n");
3197 scan_sharing_clauses (clauses, ctx, base_pointers_restrict);
3198 scan_omp (gimple_omp_body_ptr (stmt), ctx);
3200 if (TYPE_FIELDS (ctx->record_type) == NULL)
3201 ctx->record_type = ctx->receiver_decl = NULL;
3204 TYPE_FIELDS (ctx->record_type)
3205 = nreverse (TYPE_FIELDS (ctx->record_type));
3208 unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
3209 for (tree field = TYPE_FIELDS (ctx->record_type);
3211 field = DECL_CHAIN (field))
3212 gcc_assert (DECL_ALIGN (field) == align);
3214 layout_type (ctx->record_type);
3216 fixup_child_record_type (ctx);
3220 /* Scan an OpenMP teams directive. */
3223 scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
3225 omp_context *ctx = new_omp_context (stmt, outer_ctx);
3226 scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3227 scan_omp (gimple_omp_body_ptr (stmt), ctx);
3230 /* Check nesting restrictions. */
3232 check_omp_nesting_restrictions (gimple *stmt, omp_context *ctx)
3236 if (ctx && gimple_code (ctx->stmt) == GIMPLE_OMP_GRID_BODY)
3237 /* GRID_BODY is an artificial construct, nesting rules will be checked in
3238 the original copy of its contents. */
3241 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3242 inside an OpenACC CTX. */
3243 if (!(is_gimple_omp (stmt)
3244 && is_gimple_omp_oacc (stmt))
3245 /* Except for atomic codes that we share with OpenMP. */
3246 && !(gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3247 || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE))
3249 if (get_oacc_fn_attrib (cfun->decl) != NULL)
3251 error_at (gimple_location (stmt),
3252 "non-OpenACC construct inside of OpenACC routine");
3256 for (omp_context *octx = ctx; octx != NULL; octx = octx->outer)
3257 if (is_gimple_omp (octx->stmt)
3258 && is_gimple_omp_oacc (octx->stmt))
3260 error_at (gimple_location (stmt),
3261 "non-OpenACC construct inside of OpenACC region");
3268 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3269 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
3272 if (gimple_code (stmt) == GIMPLE_OMP_ORDERED)
3274 c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3275 if (find_omp_clause (c, OMP_CLAUSE_SIMD))
3277 if (find_omp_clause (c, OMP_CLAUSE_THREADS)
3278 && (ctx->outer == NULL
3279 || !gimple_omp_for_combined_into_p (ctx->stmt)
3280 || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR
3281 || (gimple_omp_for_kind (ctx->outer->stmt)
3282 != GF_OMP_FOR_KIND_FOR)
3283 || !gimple_omp_for_combined_p (ctx->outer->stmt)))
3285 error_at (gimple_location (stmt),
3286 "%<ordered simd threads%> must be closely "
3287 "nested inside of %<for simd%> region");
3293 error_at (gimple_location (stmt),
3294 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3295 " may not be nested inside %<simd%> region");
3298 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3300 if ((gimple_code (stmt) != GIMPLE_OMP_FOR
3301 || (gimple_omp_for_kind (stmt)
3302 != GF_OMP_FOR_KIND_DISTRIBUTE))
3303 && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
3305 error_at (gimple_location (stmt),
3306 "only %<distribute%> or %<parallel%> regions are "
3307 "allowed to be strictly nested inside %<teams%> "
3313 switch (gimple_code (stmt))
3315 case GIMPLE_OMP_FOR:
3316 if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_SIMD)
3318 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
3320 if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
3322 error_at (gimple_location (stmt),
3323 "%<distribute%> region must be strictly nested "
3324 "inside %<teams%> construct");
3329 /* We split taskloop into task and nested taskloop in it. */
3330 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3332 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
3337 switch (gimple_code (ctx->stmt))
3339 case GIMPLE_OMP_FOR:
3340 ok = (gimple_omp_for_kind (ctx->stmt)
3341 == GF_OMP_FOR_KIND_OACC_LOOP);
3344 case GIMPLE_OMP_TARGET:
3345 switch (gimple_omp_target_kind (ctx->stmt))
3347 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3348 case GF_OMP_TARGET_KIND_OACC_KERNELS:
3359 else if (get_oacc_fn_attrib (current_function_decl))
3363 error_at (gimple_location (stmt),
3364 "OpenACC loop directive must be associated with"
3365 " an OpenACC compute region");
3371 if (is_gimple_call (stmt)
3372 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3373 == BUILT_IN_GOMP_CANCEL
3374 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3375 == BUILT_IN_GOMP_CANCELLATION_POINT))
3377 const char *bad = NULL;
3378 const char *kind = NULL;
3379 const char *construct
3380 = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3381 == BUILT_IN_GOMP_CANCEL)
3382 ? "#pragma omp cancel"
3383 : "#pragma omp cancellation point";
3386 error_at (gimple_location (stmt), "orphaned %qs construct",
3390 switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
3391 ? tree_to_shwi (gimple_call_arg (stmt, 0))
3395 if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
3396 bad = "#pragma omp parallel";
3397 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3398 == BUILT_IN_GOMP_CANCEL
3399 && !integer_zerop (gimple_call_arg (stmt, 1)))
3400 ctx->cancellable = true;
3404 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3405 || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
3406 bad = "#pragma omp for";
3407 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3408 == BUILT_IN_GOMP_CANCEL
3409 && !integer_zerop (gimple_call_arg (stmt, 1)))
3411 ctx->cancellable = true;
3412 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3414 warning_at (gimple_location (stmt), 0,
3415 "%<#pragma omp cancel for%> inside "
3416 "%<nowait%> for construct");
3417 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3418 OMP_CLAUSE_ORDERED))
3419 warning_at (gimple_location (stmt), 0,
3420 "%<#pragma omp cancel for%> inside "
3421 "%<ordered%> for construct");
3426 if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
3427 && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
3428 bad = "#pragma omp sections";
3429 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3430 == BUILT_IN_GOMP_CANCEL
3431 && !integer_zerop (gimple_call_arg (stmt, 1)))
3433 if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
3435 ctx->cancellable = true;
3436 if (find_omp_clause (gimple_omp_sections_clauses
3439 warning_at (gimple_location (stmt), 0,
3440 "%<#pragma omp cancel sections%> inside "
3441 "%<nowait%> sections construct");
3445 gcc_assert (ctx->outer
3446 && gimple_code (ctx->outer->stmt)
3447 == GIMPLE_OMP_SECTIONS);
3448 ctx->outer->cancellable = true;
3449 if (find_omp_clause (gimple_omp_sections_clauses
3452 warning_at (gimple_location (stmt), 0,
3453 "%<#pragma omp cancel sections%> inside "
3454 "%<nowait%> sections construct");
3460 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK)
3461 bad = "#pragma omp task";
3464 for (omp_context *octx = ctx->outer;
3465 octx; octx = octx->outer)
3467 switch (gimple_code (octx->stmt))
3469 case GIMPLE_OMP_TASKGROUP:
3471 case GIMPLE_OMP_TARGET:
3472 if (gimple_omp_target_kind (octx->stmt)
3473 != GF_OMP_TARGET_KIND_REGION)
3476 case GIMPLE_OMP_PARALLEL:
3477 case GIMPLE_OMP_TEAMS:
3478 error_at (gimple_location (stmt),
3479 "%<%s taskgroup%> construct not closely "
3480 "nested inside of %<taskgroup%> region",
3488 ctx->cancellable = true;
3493 error_at (gimple_location (stmt), "invalid arguments");
3498 error_at (gimple_location (stmt),
3499 "%<%s %s%> construct not closely nested inside of %qs",
3500 construct, kind, bad);
3505 case GIMPLE_OMP_SECTIONS:
3506 case GIMPLE_OMP_SINGLE:
3507 for (; ctx != NULL; ctx = ctx->outer)
3508 switch (gimple_code (ctx->stmt))
3510 case GIMPLE_OMP_FOR:
3511 if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3512 && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3515 case GIMPLE_OMP_SECTIONS:
3516 case GIMPLE_OMP_SINGLE:
3517 case GIMPLE_OMP_ORDERED:
3518 case GIMPLE_OMP_MASTER:
3519 case GIMPLE_OMP_TASK:
3520 case GIMPLE_OMP_CRITICAL:
3521 if (is_gimple_call (stmt))
3523 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3524 != BUILT_IN_GOMP_BARRIER)
3526 error_at (gimple_location (stmt),
3527 "barrier region may not be closely nested inside "
3528 "of work-sharing, %<critical%>, %<ordered%>, "
3529 "%<master%>, explicit %<task%> or %<taskloop%> "
3533 error_at (gimple_location (stmt),
3534 "work-sharing region may not be closely nested inside "
3535 "of work-sharing, %<critical%>, %<ordered%>, "
3536 "%<master%>, explicit %<task%> or %<taskloop%> region");
3538 case GIMPLE_OMP_PARALLEL:
3539 case GIMPLE_OMP_TEAMS:
3541 case GIMPLE_OMP_TARGET:
3542 if (gimple_omp_target_kind (ctx->stmt)
3543 == GF_OMP_TARGET_KIND_REGION)
3550 case GIMPLE_OMP_MASTER:
3551 for (; ctx != NULL; ctx = ctx->outer)
3552 switch (gimple_code (ctx->stmt))
3554 case GIMPLE_OMP_FOR:
3555 if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3556 && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3559 case GIMPLE_OMP_SECTIONS:
3560 case GIMPLE_OMP_SINGLE:
3561 case GIMPLE_OMP_TASK:
3562 error_at (gimple_location (stmt),
3563 "%<master%> region may not be closely nested inside "
3564 "of work-sharing, explicit %<task%> or %<taskloop%> "
3567 case GIMPLE_OMP_PARALLEL:
3568 case GIMPLE_OMP_TEAMS:
3570 case GIMPLE_OMP_TARGET:
3571 if (gimple_omp_target_kind (ctx->stmt)
3572 == GF_OMP_TARGET_KIND_REGION)
3579 case GIMPLE_OMP_TASK:
3580 for (c = gimple_omp_task_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3581 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
3582 && (OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_SOURCE
3583 || OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_SINK))
3585 enum omp_clause_depend_kind kind = OMP_CLAUSE_DEPEND_KIND (c);
3586 error_at (OMP_CLAUSE_LOCATION (c),
3587 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3588 kind == OMP_CLAUSE_DEPEND_SOURCE ? "source" : "sink");
3592 case GIMPLE_OMP_ORDERED:
3593 for (c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3594 c; c = OMP_CLAUSE_CHAIN (c))
3596 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND)
3598 gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREADS
3599 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SIMD);
3602 enum omp_clause_depend_kind kind = OMP_CLAUSE_DEPEND_KIND (c);
3603 if (kind == OMP_CLAUSE_DEPEND_SOURCE
3604 || kind == OMP_CLAUSE_DEPEND_SINK)
3607 /* Look for containing ordered(N) loop. */
3609 || gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3611 = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3612 OMP_CLAUSE_ORDERED)) == NULL_TREE)
3614 error_at (OMP_CLAUSE_LOCATION (c),
3615 "%<ordered%> construct with %<depend%> clause "
3616 "must be closely nested inside an %<ordered%> "
3620 else if (OMP_CLAUSE_ORDERED_EXPR (oclause) == NULL_TREE)
3622 error_at (OMP_CLAUSE_LOCATION (c),
3623 "%<ordered%> construct with %<depend%> clause "
3624 "must be closely nested inside a loop with "
3625 "%<ordered%> clause with a parameter");
3631 error_at (OMP_CLAUSE_LOCATION (c),
3632 "invalid depend kind in omp %<ordered%> %<depend%>");
3636 c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3637 if (find_omp_clause (c, OMP_CLAUSE_SIMD))
3639 /* ordered simd must be closely nested inside of simd region,
3640 and simd region must not encounter constructs other than
3641 ordered simd, therefore ordered simd may be either orphaned,
3642 or ctx->stmt must be simd. The latter case is handled already
3646 error_at (gimple_location (stmt),
3647 "%<ordered%> %<simd%> must be closely nested inside "
3652 for (; ctx != NULL; ctx = ctx->outer)
3653 switch (gimple_code (ctx->stmt))
3655 case GIMPLE_OMP_CRITICAL:
3656 case GIMPLE_OMP_TASK:
3657 case GIMPLE_OMP_ORDERED:
3658 ordered_in_taskloop:
3659 error_at (gimple_location (stmt),
3660 "%<ordered%> region may not be closely nested inside "
3661 "of %<critical%>, %<ordered%>, explicit %<task%> or "
3662 "%<taskloop%> region");
3664 case GIMPLE_OMP_FOR:
3665 if (gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3666 goto ordered_in_taskloop;
3667 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3668 OMP_CLAUSE_ORDERED) == NULL)
3670 error_at (gimple_location (stmt),
3671 "%<ordered%> region must be closely nested inside "
3672 "a loop region with an %<ordered%> clause");
3676 case GIMPLE_OMP_TARGET:
3677 if (gimple_omp_target_kind (ctx->stmt)
3678 != GF_OMP_TARGET_KIND_REGION)
3681 case GIMPLE_OMP_PARALLEL:
3682 case GIMPLE_OMP_TEAMS:
3683 error_at (gimple_location (stmt),
3684 "%<ordered%> region must be closely nested inside "
3685 "a loop region with an %<ordered%> clause");
3691 case GIMPLE_OMP_CRITICAL:
3694 = gimple_omp_critical_name (as_a <gomp_critical *> (stmt));
3695 for (; ctx != NULL; ctx = ctx->outer)
3696 if (gomp_critical *other_crit
3697 = dyn_cast <gomp_critical *> (ctx->stmt))
3698 if (this_stmt_name == gimple_omp_critical_name (other_crit))
3700 error_at (gimple_location (stmt),
3701 "%<critical%> region may not be nested inside "
3702 "a %<critical%> region with the same name");
3707 case GIMPLE_OMP_TEAMS:
3709 || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
3710 || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION)
3712 error_at (gimple_location (stmt),
3713 "%<teams%> construct not closely nested inside of "
3714 "%<target%> construct");
3718 case GIMPLE_OMP_TARGET:
3719 for (c = gimple_omp_target_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3720 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
3721 && (OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_SOURCE
3722 || OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_SINK))
3724 enum omp_clause_depend_kind kind = OMP_CLAUSE_DEPEND_KIND (c);
3725 error_at (OMP_CLAUSE_LOCATION (c),
3726 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3727 kind == OMP_CLAUSE_DEPEND_SOURCE ? "source" : "sink");
3730 if (is_gimple_omp_offloaded (stmt)
3731 && get_oacc_fn_attrib (cfun->decl) != NULL)
3733 error_at (gimple_location (stmt),
3734 "OpenACC region inside of OpenACC routine, nested "
3735 "parallelism not supported yet");
3738 for (; ctx != NULL; ctx = ctx->outer)
3740 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
3742 if (is_gimple_omp (stmt)
3743 && is_gimple_omp_oacc (stmt)
3744 && is_gimple_omp (ctx->stmt))
3746 error_at (gimple_location (stmt),
3747 "OpenACC construct inside of non-OpenACC region");
3753 const char *stmt_name, *ctx_stmt_name;
3754 switch (gimple_omp_target_kind (stmt))
3756 case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break;
3757 case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break;
3758 case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break;
3759 case GF_OMP_TARGET_KIND_ENTER_DATA:
3760 stmt_name = "target enter data"; break;
3761 case GF_OMP_TARGET_KIND_EXIT_DATA:
3762 stmt_name = "target exit data"; break;
3763 case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break;
3764 case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break;
3765 case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break;
3766 case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break;
3767 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
3768 stmt_name = "enter/exit data"; break;
3769 case GF_OMP_TARGET_KIND_OACC_HOST_DATA: stmt_name = "host_data";
3771 default: gcc_unreachable ();
3773 switch (gimple_omp_target_kind (ctx->stmt))
3775 case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break;
3776 case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break;
3777 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3778 ctx_stmt_name = "parallel"; break;
3779 case GF_OMP_TARGET_KIND_OACC_KERNELS:
3780 ctx_stmt_name = "kernels"; break;
3781 case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break;
3782 case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
3783 ctx_stmt_name = "host_data"; break;
3784 default: gcc_unreachable ();
3787 /* OpenACC/OpenMP mismatch? */
3788 if (is_gimple_omp_oacc (stmt)
3789 != is_gimple_omp_oacc (ctx->stmt))
3791 error_at (gimple_location (stmt),
3792 "%s %qs construct inside of %s %qs region",
3793 (is_gimple_omp_oacc (stmt)
3794 ? "OpenACC" : "OpenMP"), stmt_name,
3795 (is_gimple_omp_oacc (ctx->stmt)
3796 ? "OpenACC" : "OpenMP"), ctx_stmt_name);
3799 if (is_gimple_omp_offloaded (ctx->stmt))
3801 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3802 if (is_gimple_omp_oacc (ctx->stmt))
3804 error_at (gimple_location (stmt),
3805 "%qs construct inside of %qs region",
3806 stmt_name, ctx_stmt_name);
3811 warning_at (gimple_location (stmt), 0,
3812 "%qs construct inside of %qs region",
3813 stmt_name, ctx_stmt_name);
3825 /* Helper function scan_omp.
3827 Callback for walk_tree or operators in walk_gimple_stmt used to
3828 scan for OMP directives in TP. */
3831 scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
3833 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
3834 omp_context *ctx = (omp_context *) wi->info;
3837 switch (TREE_CODE (t))
3845 tree repl = remap_decl (t, &ctx->cb);
3846 gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
3852 if (ctx && TYPE_P (t))
3853 *tp = remap_type (t, &ctx->cb);
3854 else if (!DECL_P (t))
3859 tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
3860 if (tem != TREE_TYPE (t))
3862 if (TREE_CODE (t) == INTEGER_CST)
3863 *tp = wide_int_to_tree (tem, t);
3865 TREE_TYPE (t) = tem;
3875 /* Return true if FNDECL is a setjmp or a longjmp. */
3878 setjmp_or_longjmp_p (const_tree fndecl)
3880 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
3881 && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP
3882 || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP))
3885 tree declname = DECL_NAME (fndecl);
3888 const char *name = IDENTIFIER_POINTER (declname);
3889 return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
3893 /* Helper function for scan_omp.
3895 Callback for walk_gimple_stmt used to scan for OMP directives in
3896 the current statement in GSI. */
3899 scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
3900 struct walk_stmt_info *wi)
3902 gimple *stmt = gsi_stmt (*gsi);
3903 omp_context *ctx = (omp_context *) wi->info;
3905 if (gimple_has_location (stmt))
3906 input_location = gimple_location (stmt);
3908 /* Check the nesting restrictions. */
3909 bool remove = false;
3910 if (is_gimple_omp (stmt))
3911 remove = !check_omp_nesting_restrictions (stmt, ctx);
3912 else if (is_gimple_call (stmt))
3914 tree fndecl = gimple_call_fndecl (stmt);
3917 if (setjmp_or_longjmp_p (fndecl)
3919 && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3920 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
3923 error_at (gimple_location (stmt),
3924 "setjmp/longjmp inside simd construct");
3926 else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
3927 switch (DECL_FUNCTION_CODE (fndecl))
3929 case BUILT_IN_GOMP_BARRIER:
3930 case BUILT_IN_GOMP_CANCEL:
3931 case BUILT_IN_GOMP_CANCELLATION_POINT:
3932 case BUILT_IN_GOMP_TASKYIELD:
3933 case BUILT_IN_GOMP_TASKWAIT:
3934 case BUILT_IN_GOMP_TASKGROUP_START:
3935 case BUILT_IN_GOMP_TASKGROUP_END:
3936 remove = !check_omp_nesting_restrictions (stmt, ctx);
3945 stmt = gimple_build_nop ();
3946 gsi_replace (gsi, stmt, false);
3949 *handled_ops_p = true;
3951 switch (gimple_code (stmt))
3953 case GIMPLE_OMP_PARALLEL:
3954 taskreg_nesting_level++;
3955 scan_omp_parallel (gsi, ctx);
3956 taskreg_nesting_level--;
3959 case GIMPLE_OMP_TASK:
3960 taskreg_nesting_level++;
3961 scan_omp_task (gsi, ctx);
3962 taskreg_nesting_level--;
3965 case GIMPLE_OMP_FOR:
3966 scan_omp_for (as_a <gomp_for *> (stmt), ctx);
3969 case GIMPLE_OMP_SECTIONS:
3970 scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
3973 case GIMPLE_OMP_SINGLE:
3974 scan_omp_single (as_a <gomp_single *> (stmt), ctx);
3977 case GIMPLE_OMP_SECTION:
3978 case GIMPLE_OMP_MASTER:
3979 case GIMPLE_OMP_TASKGROUP:
3980 case GIMPLE_OMP_ORDERED:
3981 case GIMPLE_OMP_CRITICAL:
3982 case GIMPLE_OMP_GRID_BODY:
3983 ctx = new_omp_context (stmt, ctx);
3984 scan_omp (gimple_omp_body_ptr (stmt), ctx);
3987 case GIMPLE_OMP_TARGET:
3988 scan_omp_target (as_a <gomp_target *> (stmt), ctx);
3991 case GIMPLE_OMP_TEAMS:
3992 scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
3999 *handled_ops_p = false;
4001 for (var = gimple_bind_vars (as_a <gbind *> (stmt));
4003 var = DECL_CHAIN (var))
4004 insert_decl_map (&ctx->cb, var, var);
4008 *handled_ops_p = false;
4016 /* Scan all the statements starting at the current statement. CTX
4017 contains context information about the OMP directives and
4018 clauses found during the scan. */
4021 scan_omp (gimple_seq *body_p, omp_context *ctx)
4023 location_t saved_location;
4024 struct walk_stmt_info wi;
4026 memset (&wi, 0, sizeof (wi));
4028 wi.want_locations = true;
4030 saved_location = input_location;
4031 walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
4032 input_location = saved_location;
4035 /* Re-gimplification and code generation routines. */
4037 /* Build a call to GOMP_barrier. */
4040 build_omp_barrier (tree lhs)
4042 tree fndecl = builtin_decl_explicit (lhs ? BUILT_IN_GOMP_BARRIER_CANCEL
4043 : BUILT_IN_GOMP_BARRIER);
4044 gcall *g = gimple_build_call (fndecl, 0);
4046 gimple_call_set_lhs (g, lhs);
4050 /* If a context was created for STMT when it was scanned, return it. */
4052 static omp_context *
4053 maybe_lookup_ctx (gimple *stmt)
4056 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
4057 return n ? (omp_context *) n->value : NULL;
4061 /* Find the mapping for DECL in CTX or the immediately enclosing
4062 context that has a mapping for DECL.
4064 If CTX is a nested parallel directive, we may have to use the decl
4065 mappings created in CTX's parent context. Suppose that we have the
4066 following parallel nesting (variable UIDs showed for clarity):
4069 #omp parallel shared(iD.1562) -> outer parallel
4070 iD.1562 = iD.1562 + 1;
4072 #omp parallel shared (iD.1562) -> inner parallel
4073 iD.1562 = iD.1562 - 1;
4075 Each parallel structure will create a distinct .omp_data_s structure
4076 for copying iD.1562 in/out of the directive:
4078 outer parallel .omp_data_s.1.i -> iD.1562
4079 inner parallel .omp_data_s.2.i -> iD.1562
4081 A shared variable mapping will produce a copy-out operation before
4082 the parallel directive and a copy-in operation after it. So, in
4083 this case we would have:
4086 .omp_data_o.1.i = iD.1562;
4087 #omp parallel shared(iD.1562) -> outer parallel
4088 .omp_data_i.1 = &.omp_data_o.1
4089 .omp_data_i.1->i = .omp_data_i.1->i + 1;
4091 .omp_data_o.2.i = iD.1562; -> **
4092 #omp parallel shared(iD.1562) -> inner parallel
4093 .omp_data_i.2 = &.omp_data_o.2
4094 .omp_data_i.2->i = .omp_data_i.2->i - 1;
4097 ** This is a problem. The symbol iD.1562 cannot be referenced
4098 inside the body of the outer parallel region. But since we are
4099 emitting this copy operation while expanding the inner parallel
4100 directive, we need to access the CTX structure of the outer
4101 parallel directive to get the correct mapping:
4103 .omp_data_o.2.i = .omp_data_i.1->i
4105 Since there may be other workshare or parallel directives enclosing
4106 the parallel directive, it may be necessary to walk up the context
4107 parent chain. This is not a problem in general because nested
4108 parallelism happens only rarely. */
4111 lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4116 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4117 t = maybe_lookup_decl (decl, up);
4119 gcc_assert (!ctx->is_nested || t || is_global_var (decl));
4121 return t ? t : decl;
4125 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4126 in outer contexts. */
4129 maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4134 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4135 t = maybe_lookup_decl (decl, up);
4137 return t ? t : decl;
4141 /* Construct the initialization value for reduction operation OP. */
4144 omp_reduction_init_op (location_t loc, enum tree_code op, tree type)
4153 case TRUTH_ORIF_EXPR:
4154 case TRUTH_XOR_EXPR:
4156 return build_zero_cst (type);
4159 case TRUTH_AND_EXPR:
4160 case TRUTH_ANDIF_EXPR:
4162 return fold_convert_loc (loc, type, integer_one_node);
4165 return fold_convert_loc (loc, type, integer_minus_one_node);
4168 if (SCALAR_FLOAT_TYPE_P (type))
4170 REAL_VALUE_TYPE max, min;
4171 if (HONOR_INFINITIES (type))
4174 real_arithmetic (&min, NEGATE_EXPR, &max, NULL);
4177 real_maxval (&min, 1, TYPE_MODE (type));
4178 return build_real (type, min);
4180 else if (POINTER_TYPE_P (type))
4183 = wi::min_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4184 return wide_int_to_tree (type, min);
4188 gcc_assert (INTEGRAL_TYPE_P (type));
4189 return TYPE_MIN_VALUE (type);
4193 if (SCALAR_FLOAT_TYPE_P (type))
4195 REAL_VALUE_TYPE max;
4196 if (HONOR_INFINITIES (type))
4199 real_maxval (&max, 0, TYPE_MODE (type));
4200 return build_real (type, max);
4202 else if (POINTER_TYPE_P (type))
4205 = wi::max_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4206 return wide_int_to_tree (type, max);
4210 gcc_assert (INTEGRAL_TYPE_P (type));
4211 return TYPE_MAX_VALUE (type);
4219 /* Construct the initialization value for reduction CLAUSE. */
4222 omp_reduction_init (tree clause, tree type)
4224 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause),
4225 OMP_CLAUSE_REDUCTION_CODE (clause), type);
4228 /* Return alignment to be assumed for var in CLAUSE, which should be
4229 OMP_CLAUSE_ALIGNED. */
4232 omp_clause_aligned_alignment (tree clause)
4234 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
4235 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
4237 /* Otherwise return implementation defined alignment. */
4238 unsigned int al = 1;
4239 machine_mode mode, vmode;
4240 int vs = targetm.vectorize.autovectorize_vector_sizes ();
4242 vs = 1 << floor_log2 (vs);
4243 static enum mode_class classes[]
4244 = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
4245 for (int i = 0; i < 4; i += 2)
4246 for (mode = GET_CLASS_NARROWEST_MODE (classes[i]);
4248 mode = GET_MODE_WIDER_MODE (mode))
4250 vmode = targetm.vectorize.preferred_simd_mode (mode);
4251 if (GET_MODE_CLASS (vmode) != classes[i + 1])
4254 && GET_MODE_SIZE (vmode) < vs
4255 && GET_MODE_2XWIDER_MODE (vmode) != VOIDmode)
4256 vmode = GET_MODE_2XWIDER_MODE (vmode);
4258 tree type = lang_hooks.types.type_for_mode (mode, 1);
4259 if (type == NULL_TREE || TYPE_MODE (type) != mode)
4261 type = build_vector_type (type, GET_MODE_SIZE (vmode)
4262 / GET_MODE_SIZE (mode));
4263 if (TYPE_MODE (type) != vmode)
4265 if (TYPE_ALIGN_UNIT (type) > al)
4266 al = TYPE_ALIGN_UNIT (type);
4268 return build_int_cst (integer_type_node, al);
4271 /* Return maximum possible vectorization factor for the target. */
4278 || !flag_tree_loop_optimize
4279 || (!flag_tree_loop_vectorize
4280 && (global_options_set.x_flag_tree_loop_vectorize
4281 || global_options_set.x_flag_tree_vectorize)))
4284 int vs = targetm.vectorize.autovectorize_vector_sizes ();
4287 vs = 1 << floor_log2 (vs);
4290 machine_mode vqimode = targetm.vectorize.preferred_simd_mode (QImode);
4291 if (GET_MODE_CLASS (vqimode) == MODE_VECTOR_INT)
4292 return GET_MODE_NUNITS (vqimode);
4296 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4300 lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, int &max_vf,
4301 tree &idx, tree &lane, tree &ivar, tree &lvar)
4305 max_vf = omp_max_vf ();
4308 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
4309 OMP_CLAUSE_SAFELEN);
4311 && (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) != INTEGER_CST
4312 || tree_int_cst_sgn (OMP_CLAUSE_SAFELEN_EXPR (c)) != 1))
4314 else if (c && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
4316 max_vf = tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c));
4320 idx = create_tmp_var (unsigned_type_node);
4321 lane = create_tmp_var (unsigned_type_node);
4327 tree atype = build_array_type_nelts (TREE_TYPE (new_var), max_vf);
4328 tree avar = create_tmp_var_raw (atype);
4329 if (TREE_ADDRESSABLE (new_var))
4330 TREE_ADDRESSABLE (avar) = 1;
4331 DECL_ATTRIBUTES (avar)
4332 = tree_cons (get_identifier ("omp simd array"), NULL,
4333 DECL_ATTRIBUTES (avar));
4334 gimple_add_tmp_var (avar);
4335 ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, idx,
4336 NULL_TREE, NULL_TREE);
4337 lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, lane,
4338 NULL_TREE, NULL_TREE);
4339 if (DECL_P (new_var))
4341 SET_DECL_VALUE_EXPR (new_var, lvar);
4342 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4347 /* Helper function of lower_rec_input_clauses. For a reference
4348 in simd reduction, add an underlying variable it will reference. */
4351 handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
4353 tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
4354 if (TREE_CONSTANT (z))
4356 z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)),
4357 get_name (new_vard));
4358 gimple_add_tmp_var (z);
4359 TREE_ADDRESSABLE (z) = 1;
4360 z = build_fold_addr_expr_loc (loc, z);
4361 gimplify_assign (new_vard, z, ilist);
4365 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4366 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4367 private variables. Initialization statements go in ILIST, while calls
4368 to destructors go in DLIST. */
4371 lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
4372 omp_context *ctx, struct omp_for_data *fd)
4374 tree c, dtor, copyin_seq, x, ptr;
4375 bool copyin_by_ref = false;
4376 bool lastprivate_firstprivate = false;
4377 bool reduction_omp_orig_ref = false;
4379 bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4380 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD);
4382 tree lane = NULL_TREE, idx = NULL_TREE;
4383 tree ivar = NULL_TREE, lvar = NULL_TREE;
4384 gimple_seq llist[2] = { NULL, NULL };
4388 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4389 with data sharing clauses referencing variable sized vars. That
4390 is unnecessarily hard to support and very unlikely to result in
4391 vectorized code anyway. */
4393 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4394 switch (OMP_CLAUSE_CODE (c))
4396 case OMP_CLAUSE_LINEAR:
4397 if (OMP_CLAUSE_LINEAR_ARRAY (c))
4400 case OMP_CLAUSE_PRIVATE:
4401 case OMP_CLAUSE_FIRSTPRIVATE:
4402 case OMP_CLAUSE_LASTPRIVATE:
4403 if (is_variable_sized (OMP_CLAUSE_DECL (c)))
4406 case OMP_CLAUSE_REDUCTION:
4407 if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF
4408 || is_variable_sized (OMP_CLAUSE_DECL (c)))
4415 /* Do all the fixed sized types in the first pass, and the variable sized
4416 types in the second pass. This makes sure that the scalar arguments to
4417 the variable sized types are processed before we use them in the
4418 variable sized operations. */
4419 for (pass = 0; pass < 2; ++pass)
4421 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4423 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
4426 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4430 case OMP_CLAUSE_PRIVATE:
4431 if (OMP_CLAUSE_PRIVATE_DEBUG (c))
4434 case OMP_CLAUSE_SHARED:
4435 /* Ignore shared directives in teams construct. */
4436 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
4438 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
4440 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)
4441 || is_global_var (OMP_CLAUSE_DECL (c)));
4444 case OMP_CLAUSE_FIRSTPRIVATE:
4445 case OMP_CLAUSE_COPYIN:
4447 case OMP_CLAUSE_LINEAR:
4448 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)
4449 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
4450 lastprivate_firstprivate = true;
4452 case OMP_CLAUSE_REDUCTION:
4453 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
4454 reduction_omp_orig_ref = true;
4456 case OMP_CLAUSE__LOOPTEMP_:
4457 /* Handle _looptemp_ clauses only on parallel/task. */
4461 case OMP_CLAUSE_LASTPRIVATE:
4462 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
4464 lastprivate_firstprivate = true;
4465 if (pass != 0 || is_taskloop_ctx (ctx))
4468 /* Even without corresponding firstprivate, if
4469 decl is Fortran allocatable, it needs outer var
4472 && lang_hooks.decls.omp_private_outer_ref
4473 (OMP_CLAUSE_DECL (c)))
4474 lastprivate_firstprivate = true;
4476 case OMP_CLAUSE_ALIGNED:
4479 var = OMP_CLAUSE_DECL (c);
4480 if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
4481 && !is_global_var (var))
4483 new_var = maybe_lookup_decl (var, ctx);
4484 if (new_var == NULL_TREE)
4485 new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
4486 x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
4487 tree alarg = omp_clause_aligned_alignment (c);
4488 alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
4489 x = build_call_expr_loc (clause_loc, x, 2, new_var, alarg);
4490 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
4491 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
4492 gimplify_and_add (x, ilist);
4494 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
4495 && is_global_var (var))
4497 tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
4498 new_var = lookup_decl (var, ctx);
4499 t = maybe_lookup_decl_in_outer_ctx (var, ctx);
4500 t = build_fold_addr_expr_loc (clause_loc, t);
4501 t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
4502 tree alarg = omp_clause_aligned_alignment (c);
4503 alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
4504 t = build_call_expr_loc (clause_loc, t2, 2, t, alarg);
4505 t = fold_convert_loc (clause_loc, ptype, t);
4506 x = create_tmp_var (ptype);
4507 t = build2 (MODIFY_EXPR, ptype, x, t);
4508 gimplify_and_add (t, ilist);
4509 t = build_simple_mem_ref_loc (clause_loc, x);
4510 SET_DECL_VALUE_EXPR (new_var, t);
4511 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4518 new_var = var = OMP_CLAUSE_DECL (c);
4519 if (c_kind == OMP_CLAUSE_REDUCTION && TREE_CODE (var) == MEM_REF)
4521 var = TREE_OPERAND (var, 0);
4522 if (TREE_CODE (var) == POINTER_PLUS_EXPR)
4523 var = TREE_OPERAND (var, 0);
4524 if (TREE_CODE (var) == INDIRECT_REF
4525 || TREE_CODE (var) == ADDR_EXPR)
4526 var = TREE_OPERAND (var, 0);
4527 if (is_variable_sized (var))
4529 gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
4530 var = DECL_VALUE_EXPR (var);
4531 gcc_assert (TREE_CODE (var) == INDIRECT_REF);
4532 var = TREE_OPERAND (var, 0);
4533 gcc_assert (DECL_P (var));
4537 if (c_kind != OMP_CLAUSE_COPYIN)
4538 new_var = lookup_decl (var, ctx);
4540 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
4545 /* C/C++ array section reductions. */
4546 else if (c_kind == OMP_CLAUSE_REDUCTION
4547 && var != OMP_CLAUSE_DECL (c))
4552 tree bias = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
4553 tree orig_var = TREE_OPERAND (OMP_CLAUSE_DECL (c), 0);
4554 if (TREE_CODE (orig_var) == POINTER_PLUS_EXPR)
4556 tree b = TREE_OPERAND (orig_var, 1);
4557 b = maybe_lookup_decl (b, ctx);
4560 b = TREE_OPERAND (orig_var, 1);
4561 b = maybe_lookup_decl_in_outer_ctx (b, ctx);
4563 if (integer_zerop (bias))
4567 bias = fold_convert_loc (clause_loc,
4568 TREE_TYPE (b), bias);
4569 bias = fold_build2_loc (clause_loc, PLUS_EXPR,
4570 TREE_TYPE (b), b, bias);
4572 orig_var = TREE_OPERAND (orig_var, 0);
4574 if (TREE_CODE (orig_var) == INDIRECT_REF
4575 || TREE_CODE (orig_var) == ADDR_EXPR)
4576 orig_var = TREE_OPERAND (orig_var, 0);
4577 tree d = OMP_CLAUSE_DECL (c);
4578 tree type = TREE_TYPE (d);
4579 gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
4580 tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
4581 const char *name = get_name (orig_var);
4582 if (TREE_CONSTANT (v))
4584 x = create_tmp_var_raw (type, name);
4585 gimple_add_tmp_var (x);
4586 TREE_ADDRESSABLE (x) = 1;
4587 x = build_fold_addr_expr_loc (clause_loc, x);
4592 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
4593 tree t = maybe_lookup_decl (v, ctx);
4597 v = maybe_lookup_decl_in_outer_ctx (v, ctx);
4598 gimplify_expr (&v, ilist, NULL, is_gimple_val, fb_rvalue);
4599 t = fold_build2_loc (clause_loc, PLUS_EXPR,
4601 build_int_cst (TREE_TYPE (v), 1));
4602 t = fold_build2_loc (clause_loc, MULT_EXPR,
4604 TYPE_SIZE_UNIT (TREE_TYPE (type)));
4605 tree al = size_int (TYPE_ALIGN (TREE_TYPE (type)));
4606 x = build_call_expr_loc (clause_loc, atmp, 2, t, al);
4609 tree ptype = build_pointer_type (TREE_TYPE (type));
4610 x = fold_convert_loc (clause_loc, ptype, x);
4611 tree y = create_tmp_var (ptype, name);
4612 gimplify_assign (y, x, ilist);
4616 if (!integer_zerop (bias))
4618 bias = fold_convert_loc (clause_loc, pointer_sized_int_node,
4620 yb = fold_convert_loc (clause_loc, pointer_sized_int_node,
4622 yb = fold_build2_loc (clause_loc, MINUS_EXPR,
4623 pointer_sized_int_node, yb, bias);
4624 x = fold_convert_loc (clause_loc, TREE_TYPE (x), yb);
4625 yb = create_tmp_var (ptype, name);
4626 gimplify_assign (yb, x, ilist);
4630 d = TREE_OPERAND (d, 0);
4631 if (TREE_CODE (d) == POINTER_PLUS_EXPR)
4632 d = TREE_OPERAND (d, 0);
4633 if (TREE_CODE (d) == ADDR_EXPR)
4635 if (orig_var != var)
4637 gcc_assert (is_variable_sized (orig_var));
4638 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var),
4640 gimplify_assign (new_var, x, ilist);
4641 tree new_orig_var = lookup_decl (orig_var, ctx);
4642 tree t = build_fold_indirect_ref (new_var);
4643 DECL_IGNORED_P (new_var) = 0;
4644 TREE_THIS_NOTRAP (t);
4645 SET_DECL_VALUE_EXPR (new_orig_var, t);
4646 DECL_HAS_VALUE_EXPR_P (new_orig_var) = 1;
4650 x = build2 (MEM_REF, TREE_TYPE (new_var), x,
4651 build_int_cst (ptype, 0));
4652 SET_DECL_VALUE_EXPR (new_var, x);
4653 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4658 gcc_assert (orig_var == var);
4659 if (TREE_CODE (d) == INDIRECT_REF)
4661 x = create_tmp_var (ptype, name);
4662 TREE_ADDRESSABLE (x) = 1;
4663 gimplify_assign (x, yb, ilist);
4664 x = build_fold_addr_expr_loc (clause_loc, x);
4666 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
4667 gimplify_assign (new_var, x, ilist);
4669 tree y1 = create_tmp_var (ptype, NULL);
4670 gimplify_assign (y1, y, ilist);
4671 tree i2 = NULL_TREE, y2 = NULL_TREE;
4672 tree body2 = NULL_TREE, end2 = NULL_TREE;
4673 tree y3 = NULL_TREE, y4 = NULL_TREE;
4674 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) || is_simd)
4676 y2 = create_tmp_var (ptype, NULL);
4677 gimplify_assign (y2, y, ilist);
4678 tree ref = build_outer_var_ref (var, ctx);
4679 /* For ref build_outer_var_ref already performs this. */
4680 if (TREE_CODE (d) == INDIRECT_REF)
4681 gcc_assert (is_reference (var));
4682 else if (TREE_CODE (d) == ADDR_EXPR)
4683 ref = build_fold_addr_expr (ref);
4684 else if (is_reference (var))
4685 ref = build_fold_addr_expr (ref);
4686 ref = fold_convert_loc (clause_loc, ptype, ref);
4687 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
4688 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
4690 y3 = create_tmp_var (ptype, NULL);
4691 gimplify_assign (y3, unshare_expr (ref), ilist);
4695 y4 = create_tmp_var (ptype, NULL);
4696 gimplify_assign (y4, ref, dlist);
4699 tree i = create_tmp_var (TREE_TYPE (v), NULL);
4700 gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), ilist);
4701 tree body = create_artificial_label (UNKNOWN_LOCATION);
4702 tree end = create_artificial_label (UNKNOWN_LOCATION);
4703 gimple_seq_add_stmt (ilist, gimple_build_label (body));
4706 i2 = create_tmp_var (TREE_TYPE (v), NULL);
4707 gimplify_assign (i2, build_int_cst (TREE_TYPE (v), 0), dlist);
4708 body2 = create_artificial_label (UNKNOWN_LOCATION);
4709 end2 = create_artificial_label (UNKNOWN_LOCATION);
4710 gimple_seq_add_stmt (dlist, gimple_build_label (body2));
4712 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4714 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
4715 tree decl_placeholder
4716 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
4717 SET_DECL_VALUE_EXPR (decl_placeholder,
4718 build_simple_mem_ref (y1));
4719 DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
4720 SET_DECL_VALUE_EXPR (placeholder,
4721 y3 ? build_simple_mem_ref (y3)
4723 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
4724 x = lang_hooks.decls.omp_clause_default_ctor
4725 (c, build_simple_mem_ref (y1),
4726 y3 ? build_simple_mem_ref (y3) : NULL_TREE);
4728 gimplify_and_add (x, ilist);
4729 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
4731 gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
4732 lower_omp (&tseq, ctx);
4733 gimple_seq_add_seq (ilist, tseq);
4735 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
4738 SET_DECL_VALUE_EXPR (decl_placeholder,
4739 build_simple_mem_ref (y2));
4740 SET_DECL_VALUE_EXPR (placeholder,
4741 build_simple_mem_ref (y4));
4742 gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
4743 lower_omp (&tseq, ctx);
4744 gimple_seq_add_seq (dlist, tseq);
4745 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
4747 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
4748 DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 0;
4749 x = lang_hooks.decls.omp_clause_dtor
4750 (c, build_simple_mem_ref (y2));
4753 gimple_seq tseq = NULL;
4755 gimplify_stmt (&dtor, &tseq);
4756 gimple_seq_add_seq (dlist, tseq);
4761 x = omp_reduction_init (c, TREE_TYPE (type));
4762 enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
4764 /* reduction(-:var) sums up the partial results, so it
4765 acts identically to reduction(+:var). */
4766 if (code == MINUS_EXPR)
4769 gimplify_assign (build_simple_mem_ref (y1), x, ilist);
4772 x = build2 (code, TREE_TYPE (type),
4773 build_simple_mem_ref (y4),
4774 build_simple_mem_ref (y2));
4775 gimplify_assign (build_simple_mem_ref (y4), x, dlist);
4779 = gimple_build_assign (y1, POINTER_PLUS_EXPR, y1,
4780 TYPE_SIZE_UNIT (TREE_TYPE (type)));
4781 gimple_seq_add_stmt (ilist, g);
4784 g = gimple_build_assign (y3, POINTER_PLUS_EXPR, y3,
4785 TYPE_SIZE_UNIT (TREE_TYPE (type)));
4786 gimple_seq_add_stmt (ilist, g);
4788 g = gimple_build_assign (i, PLUS_EXPR, i,
4789 build_int_cst (TREE_TYPE (i), 1));
4790 gimple_seq_add_stmt (ilist, g);
4791 g = gimple_build_cond (LE_EXPR, i, v, body, end);
4792 gimple_seq_add_stmt (ilist, g);
4793 gimple_seq_add_stmt (ilist, gimple_build_label (end));
4796 g = gimple_build_assign (y2, POINTER_PLUS_EXPR, y2,
4797 TYPE_SIZE_UNIT (TREE_TYPE (type)));
4798 gimple_seq_add_stmt (dlist, g);
4801 g = gimple_build_assign
4802 (y4, POINTER_PLUS_EXPR, y4,
4803 TYPE_SIZE_UNIT (TREE_TYPE (type)));
4804 gimple_seq_add_stmt (dlist, g);
4806 g = gimple_build_assign (i2, PLUS_EXPR, i2,
4807 build_int_cst (TREE_TYPE (i2), 1));
4808 gimple_seq_add_stmt (dlist, g);
4809 g = gimple_build_cond (LE_EXPR, i2, v, body2, end2);
4810 gimple_seq_add_stmt (dlist, g);
4811 gimple_seq_add_stmt (dlist, gimple_build_label (end2));
4815 else if (is_variable_sized (var))
4817 /* For variable sized types, we need to allocate the
4818 actual storage here. Call alloca and store the
4819 result in the pointer decl that we created elsewhere. */
4823 if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
4828 ptr = DECL_VALUE_EXPR (new_var);
4829 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
4830 ptr = TREE_OPERAND (ptr, 0);
4831 gcc_assert (DECL_P (ptr));
4832 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
4834 /* void *tmp = __builtin_alloca */
4835 atmp = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
4836 stmt = gimple_build_call (atmp, 2, x,
4837 size_int (DECL_ALIGN (var)));
4838 tmp = create_tmp_var_raw (ptr_type_node);
4839 gimple_add_tmp_var (tmp);
4840 gimple_call_set_lhs (stmt, tmp);
4842 gimple_seq_add_stmt (ilist, stmt);
4844 x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
4845 gimplify_assign (ptr, x, ilist);
4848 else if (is_reference (var))
4850 /* For references that are being privatized for Fortran,
4851 allocate new backing storage for the new pointer
4852 variable. This allows us to avoid changing all the
4853 code that expects a pointer to something that expects
4854 a direct variable. */
4858 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
4859 if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
4861 x = build_receiver_ref (var, false, ctx);
4862 x = build_fold_addr_expr_loc (clause_loc, x);
4864 else if (TREE_CONSTANT (x))
4866 /* For reduction in SIMD loop, defer adding the
4867 initialization of the reference, because if we decide
4868 to use SIMD array for it, the initilization could cause
4870 if (c_kind == OMP_CLAUSE_REDUCTION && is_simd)
4874 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
4876 gimple_add_tmp_var (x);
4877 TREE_ADDRESSABLE (x) = 1;
4878 x = build_fold_addr_expr_loc (clause_loc, x);
4884 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
4885 tree rtype = TREE_TYPE (TREE_TYPE (new_var));
4886 tree al = size_int (TYPE_ALIGN (rtype));
4887 x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
4892 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
4893 gimplify_assign (new_var, x, ilist);
4896 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4898 else if (c_kind == OMP_CLAUSE_REDUCTION
4899 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4907 switch (OMP_CLAUSE_CODE (c))
4909 case OMP_CLAUSE_SHARED:
4910 /* Ignore shared directives in teams construct. */
4911 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
4913 /* Shared global vars are just accessed directly. */
4914 if (is_global_var (new_var))
4916 /* For taskloop firstprivate/lastprivate, represented
4917 as firstprivate and shared clause on the task, new_var
4918 is the firstprivate var. */
4919 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
4921 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4922 needs to be delayed until after fixup_child_record_type so
4923 that we get the correct type during the dereference. */
4924 by_ref = use_pointer_for_field (var, ctx);
4925 x = build_receiver_ref (var, by_ref, ctx);
4926 SET_DECL_VALUE_EXPR (new_var, x);
4927 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4929 /* ??? If VAR is not passed by reference, and the variable
4930 hasn't been initialized yet, then we'll get a warning for
4931 the store into the omp_data_s structure. Ideally, we'd be
4932 able to notice this and not store anything at all, but
4933 we're generating code too early. Suppress the warning. */
4935 TREE_NO_WARNING (var) = 1;
4938 case OMP_CLAUSE_LASTPRIVATE:
4939 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
4943 case OMP_CLAUSE_PRIVATE:
4944 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
4945 x = build_outer_var_ref (var, ctx);
4946 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
4948 if (is_task_ctx (ctx))
4949 x = build_receiver_ref (var, false, ctx);
4951 x = build_outer_var_ref (var, ctx, OMP_CLAUSE_PRIVATE);
4957 nx = lang_hooks.decls.omp_clause_default_ctor
4958 (c, unshare_expr (new_var), x);
4961 tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
4962 if ((TREE_ADDRESSABLE (new_var) || nx || y
4963 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
4964 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
4965 idx, lane, ivar, lvar))
4968 x = lang_hooks.decls.omp_clause_default_ctor
4969 (c, unshare_expr (ivar), x);
4971 gimplify_and_add (x, &llist[0]);
4974 y = lang_hooks.decls.omp_clause_dtor (c, ivar);
4977 gimple_seq tseq = NULL;
4980 gimplify_stmt (&dtor, &tseq);
4981 gimple_seq_add_seq (&llist[1], tseq);
4988 gimplify_and_add (nx, ilist);
4992 x = lang_hooks.decls.omp_clause_dtor (c, new_var);
4995 gimple_seq tseq = NULL;
4998 gimplify_stmt (&dtor, &tseq);
4999 gimple_seq_add_seq (dlist, tseq);
5003 case OMP_CLAUSE_LINEAR:
5004 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
5005 goto do_firstprivate;
5006 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
5009 x = build_outer_var_ref (var, ctx);
5012 case OMP_CLAUSE_FIRSTPRIVATE:
5013 if (is_task_ctx (ctx))
5015 if (is_reference (var) || is_variable_sized (var))
5017 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
5019 || use_pointer_for_field (var, NULL))
5021 x = build_receiver_ref (var, false, ctx);
5022 SET_DECL_VALUE_EXPR (new_var, x);
5023 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5028 x = build_outer_var_ref (var, ctx);
5031 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
5032 && gimple_omp_for_combined_into_p (ctx->stmt))
5034 tree t = OMP_CLAUSE_LINEAR_STEP (c);
5035 tree stept = TREE_TYPE (t);
5036 tree ct = find_omp_clause (clauses,
5037 OMP_CLAUSE__LOOPTEMP_);
5039 tree l = OMP_CLAUSE_DECL (ct);
5040 tree n1 = fd->loop.n1;
5041 tree step = fd->loop.step;
5042 tree itype = TREE_TYPE (l);
5043 if (POINTER_TYPE_P (itype))
5044 itype = signed_type_for (itype);
5045 l = fold_build2 (MINUS_EXPR, itype, l, n1);
5046 if (TYPE_UNSIGNED (itype)
5047 && fd->loop.cond_code == GT_EXPR)
5048 l = fold_build2 (TRUNC_DIV_EXPR, itype,
5049 fold_build1 (NEGATE_EXPR, itype, l),
5050 fold_build1 (NEGATE_EXPR,
5053 l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
5054 t = fold_build2 (MULT_EXPR, stept,
5055 fold_convert (stept, l), t);
5057 if (OMP_CLAUSE_LINEAR_ARRAY (c))
5059 x = lang_hooks.decls.omp_clause_linear_ctor
5061 gimplify_and_add (x, ilist);
5065 if (POINTER_TYPE_P (TREE_TYPE (x)))
5066 x = fold_build2 (POINTER_PLUS_EXPR,
5067 TREE_TYPE (x), x, t);
5069 x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, t);
5072 if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
5073 || TREE_ADDRESSABLE (new_var))
5074 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
5075 idx, lane, ivar, lvar))
5077 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
5079 tree iv = create_tmp_var (TREE_TYPE (new_var));
5080 x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
5081 gimplify_and_add (x, ilist);
5082 gimple_stmt_iterator gsi
5083 = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
5085 = gimple_build_assign (unshare_expr (lvar), iv);
5086 gsi_insert_before_without_update (&gsi, g,
5088 tree t = OMP_CLAUSE_LINEAR_STEP (c);
5089 enum tree_code code = PLUS_EXPR;
5090 if (POINTER_TYPE_P (TREE_TYPE (new_var)))
5091 code = POINTER_PLUS_EXPR;
5092 g = gimple_build_assign (iv, code, iv, t);
5093 gsi_insert_before_without_update (&gsi, g,
5097 x = lang_hooks.decls.omp_clause_copy_ctor
5098 (c, unshare_expr (ivar), x);
5099 gimplify_and_add (x, &llist[0]);
5100 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
5103 gimple_seq tseq = NULL;
5106 gimplify_stmt (&dtor, &tseq);
5107 gimple_seq_add_seq (&llist[1], tseq);
5112 x = lang_hooks.decls.omp_clause_copy_ctor
5113 (c, unshare_expr (new_var), x);
5114 gimplify_and_add (x, ilist);
5117 case OMP_CLAUSE__LOOPTEMP_:
5118 gcc_assert (is_taskreg_ctx (ctx));
5119 x = build_outer_var_ref (var, ctx);
5120 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
5121 gimplify_and_add (x, ilist);
5124 case OMP_CLAUSE_COPYIN:
5125 by_ref = use_pointer_for_field (var, NULL);
5126 x = build_receiver_ref (var, by_ref, ctx);
5127 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
5128 append_to_statement_list (x, ©in_seq);
5129 copyin_by_ref |= by_ref;
5132 case OMP_CLAUSE_REDUCTION:
5133 /* OpenACC reductions are initialized using the
5134 GOACC_REDUCTION internal function. */
5135 if (is_gimple_omp_oacc (ctx->stmt))
5137 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5139 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
5141 x = build_outer_var_ref (var, ctx);
5143 if (is_reference (var)
5144 && !useless_type_conversion_p (TREE_TYPE (placeholder),
5146 x = build_fold_addr_expr_loc (clause_loc, x);
5147 SET_DECL_VALUE_EXPR (placeholder, x);
5148 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
5149 tree new_vard = new_var;
5150 if (is_reference (var))
5152 gcc_assert (TREE_CODE (new_var) == MEM_REF);
5153 new_vard = TREE_OPERAND (new_var, 0);
5154 gcc_assert (DECL_P (new_vard));
5157 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
5158 idx, lane, ivar, lvar))
5160 if (new_vard == new_var)
5162 gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
5163 SET_DECL_VALUE_EXPR (new_var, ivar);
5167 SET_DECL_VALUE_EXPR (new_vard,
5168 build_fold_addr_expr (ivar));
5169 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
5171 x = lang_hooks.decls.omp_clause_default_ctor
5172 (c, unshare_expr (ivar),
5173 build_outer_var_ref (var, ctx));
5175 gimplify_and_add (x, &llist[0]);
5176 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
5178 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
5179 lower_omp (&tseq, ctx);
5180 gimple_seq_add_seq (&llist[0], tseq);
5182 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
5183 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
5184 lower_omp (&tseq, ctx);
5185 gimple_seq_add_seq (&llist[1], tseq);
5186 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
5187 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
5188 if (new_vard == new_var)
5189 SET_DECL_VALUE_EXPR (new_var, lvar);
5191 SET_DECL_VALUE_EXPR (new_vard,
5192 build_fold_addr_expr (lvar));
5193 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
5198 gimplify_stmt (&dtor, &tseq);
5199 gimple_seq_add_seq (&llist[1], tseq);
5203 /* If this is a reference to constant size reduction var
5204 with placeholder, we haven't emitted the initializer
5205 for it because it is undesirable if SIMD arrays are used.
5206 But if they aren't used, we need to emit the deferred
5207 initialization now. */
5208 else if (is_reference (var) && is_simd)
5209 handle_simd_reference (clause_loc, new_vard, ilist);
5210 x = lang_hooks.decls.omp_clause_default_ctor
5211 (c, unshare_expr (new_var),
5212 build_outer_var_ref (var, ctx));
5214 gimplify_and_add (x, ilist);
5215 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
5217 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
5218 lower_omp (&tseq, ctx);
5219 gimple_seq_add_seq (ilist, tseq);
5221 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
5224 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
5225 lower_omp (&tseq, ctx);
5226 gimple_seq_add_seq (dlist, tseq);
5227 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
5229 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
5234 x = omp_reduction_init (c, TREE_TYPE (new_var));
5235 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
5236 enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
5238 /* reduction(-:var) sums up the partial results, so it
5239 acts identically to reduction(+:var). */
5240 if (code == MINUS_EXPR)
5243 tree new_vard = new_var;
5244 if (is_simd && is_reference (var))
5246 gcc_assert (TREE_CODE (new_var) == MEM_REF);
5247 new_vard = TREE_OPERAND (new_var, 0);
5248 gcc_assert (DECL_P (new_vard));
5251 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
5252 idx, lane, ivar, lvar))
5254 tree ref = build_outer_var_ref (var, ctx);
5256 gimplify_assign (unshare_expr (ivar), x, &llist[0]);
5258 x = build2 (code, TREE_TYPE (ref), ref, ivar);
5259 ref = build_outer_var_ref (var, ctx);
5260 gimplify_assign (ref, x, &llist[1]);
5262 if (new_vard != new_var)
5264 SET_DECL_VALUE_EXPR (new_vard,
5265 build_fold_addr_expr (lvar));
5266 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
5271 if (is_reference (var) && is_simd)
5272 handle_simd_reference (clause_loc, new_vard, ilist);
5273 gimplify_assign (new_var, x, ilist);
5276 tree ref = build_outer_var_ref (var, ctx);
5278 x = build2 (code, TREE_TYPE (ref), ref, new_var);
5279 ref = build_outer_var_ref (var, ctx);
5280 gimplify_assign (ref, x, dlist);
5294 tree uid = create_tmp_var (ptr_type_node, "simduid");
5295 /* Don't want uninit warnings on simduid, it is always uninitialized,
5296 but we use it not for the value, but for the DECL_UID only. */
5297 TREE_NO_WARNING (uid) = 1;
5299 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid);
5300 gimple_call_set_lhs (g, lane);
5301 gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
5302 gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
5303 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
5304 OMP_CLAUSE__SIMDUID__DECL (c) = uid;
5305 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
5306 gimple_omp_for_set_clauses (ctx->stmt, c);
5307 g = gimple_build_assign (lane, INTEGER_CST,
5308 build_int_cst (unsigned_type_node, 0));
5309 gimple_seq_add_stmt (ilist, g);
5310 for (int i = 0; i < 2; i++)
5313 tree vf = create_tmp_var (unsigned_type_node);
5314 g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
5315 gimple_call_set_lhs (g, vf);
5316 gimple_seq *seq = i == 0 ? ilist : dlist;
5317 gimple_seq_add_stmt (seq, g);
5318 tree t = build_int_cst (unsigned_type_node, 0);
5319 g = gimple_build_assign (idx, INTEGER_CST, t);
5320 gimple_seq_add_stmt (seq, g);
5321 tree body = create_artificial_label (UNKNOWN_LOCATION);
5322 tree header = create_artificial_label (UNKNOWN_LOCATION);
5323 tree end = create_artificial_label (UNKNOWN_LOCATION);
5324 gimple_seq_add_stmt (seq, gimple_build_goto (header));
5325 gimple_seq_add_stmt (seq, gimple_build_label (body));
5326 gimple_seq_add_seq (seq, llist[i]);
5327 t = build_int_cst (unsigned_type_node, 1);
5328 g = gimple_build_assign (idx, PLUS_EXPR, idx, t);
5329 gimple_seq_add_stmt (seq, g);
5330 gimple_seq_add_stmt (seq, gimple_build_label (header));
5331 g = gimple_build_cond (LT_EXPR, idx, vf, body, end);
5332 gimple_seq_add_stmt (seq, g);
5333 gimple_seq_add_stmt (seq, gimple_build_label (end));
5337 /* The copyin sequence is not to be executed by the main thread, since
5338 that would result in self-copies. Perhaps not visible to scalars,
5339 but it certainly is to C++ operator=. */
5342 x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
5344 x = build2 (NE_EXPR, boolean_type_node, x,
5345 build_int_cst (TREE_TYPE (x), 0));
5346 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
5347 gimplify_and_add (x, ilist);
5350 /* If any copyin variable is passed by reference, we must ensure the
5351 master thread doesn't modify it before it is copied over in all
5352 threads. Similarly for variables in both firstprivate and
5353 lastprivate clauses we need to ensure the lastprivate copying
5354 happens after firstprivate copying in all threads. And similarly
5355 for UDRs if initializer expression refers to omp_orig. */
5356 if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref)
5358 /* Don't add any barrier for #pragma omp simd or
5359 #pragma omp distribute. */
5360 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
5361 || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR)
5362 gimple_seq_add_stmt (ilist, build_omp_barrier (NULL_TREE));
5365 /* If max_vf is non-zero, then we can use only a vectorization factor
5366 up to the max_vf we chose. So stick it into the safelen clause. */
5369 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
5370 OMP_CLAUSE_SAFELEN);
5372 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) == INTEGER_CST
5373 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
5376 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
5377 OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
5379 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
5380 gimple_omp_for_set_clauses (ctx->stmt, c);
5386 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5387 both parallel and workshare constructs. PREDICATE may be NULL if it's
5391 lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list,
5394 tree x, c, label = NULL, orig_clauses = clauses;
5395 bool par_clauses = false;
5396 tree simduid = NULL, lastlane = NULL;
5398 /* Early exit if there are no lastprivate or linear clauses. */
5399 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
5400 if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
5401 || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
5402 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
5404 if (clauses == NULL)
5406 /* If this was a workshare clause, see if it had been combined
5407 with its parallel. In that case, look for the clauses on the
5408 parallel statement itself. */
5409 if (is_parallel_ctx (ctx))
5413 if (ctx == NULL || !is_parallel_ctx (ctx))
5416 clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
5417 OMP_CLAUSE_LASTPRIVATE);
5418 if (clauses == NULL)
5426 tree label_true, arm1, arm2;
5428 label = create_artificial_label (UNKNOWN_LOCATION);
5429 label_true = create_artificial_label (UNKNOWN_LOCATION);
5430 arm1 = TREE_OPERAND (predicate, 0);
5431 arm2 = TREE_OPERAND (predicate, 1);
5432 gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
5433 gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
5434 stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2,
5436 gimple_seq_add_stmt (stmt_list, stmt);
5437 gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
5440 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
5441 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
5443 simduid = find_omp_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
5445 simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
5448 for (c = clauses; c ;)
5451 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
5453 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
5454 || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
5455 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
5457 var = OMP_CLAUSE_DECL (c);
5458 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
5459 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)
5460 && is_taskloop_ctx (ctx))
5462 gcc_checking_assert (ctx->outer && is_task_ctx (ctx->outer));
5463 new_var = lookup_decl (var, ctx->outer);
5466 new_var = lookup_decl (var, ctx);
5468 if (simduid && DECL_HAS_VALUE_EXPR_P (new_var))
5470 tree val = DECL_VALUE_EXPR (new_var);
5471 if (TREE_CODE (val) == ARRAY_REF
5472 && VAR_P (TREE_OPERAND (val, 0))
5473 && lookup_attribute ("omp simd array",
5474 DECL_ATTRIBUTES (TREE_OPERAND (val,
5477 if (lastlane == NULL)
5479 lastlane = create_tmp_var (unsigned_type_node);
5481 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
5483 TREE_OPERAND (val, 1));
5484 gimple_call_set_lhs (g, lastlane);
5485 gimple_seq_add_stmt (stmt_list, g);
5487 new_var = build4 (ARRAY_REF, TREE_TYPE (val),
5488 TREE_OPERAND (val, 0), lastlane,
5489 NULL_TREE, NULL_TREE);
5493 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
5494 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
5496 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
5497 gimple_seq_add_seq (stmt_list,
5498 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
5499 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
5501 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
5502 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
5504 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
5505 gimple_seq_add_seq (stmt_list,
5506 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
5507 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
5511 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
5512 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c))
5514 gcc_checking_assert (is_taskloop_ctx (ctx));
5515 tree ovar = maybe_lookup_decl_in_outer_ctx (var,
5517 if (is_global_var (ovar))
5521 x = build_outer_var_ref (var, ctx, OMP_CLAUSE_LASTPRIVATE);
5522 if (is_reference (var))
5523 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
5524 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
5525 gimplify_and_add (x, stmt_list);
5527 c = OMP_CLAUSE_CHAIN (c);
5528 if (c == NULL && !par_clauses)
5530 /* If this was a workshare clause, see if it had been combined
5531 with its parallel. In that case, continue looking for the
5532 clauses also on the parallel statement itself. */
5533 if (is_parallel_ctx (ctx))
5537 if (ctx == NULL || !is_parallel_ctx (ctx))
5540 c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
5541 OMP_CLAUSE_LASTPRIVATE);
5547 gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
5550 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
5551 (which might be a placeholder). INNER is true if this is an inner
5552 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
5553 join markers. Generate the before-loop forking sequence in
5554 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
5555 general form of these sequences is
5557 GOACC_REDUCTION_SETUP
5559 GOACC_REDUCTION_INIT
5561 GOACC_REDUCTION_FINI
5563 GOACC_REDUCTION_TEARDOWN. */
5566 lower_oacc_reductions (location_t loc, tree clauses, tree level, bool inner,
5567 gcall *fork, gcall *join, gimple_seq *fork_seq,
5568 gimple_seq *join_seq, omp_context *ctx)
5570 gimple_seq before_fork = NULL;
5571 gimple_seq after_fork = NULL;
5572 gimple_seq before_join = NULL;
5573 gimple_seq after_join = NULL;
5574 tree init_code = NULL_TREE, fini_code = NULL_TREE,
5575 setup_code = NULL_TREE, teardown_code = NULL_TREE;
5576 unsigned offset = 0;
5578 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
5579 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
5581 tree orig = OMP_CLAUSE_DECL (c);
5582 tree var = maybe_lookup_decl (orig, ctx);
5583 tree ref_to_res = NULL_TREE;
5584 tree incoming, outgoing, v1, v2, v3;
5585 bool is_private = false;
5587 enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
5588 if (rcode == MINUS_EXPR)
5590 else if (rcode == TRUTH_ANDIF_EXPR)
5591 rcode = BIT_AND_EXPR;
5592 else if (rcode == TRUTH_ORIF_EXPR)
5593 rcode = BIT_IOR_EXPR;
5594 tree op = build_int_cst (unsigned_type_node, rcode);
5599 incoming = outgoing = var;
5603 /* See if an outer construct also reduces this variable. */
5604 omp_context *outer = ctx;
5606 while (omp_context *probe = outer->outer)
5608 enum gimple_code type = gimple_code (probe->stmt);
5613 case GIMPLE_OMP_FOR:
5614 cls = gimple_omp_for_clauses (probe->stmt);
5617 case GIMPLE_OMP_TARGET:
5618 if (gimple_omp_target_kind (probe->stmt)
5619 != GF_OMP_TARGET_KIND_OACC_PARALLEL)
5622 cls = gimple_omp_target_clauses (probe->stmt);
5630 for (; cls; cls = OMP_CLAUSE_CHAIN (cls))
5631 if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_REDUCTION
5632 && orig == OMP_CLAUSE_DECL (cls))
5634 incoming = outgoing = lookup_decl (orig, probe);
5635 goto has_outer_reduction;
5637 else if ((OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_FIRSTPRIVATE
5638 || OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_PRIVATE)
5639 && orig == OMP_CLAUSE_DECL (cls))
5647 /* This is the outermost construct with this reduction,
5648 see if there's a mapping for it. */
5649 if (gimple_code (outer->stmt) == GIMPLE_OMP_TARGET
5650 && maybe_lookup_field (orig, outer) && !is_private)
5652 ref_to_res = build_receiver_ref (orig, false, outer);
5653 if (is_reference (orig))
5654 ref_to_res = build_simple_mem_ref (ref_to_res);
5656 tree type = TREE_TYPE (var);
5657 if (POINTER_TYPE_P (type))
5658 type = TREE_TYPE (type);
5661 incoming = omp_reduction_init_op (loc, rcode, type);
5663 else if (ctx->outer)
5664 incoming = outgoing = lookup_decl (orig, ctx->outer);
5666 incoming = outgoing = orig;
5668 has_outer_reduction:;
5672 ref_to_res = integer_zero_node;
5674 if (is_reference (orig))
5676 tree type = TREE_TYPE (var);
5677 const char *id = IDENTIFIER_POINTER (DECL_NAME (var));
5681 tree x = create_tmp_var (TREE_TYPE (type), id);
5682 gimplify_assign (var, build_fold_addr_expr (x), fork_seq);
5685 v1 = create_tmp_var (type, id);
5686 v2 = create_tmp_var (type, id);
5687 v3 = create_tmp_var (type, id);
5689 gimplify_assign (v1, var, fork_seq);
5690 gimplify_assign (v2, var, fork_seq);
5691 gimplify_assign (v3, var, fork_seq);
5693 var = build_simple_mem_ref (var);
5694 v1 = build_simple_mem_ref (v1);
5695 v2 = build_simple_mem_ref (v2);
5696 v3 = build_simple_mem_ref (v3);
5697 outgoing = build_simple_mem_ref (outgoing);
5699 if (!TREE_CONSTANT (incoming))
5700 incoming = build_simple_mem_ref (incoming);
5705 /* Determine position in reduction buffer, which may be used
5707 enum machine_mode mode = TYPE_MODE (TREE_TYPE (var));
5708 unsigned align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
5709 offset = (offset + align - 1) & ~(align - 1);
5710 tree off = build_int_cst (sizetype, offset);
5711 offset += GET_MODE_SIZE (mode);
5715 init_code = build_int_cst (integer_type_node,
5716 IFN_GOACC_REDUCTION_INIT);
5717 fini_code = build_int_cst (integer_type_node,
5718 IFN_GOACC_REDUCTION_FINI);
5719 setup_code = build_int_cst (integer_type_node,
5720 IFN_GOACC_REDUCTION_SETUP);
5721 teardown_code = build_int_cst (integer_type_node,
5722 IFN_GOACC_REDUCTION_TEARDOWN);
5726 = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
5727 TREE_TYPE (var), 6, setup_code,
5728 unshare_expr (ref_to_res),
5729 incoming, level, op, off);
5731 = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
5732 TREE_TYPE (var), 6, init_code,
5733 unshare_expr (ref_to_res),
5734 v1, level, op, off);
5736 = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
5737 TREE_TYPE (var), 6, fini_code,
5738 unshare_expr (ref_to_res),
5739 v2, level, op, off);
5741 = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
5742 TREE_TYPE (var), 6, teardown_code,
5743 ref_to_res, v3, level, op, off);
5745 gimplify_assign (v1, setup_call, &before_fork);
5746 gimplify_assign (v2, init_call, &after_fork);
5747 gimplify_assign (v3, fini_call, &before_join);
5748 gimplify_assign (outgoing, teardown_call, &after_join);
5751 /* Now stitch things together. */
5752 gimple_seq_add_seq (fork_seq, before_fork);
5754 gimple_seq_add_stmt (fork_seq, fork);
5755 gimple_seq_add_seq (fork_seq, after_fork);
5757 gimple_seq_add_seq (join_seq, before_join);
5759 gimple_seq_add_stmt (join_seq, join);
5760 gimple_seq_add_seq (join_seq, after_join);
5763 /* Generate code to implement the REDUCTION clauses. */
5766 lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx)
5768 gimple_seq sub_seq = NULL;
5773 /* OpenACC loop reductions are handled elsewhere. */
5774 if (is_gimple_omp_oacc (ctx->stmt))
5777 /* SIMD reductions are handled in lower_rec_input_clauses. */
5778 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
5779 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
5782 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5783 update in that case, otherwise use a lock. */
5784 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
5785 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
5787 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
5788 || TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
5790 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5800 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
5802 tree var, ref, new_var, orig_var;
5803 enum tree_code code;
5804 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
5806 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5809 orig_var = var = OMP_CLAUSE_DECL (c);
5810 if (TREE_CODE (var) == MEM_REF)
5812 var = TREE_OPERAND (var, 0);
5813 if (TREE_CODE (var) == POINTER_PLUS_EXPR)
5814 var = TREE_OPERAND (var, 0);
5815 if (TREE_CODE (var) == INDIRECT_REF
5816 || TREE_CODE (var) == ADDR_EXPR)
5817 var = TREE_OPERAND (var, 0);
5819 if (is_variable_sized (var))
5821 gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
5822 var = DECL_VALUE_EXPR (var);
5823 gcc_assert (TREE_CODE (var) == INDIRECT_REF);
5824 var = TREE_OPERAND (var, 0);
5825 gcc_assert (DECL_P (var));
5828 new_var = lookup_decl (var, ctx);
5829 if (var == OMP_CLAUSE_DECL (c) && is_reference (var))
5830 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
5831 ref = build_outer_var_ref (var, ctx);
5832 code = OMP_CLAUSE_REDUCTION_CODE (c);
5834 /* reduction(-:var) sums up the partial results, so it acts
5835 identically to reduction(+:var). */
5836 if (code == MINUS_EXPR)
5841 tree addr = build_fold_addr_expr_loc (clause_loc, ref);
5843 addr = save_expr (addr);
5844 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
5845 x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var);
5846 x = build2 (OMP_ATOMIC, void_type_node, addr, x);
5847 gimplify_and_add (x, stmt_seqp);
5850 else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
5852 tree d = OMP_CLAUSE_DECL (c);
5853 tree type = TREE_TYPE (d);
5854 tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
5855 tree i = create_tmp_var (TREE_TYPE (v), NULL);
5856 tree ptype = build_pointer_type (TREE_TYPE (type));
5857 tree bias = TREE_OPERAND (d, 1);
5858 d = TREE_OPERAND (d, 0);
5859 if (TREE_CODE (d) == POINTER_PLUS_EXPR)
5861 tree b = TREE_OPERAND (d, 1);
5862 b = maybe_lookup_decl (b, ctx);
5865 b = TREE_OPERAND (d, 1);
5866 b = maybe_lookup_decl_in_outer_ctx (b, ctx);
5868 if (integer_zerop (bias))
5872 bias = fold_convert_loc (clause_loc, TREE_TYPE (b), bias);
5873 bias = fold_build2_loc (clause_loc, PLUS_EXPR,
5874 TREE_TYPE (b), b, bias);
5876 d = TREE_OPERAND (d, 0);
5878 /* For ref build_outer_var_ref already performs this, so
5879 only new_var needs a dereference. */
5880 if (TREE_CODE (d) == INDIRECT_REF)
5882 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
5883 gcc_assert (is_reference (var) && var == orig_var);
5885 else if (TREE_CODE (d) == ADDR_EXPR)
5887 if (orig_var == var)
5889 new_var = build_fold_addr_expr (new_var);
5890 ref = build_fold_addr_expr (ref);
5895 gcc_assert (orig_var == var);
5896 if (is_reference (var))
5897 ref = build_fold_addr_expr (ref);
5901 tree t = maybe_lookup_decl (v, ctx);
5905 v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5906 gimplify_expr (&v, stmt_seqp, NULL, is_gimple_val, fb_rvalue);
5908 if (!integer_zerop (bias))
5910 bias = fold_convert_loc (clause_loc, sizetype, bias);
5911 new_var = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
5912 TREE_TYPE (new_var), new_var,
5913 unshare_expr (bias));
5914 ref = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
5915 TREE_TYPE (ref), ref, bias);
5917 new_var = fold_convert_loc (clause_loc, ptype, new_var);
5918 ref = fold_convert_loc (clause_loc, ptype, ref);
5919 tree m = create_tmp_var (ptype, NULL);
5920 gimplify_assign (m, new_var, stmt_seqp);
5922 m = create_tmp_var (ptype, NULL);
5923 gimplify_assign (m, ref, stmt_seqp);
5925 gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), stmt_seqp);
5926 tree body = create_artificial_label (UNKNOWN_LOCATION);
5927 tree end = create_artificial_label (UNKNOWN_LOCATION);
5928 gimple_seq_add_stmt (&sub_seq, gimple_build_label (body));
5929 tree priv = build_simple_mem_ref_loc (clause_loc, new_var);
5930 tree out = build_simple_mem_ref_loc (clause_loc, ref);
5931 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5933 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
5934 tree decl_placeholder
5935 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
5936 SET_DECL_VALUE_EXPR (placeholder, out);
5937 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
5938 SET_DECL_VALUE_EXPR (decl_placeholder, priv);
5939 DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
5940 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
5941 gimple_seq_add_seq (&sub_seq,
5942 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
5943 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
5944 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
5945 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
5949 x = build2 (code, TREE_TYPE (out), out, priv);
5950 out = unshare_expr (out);
5951 gimplify_assign (out, x, &sub_seq);
5953 gimple *g = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
5954 TYPE_SIZE_UNIT (TREE_TYPE (type)));
5955 gimple_seq_add_stmt (&sub_seq, g);
5956 g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
5957 TYPE_SIZE_UNIT (TREE_TYPE (type)));
5958 gimple_seq_add_stmt (&sub_seq, g);
5959 g = gimple_build_assign (i, PLUS_EXPR, i,
5960 build_int_cst (TREE_TYPE (i), 1));
5961 gimple_seq_add_stmt (&sub_seq, g);
5962 g = gimple_build_cond (LE_EXPR, i, v, body, end);
5963 gimple_seq_add_stmt (&sub_seq, g);
5964 gimple_seq_add_stmt (&sub_seq, gimple_build_label (end));
5966 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5968 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
5970 if (is_reference (var)
5971 && !useless_type_conversion_p (TREE_TYPE (placeholder),
5973 ref = build_fold_addr_expr_loc (clause_loc, ref);
5974 SET_DECL_VALUE_EXPR (placeholder, ref);
5975 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
5976 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
5977 gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
5978 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
5979 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
5983 x = build2 (code, TREE_TYPE (ref), ref, new_var);
5984 ref = build_outer_var_ref (var, ctx);
5985 gimplify_assign (ref, x, &sub_seq);
5989 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START),
5991 gimple_seq_add_stmt (stmt_seqp, stmt);
5993 gimple_seq_add_seq (stmt_seqp, sub_seq);
5995 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
5997 gimple_seq_add_stmt (stmt_seqp, stmt);
6001 /* Generate code to implement the COPYPRIVATE clauses. */
6004 lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
6009 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
6011 tree var, new_var, ref, x;
6013 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
6015 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
6018 var = OMP_CLAUSE_DECL (c);
6019 by_ref = use_pointer_for_field (var, NULL);
6021 ref = build_sender_ref (var, ctx);
6022 x = new_var = lookup_decl_in_outer_ctx (var, ctx);
6025 x = build_fold_addr_expr_loc (clause_loc, new_var);
6026 x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
6028 gimplify_assign (ref, x, slist);
6030 ref = build_receiver_ref (var, false, ctx);
6033 ref = fold_convert_loc (clause_loc,
6034 build_pointer_type (TREE_TYPE (new_var)),
6036 ref = build_fold_indirect_ref_loc (clause_loc, ref);
6038 if (is_reference (var))
6040 ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
6041 ref = build_simple_mem_ref_loc (clause_loc, ref);
6042 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
6044 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
6045 gimplify_and_add (x, rlist);
6050 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
6051 and REDUCTION from the sender (aka parent) side. */
6054 lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
6058 int ignored_looptemp = 0;
6059 bool is_taskloop = false;
6061 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
6062 by GOMP_taskloop. */
6063 if (is_task_ctx (ctx) && gimple_omp_task_taskloop_p (ctx->stmt))
6065 ignored_looptemp = 2;
6069 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
6071 tree val, ref, x, var;
6072 bool by_ref, do_in = false, do_out = false;
6073 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
6075 switch (OMP_CLAUSE_CODE (c))
6077 case OMP_CLAUSE_PRIVATE:
6078 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
6081 case OMP_CLAUSE_FIRSTPRIVATE:
6082 case OMP_CLAUSE_COPYIN:
6083 case OMP_CLAUSE_LASTPRIVATE:
6084 case OMP_CLAUSE_REDUCTION:
6086 case OMP_CLAUSE_SHARED:
6087 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
6090 case OMP_CLAUSE__LOOPTEMP_:
6091 if (ignored_looptemp)
6101 val = OMP_CLAUSE_DECL (c);
6102 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6103 && TREE_CODE (val) == MEM_REF)
6105 val = TREE_OPERAND (val, 0);
6106 if (TREE_CODE (val) == POINTER_PLUS_EXPR)
6107 val = TREE_OPERAND (val, 0);
6108 if (TREE_CODE (val) == INDIRECT_REF
6109 || TREE_CODE (val) == ADDR_EXPR)
6110 val = TREE_OPERAND (val, 0);
6111 if (is_variable_sized (val))
6115 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
6116 outer taskloop region. */
6117 omp_context *ctx_for_o = ctx;
6119 && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
6120 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
6121 ctx_for_o = ctx->outer;
6123 var = lookup_decl_in_outer_ctx (val, ctx_for_o);
6125 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
6126 && is_global_var (var))
6129 t = omp_member_access_dummy_var (var);
6132 var = DECL_VALUE_EXPR (var);
6133 tree o = maybe_lookup_decl_in_outer_ctx (t, ctx_for_o);
6135 var = unshare_and_remap (var, t, o);
6137 var = unshare_expr (var);
6140 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED)
6142 /* Handle taskloop firstprivate/lastprivate, where the
6143 lastprivate on GIMPLE_OMP_TASK is represented as
6144 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
6145 tree f = lookup_sfield ((splay_tree_key) &DECL_UID (val), ctx);
6146 x = omp_build_component_ref (ctx->sender_decl, f);
6147 if (use_pointer_for_field (val, ctx))
6148 var = build_fold_addr_expr (var);
6149 gimplify_assign (x, var, ilist);
6150 DECL_ABSTRACT_ORIGIN (f) = NULL;
6154 if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
6155 || val == OMP_CLAUSE_DECL (c))
6156 && is_variable_sized (val))
6158 by_ref = use_pointer_for_field (val, NULL);
6160 switch (OMP_CLAUSE_CODE (c))
6162 case OMP_CLAUSE_FIRSTPRIVATE:
6163 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)
6165 && is_task_ctx (ctx))
6166 TREE_NO_WARNING (var) = 1;
6170 case OMP_CLAUSE_PRIVATE:
6171 case OMP_CLAUSE_COPYIN:
6172 case OMP_CLAUSE__LOOPTEMP_:
6176 case OMP_CLAUSE_LASTPRIVATE:
6177 if (by_ref || is_reference (val))
6179 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
6186 if (lang_hooks.decls.omp_private_outer_ref (val))
6191 case OMP_CLAUSE_REDUCTION:
6193 if (val == OMP_CLAUSE_DECL (c))
6194 do_out = !(by_ref || is_reference (val));
6196 by_ref = TREE_CODE (TREE_TYPE (val)) == ARRAY_TYPE;
6205 ref = build_sender_ref (val, ctx);
6206 x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
6207 gimplify_assign (ref, x, ilist);
6208 if (is_task_ctx (ctx))
6209 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
6214 ref = build_sender_ref (val, ctx);
6215 gimplify_assign (var, ref, olist);
6220 /* Generate code to implement SHARED from the sender (aka parent)
6221 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
6222 list things that got automatically shared. */
6225 lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
6227 tree var, ovar, nvar, t, f, x, record_type;
6229 if (ctx->record_type == NULL)
6232 record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
6233 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
6235 ovar = DECL_ABSTRACT_ORIGIN (f);
6236 if (!ovar || TREE_CODE (ovar) == FIELD_DECL)
6239 nvar = maybe_lookup_decl (ovar, ctx);
6240 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar))
6243 /* If CTX is a nested parallel directive. Find the immediately
6244 enclosing parallel or workshare construct that contains a
6245 mapping for OVAR. */
6246 var = lookup_decl_in_outer_ctx (ovar, ctx);
6248 t = omp_member_access_dummy_var (var);
6251 var = DECL_VALUE_EXPR (var);
6252 tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
6254 var = unshare_and_remap (var, t, o);
6256 var = unshare_expr (var);
6259 if (use_pointer_for_field (ovar, ctx))
6261 x = build_sender_ref (ovar, ctx);
6262 var = build_fold_addr_expr (var);
6263 gimplify_assign (x, var, ilist);
6267 x = build_sender_ref (ovar, ctx);
6268 gimplify_assign (x, var, ilist);
6270 if (!TREE_READONLY (var)
6271 /* We don't need to receive a new reference to a result
6272 or parm decl. In fact we may not store to it as we will
6273 invalidate any pending RSO and generate wrong gimple
6275 && !((TREE_CODE (var) == RESULT_DECL
6276 || TREE_CODE (var) == PARM_DECL)
6277 && DECL_BY_REFERENCE (var)))
6279 x = build_sender_ref (ovar, ctx);
6280 gimplify_assign (var, x, olist);
6286 /* Emit an OpenACC head marker call, encapulating the partitioning and
6287 other information that must be processed by the target compiler.
6288 Return the maximum number of dimensions the associated loop might
6289 be partitioned over. */
6292 lower_oacc_head_mark (location_t loc, tree ddvar, tree clauses,
6293 gimple_seq *seq, omp_context *ctx)
6295 unsigned levels = 0;
6297 tree gang_static = NULL_TREE;
6298 auto_vec<tree, 5> args;
6300 args.quick_push (build_int_cst
6301 (integer_type_node, IFN_UNIQUE_OACC_HEAD_MARK));
6302 args.quick_push (ddvar);
6303 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
6305 switch (OMP_CLAUSE_CODE (c))
6307 case OMP_CLAUSE_GANG:
6308 tag |= OLF_DIM_GANG;
6309 gang_static = OMP_CLAUSE_GANG_STATIC_EXPR (c);
6310 /* static:* is represented by -1, and we can ignore it, as
6311 scheduling is always static. */
6312 if (gang_static && integer_minus_onep (gang_static))
6313 gang_static = NULL_TREE;
6317 case OMP_CLAUSE_WORKER:
6318 tag |= OLF_DIM_WORKER;
6322 case OMP_CLAUSE_VECTOR:
6323 tag |= OLF_DIM_VECTOR;
6327 case OMP_CLAUSE_SEQ:
6331 case OMP_CLAUSE_AUTO:
6335 case OMP_CLAUSE_INDEPENDENT:
6336 tag |= OLF_INDEPENDENT;
6346 if (DECL_P (gang_static))
6347 gang_static = build_outer_var_ref (gang_static, ctx);
6348 tag |= OLF_GANG_STATIC;
6351 /* In a parallel region, loops are implicitly INDEPENDENT. */
6352 omp_context *tgt = enclosing_target_ctx (ctx);
6353 if (!tgt || is_oacc_parallel (tgt))
6354 tag |= OLF_INDEPENDENT;
6356 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
6357 if (!(tag & (((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1) << OLF_DIM_BASE)
6361 /* Ensure at least one level. */
6365 args.quick_push (build_int_cst (integer_type_node, levels));
6366 args.quick_push (build_int_cst (integer_type_node, tag));
6368 args.quick_push (gang_static);
6370 gcall *call = gimple_build_call_internal_vec (IFN_UNIQUE, args);
6371 gimple_set_location (call, loc);
6372 gimple_set_lhs (call, ddvar);
6373 gimple_seq_add_stmt (seq, call);
6378 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6379 partitioning level of the enclosed region. */
6382 lower_oacc_loop_marker (location_t loc, tree ddvar, bool head,
6383 tree tofollow, gimple_seq *seq)
6385 int marker_kind = (head ? IFN_UNIQUE_OACC_HEAD_MARK
6386 : IFN_UNIQUE_OACC_TAIL_MARK);
6387 tree marker = build_int_cst (integer_type_node, marker_kind);
6388 int nargs = 2 + (tofollow != NULL_TREE);
6389 gcall *call = gimple_build_call_internal (IFN_UNIQUE, nargs,
6390 marker, ddvar, tofollow);
6391 gimple_set_location (call, loc);
6392 gimple_set_lhs (call, ddvar);
6393 gimple_seq_add_stmt (seq, call);
6396 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6397 the loop clauses, from which we extract reductions. Initialize
6401 lower_oacc_head_tail (location_t loc, tree clauses,
6402 gimple_seq *head, gimple_seq *tail, omp_context *ctx)
6405 tree ddvar = create_tmp_var (integer_type_node, ".data_dep");
6406 gimple_seq_add_stmt (head, gimple_build_assign (ddvar, integer_zero_node));
6408 unsigned count = lower_oacc_head_mark (loc, ddvar, clauses, head, ctx);
6410 lower_oacc_loop_marker (loc, ddvar, false, integer_zero_node, tail);
6412 tree fork_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_FORK);
6413 tree join_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_JOIN);
6415 for (unsigned done = 1; count; count--, done++)
6417 gimple_seq fork_seq = NULL;
6418 gimple_seq join_seq = NULL;
6420 tree place = build_int_cst (integer_type_node, -1);
6421 gcall *fork = gimple_build_call_internal (IFN_UNIQUE, 3,
6422 fork_kind, ddvar, place);
6423 gimple_set_location (fork, loc);
6424 gimple_set_lhs (fork, ddvar);
6426 gcall *join = gimple_build_call_internal (IFN_UNIQUE, 3,
6427 join_kind, ddvar, place);
6428 gimple_set_location (join, loc);
6429 gimple_set_lhs (join, ddvar);
6431 /* Mark the beginning of this level sequence. */
6433 lower_oacc_loop_marker (loc, ddvar, true,
6434 build_int_cst (integer_type_node, count),
6436 lower_oacc_loop_marker (loc, ddvar, false,
6437 build_int_cst (integer_type_node, done),
6440 lower_oacc_reductions (loc, clauses, place, inner,
6441 fork, join, &fork_seq, &join_seq, ctx);
6443 /* Append this level to head. */
6444 gimple_seq_add_seq (head, fork_seq);
6445 /* Prepend it to tail. */
6446 gimple_seq_add_seq (&join_seq, *tail);
6452 /* Mark the end of the sequence. */
6453 lower_oacc_loop_marker (loc, ddvar, true, NULL_TREE, head);
6454 lower_oacc_loop_marker (loc, ddvar, false, NULL_TREE, tail);
6457 /* A convenience function to build an empty GIMPLE_COND with just the
6461 gimple_build_cond_empty (tree cond)
6463 enum tree_code pred_code;
6466 gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs);
6467 return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE);
6470 /* Return true if a parallel REGION is within a declare target function or
6471 within a target region and is not a part of a gridified target. */
6474 parallel_needs_hsa_kernel_p (struct omp_region *region)
6476 bool indirect = false;
6477 for (region = region->outer; region; region = region->outer)
6479 if (region->type == GIMPLE_OMP_PARALLEL)
6481 else if (region->type == GIMPLE_OMP_TARGET)
6483 gomp_target *tgt_stmt
6484 = as_a <gomp_target *> (last_stmt (region->entry));
6486 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt),
6487 OMP_CLAUSE__GRIDDIM_))
6494 if (lookup_attribute ("omp declare target",
6495 DECL_ATTRIBUTES (current_function_decl)))
6501 static void expand_omp_build_assign (gimple_stmt_iterator *, tree, tree,
6504 /* Build the function calls to GOMP_parallel_start etc to actually
6505 generate the parallel operation. REGION is the parallel region
6506 being expanded. BB is the block where to insert the code. WS_ARGS
6507 will be set if this is a call to a combined parallel+workshare
6508 construct, it contains the list of additional arguments needed by
6509 the workshare construct. */
6512 expand_parallel_call (struct omp_region *region, basic_block bb,
6513 gomp_parallel *entry_stmt,
6514 vec<tree, va_gc> *ws_args)
6516 tree t, t1, t2, val, cond, c, clauses, flags;
6517 gimple_stmt_iterator gsi;
6519 enum built_in_function start_ix;
6521 location_t clause_loc;
6522 vec<tree, va_gc> *args;
6524 clauses = gimple_omp_parallel_clauses (entry_stmt);
6526 /* Determine what flavor of GOMP_parallel we will be
6528 start_ix = BUILT_IN_GOMP_PARALLEL;
6529 if (is_combined_parallel (region))
6531 switch (region->inner->type)
6533 case GIMPLE_OMP_FOR:
6534 gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
6535 switch (region->inner->sched_kind)
6537 case OMP_CLAUSE_SCHEDULE_RUNTIME:
6540 case OMP_CLAUSE_SCHEDULE_DYNAMIC:
6541 case OMP_CLAUSE_SCHEDULE_GUIDED:
6542 if (region->inner->sched_modifiers
6543 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC)
6545 start_ix2 = 3 + region->inner->sched_kind;
6550 start_ix2 = region->inner->sched_kind;
6553 start_ix2 += (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC;
6554 start_ix = (enum built_in_function) start_ix2;
6556 case GIMPLE_OMP_SECTIONS:
6557 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS;
6564 /* By default, the value of NUM_THREADS is zero (selected at run time)
6565 and there is no conditional. */
6567 val = build_int_cst (unsigned_type_node, 0);
6568 flags = build_int_cst (unsigned_type_node, 0);
6570 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
6572 cond = OMP_CLAUSE_IF_EXPR (c);
6574 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS);
6577 val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
6578 clause_loc = OMP_CLAUSE_LOCATION (c);
6581 clause_loc = gimple_location (entry_stmt);
6583 c = find_omp_clause (clauses, OMP_CLAUSE_PROC_BIND);
6585 flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c));
6587 /* Ensure 'val' is of the correct type. */
6588 val = fold_convert_loc (clause_loc, unsigned_type_node, val);
6590 /* If we found the clause 'if (cond)', build either
6591 (cond != 0) or (cond ? val : 1u). */
6594 cond = gimple_boolify (cond);
6596 if (integer_zerop (val))
6597 val = fold_build2_loc (clause_loc,
6598 EQ_EXPR, unsigned_type_node, cond,
6599 build_int_cst (TREE_TYPE (cond), 0));
6602 basic_block cond_bb, then_bb, else_bb;
6603 edge e, e_then, e_else;
6604 tree tmp_then, tmp_else, tmp_join, tmp_var;
6606 tmp_var = create_tmp_var (TREE_TYPE (val));
6607 if (gimple_in_ssa_p (cfun))
6609 tmp_then = make_ssa_name (tmp_var);
6610 tmp_else = make_ssa_name (tmp_var);
6611 tmp_join = make_ssa_name (tmp_var);
6620 e = split_block_after_labels (bb);
6625 then_bb = create_empty_bb (cond_bb);
6626 else_bb = create_empty_bb (then_bb);
6627 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
6628 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
6630 stmt = gimple_build_cond_empty (cond);
6631 gsi = gsi_start_bb (cond_bb);
6632 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6634 gsi = gsi_start_bb (then_bb);
6635 expand_omp_build_assign (&gsi, tmp_then, val, true);
6637 gsi = gsi_start_bb (else_bb);
6638 expand_omp_build_assign (&gsi, tmp_else,
6639 build_int_cst (unsigned_type_node, 1),
6642 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
6643 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
6644 add_bb_to_loop (then_bb, cond_bb->loop_father);
6645 add_bb_to_loop (else_bb, cond_bb->loop_father);
6646 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
6647 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
6649 if (gimple_in_ssa_p (cfun))
6651 gphi *phi = create_phi_node (tmp_join, bb);
6652 add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION);
6653 add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION);
6659 gsi = gsi_start_bb (bb);
6660 val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE,
6661 false, GSI_CONTINUE_LINKING);
6664 gsi = gsi_last_bb (bb);
6665 t = gimple_omp_parallel_data_arg (entry_stmt);
6667 t1 = null_pointer_node;
6669 t1 = build_fold_addr_expr (t);
6670 tree child_fndecl = gimple_omp_parallel_child_fn (entry_stmt);
6671 t2 = build_fold_addr_expr (child_fndecl);
6673 vec_alloc (args, 4 + vec_safe_length (ws_args));
6674 args->quick_push (t2);
6675 args->quick_push (t1);
6676 args->quick_push (val);
6678 args->splice (*ws_args);
6679 args->quick_push (flags);
6681 t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
6682 builtin_decl_explicit (start_ix), args);
6684 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6685 false, GSI_CONTINUE_LINKING);
6687 if (hsa_gen_requested_p ()
6688 && parallel_needs_hsa_kernel_p (region))
6690 cgraph_node *child_cnode = cgraph_node::get (child_fndecl);
6691 hsa_register_kernel (child_cnode);
6695 /* Insert a function call whose name is FUNC_NAME with the information from
6696 ENTRY_STMT into the basic_block BB. */
6699 expand_cilk_for_call (basic_block bb, gomp_parallel *entry_stmt,
6700 vec <tree, va_gc> *ws_args)
6703 gimple_stmt_iterator gsi;
6704 vec <tree, va_gc> *args;
6706 gcc_assert (vec_safe_length (ws_args) == 2);
6707 tree func_name = (*ws_args)[0];
6708 tree grain = (*ws_args)[1];
6710 tree clauses = gimple_omp_parallel_clauses (entry_stmt);
6711 tree count = find_omp_clause (clauses, OMP_CLAUSE__CILK_FOR_COUNT_);
6712 gcc_assert (count != NULL_TREE);
6713 count = OMP_CLAUSE_OPERAND (count, 0);
6715 gsi = gsi_last_bb (bb);
6716 t = gimple_omp_parallel_data_arg (entry_stmt);
6718 t1 = null_pointer_node;
6720 t1 = build_fold_addr_expr (t);
6721 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
6723 vec_alloc (args, 4);
6724 args->quick_push (t2);
6725 args->quick_push (t1);
6726 args->quick_push (count);
6727 args->quick_push (grain);
6728 t = build_call_expr_loc_vec (UNKNOWN_LOCATION, func_name, args);
6730 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, false,
6731 GSI_CONTINUE_LINKING);
6734 /* Build the function call to GOMP_task to actually
6735 generate the task operation. BB is the block where to insert the code. */
6738 expand_task_call (struct omp_region *region, basic_block bb,
6739 gomp_task *entry_stmt)
6742 gimple_stmt_iterator gsi;
6743 location_t loc = gimple_location (entry_stmt);
6745 tree clauses = gimple_omp_task_clauses (entry_stmt);
6747 tree ifc = find_omp_clause (clauses, OMP_CLAUSE_IF);
6748 tree untied = find_omp_clause (clauses, OMP_CLAUSE_UNTIED);
6749 tree mergeable = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE);
6750 tree depend = find_omp_clause (clauses, OMP_CLAUSE_DEPEND);
6751 tree finalc = find_omp_clause (clauses, OMP_CLAUSE_FINAL);
6752 tree priority = find_omp_clause (clauses, OMP_CLAUSE_PRIORITY);
6755 = (untied ? GOMP_TASK_FLAG_UNTIED : 0)
6756 | (mergeable ? GOMP_TASK_FLAG_MERGEABLE : 0)
6757 | (depend ? GOMP_TASK_FLAG_DEPEND : 0);
6759 bool taskloop_p = gimple_omp_task_taskloop_p (entry_stmt);
6760 tree startvar = NULL_TREE, endvar = NULL_TREE, step = NULL_TREE;
6761 tree num_tasks = NULL_TREE;
6765 gimple *g = last_stmt (region->outer->entry);
6766 gcc_assert (gimple_code (g) == GIMPLE_OMP_FOR
6767 && gimple_omp_for_kind (g) == GF_OMP_FOR_KIND_TASKLOOP);
6768 struct omp_for_data fd;
6769 extract_omp_for_data (as_a <gomp_for *> (g), &fd, NULL);
6770 startvar = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
6771 endvar = find_omp_clause (OMP_CLAUSE_CHAIN (startvar),
6772 OMP_CLAUSE__LOOPTEMP_);
6773 startvar = OMP_CLAUSE_DECL (startvar);
6774 endvar = OMP_CLAUSE_DECL (endvar);
6775 step = fold_convert_loc (loc, fd.iter_type, fd.loop.step);
6776 if (fd.loop.cond_code == LT_EXPR)
6777 iflags |= GOMP_TASK_FLAG_UP;
6778 tree tclauses = gimple_omp_for_clauses (g);
6779 num_tasks = find_omp_clause (tclauses, OMP_CLAUSE_NUM_TASKS);
6781 num_tasks = OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks);
6784 num_tasks = find_omp_clause (tclauses, OMP_CLAUSE_GRAINSIZE);
6787 iflags |= GOMP_TASK_FLAG_GRAINSIZE;
6788 num_tasks = OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks);
6791 num_tasks = integer_zero_node;
6793 num_tasks = fold_convert_loc (loc, long_integer_type_node, num_tasks);
6794 if (ifc == NULL_TREE)
6795 iflags |= GOMP_TASK_FLAG_IF;
6796 if (find_omp_clause (tclauses, OMP_CLAUSE_NOGROUP))
6797 iflags |= GOMP_TASK_FLAG_NOGROUP;
6798 ull = fd.iter_type == long_long_unsigned_type_node;
6801 iflags |= GOMP_TASK_FLAG_PRIORITY;
6803 tree flags = build_int_cst (unsigned_type_node, iflags);
6805 tree cond = boolean_true_node;
6810 tree t = gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc));
6811 t = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, t,
6812 build_int_cst (unsigned_type_node,
6814 build_int_cst (unsigned_type_node, 0));
6815 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node,
6819 cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc));
6824 tree t = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc));
6825 t = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, t,
6826 build_int_cst (unsigned_type_node,
6827 GOMP_TASK_FLAG_FINAL),
6828 build_int_cst (unsigned_type_node, 0));
6829 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, t);
6832 depend = OMP_CLAUSE_DECL (depend);
6834 depend = build_int_cst (ptr_type_node, 0);
6836 priority = fold_convert (integer_type_node,
6837 OMP_CLAUSE_PRIORITY_EXPR (priority));
6839 priority = integer_zero_node;
6841 gsi = gsi_last_bb (bb);
6842 tree t = gimple_omp_task_data_arg (entry_stmt);
6844 t2 = null_pointer_node;
6846 t2 = build_fold_addr_expr_loc (loc, t);
6847 t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt));
6848 t = gimple_omp_task_copy_fn (entry_stmt);
6850 t3 = null_pointer_node;
6852 t3 = build_fold_addr_expr_loc (loc, t);
6855 t = build_call_expr (ull
6856 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL)
6857 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP),
6859 gimple_omp_task_arg_size (entry_stmt),
6860 gimple_omp_task_arg_align (entry_stmt), flags,
6861 num_tasks, priority, startvar, endvar, step);
6863 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
6865 gimple_omp_task_arg_size (entry_stmt),
6866 gimple_omp_task_arg_align (entry_stmt), cond, flags,
6869 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6870 false, GSI_CONTINUE_LINKING);
6874 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6875 catch handler and return it. This prevents programs from violating the
6876 structured block semantics with throws. */
6879 maybe_catch_exception (gimple_seq body)
6884 if (!flag_exceptions)
6887 if (lang_hooks.eh_protect_cleanup_actions != NULL)
6888 decl = lang_hooks.eh_protect_cleanup_actions ();
6890 decl = builtin_decl_explicit (BUILT_IN_TRAP);
6892 g = gimple_build_eh_must_not_throw (decl);
6893 g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
6896 return gimple_seq_alloc_with_stmt (g);
6899 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6902 vec2chain (vec<tree, va_gc> *v)
6904 tree chain = NULL_TREE, t;
6907 FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t)
6909 DECL_CHAIN (t) = chain;
6917 /* Remove barriers in REGION->EXIT's block. Note that this is only
6918 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6919 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6920 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6924 remove_exit_barrier (struct omp_region *region)
6926 gimple_stmt_iterator gsi;
6927 basic_block exit_bb;
6931 int any_addressable_vars = -1;
6933 exit_bb = region->exit;
6935 /* If the parallel region doesn't return, we don't have REGION->EXIT
6940 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6941 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6942 statements that can appear in between are extremely limited -- no
6943 memory operations at all. Here, we allow nothing at all, so the
6944 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6945 gsi = gsi_last_bb (exit_bb);
6946 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
6948 if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
6951 FOR_EACH_EDGE (e, ei, exit_bb->preds)
6953 gsi = gsi_last_bb (e->src);
6954 if (gsi_end_p (gsi))
6956 stmt = gsi_stmt (gsi);
6957 if (gimple_code (stmt) == GIMPLE_OMP_RETURN
6958 && !gimple_omp_return_nowait_p (stmt))
6960 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6961 in many cases. If there could be tasks queued, the barrier
6962 might be needed to let the tasks run before some local
6963 variable of the parallel that the task uses as shared
6964 runs out of scope. The task can be spawned either
6965 from within current function (this would be easy to check)
6966 or from some function it calls and gets passed an address
6967 of such a variable. */
6968 if (any_addressable_vars < 0)
6970 gomp_parallel *parallel_stmt
6971 = as_a <gomp_parallel *> (last_stmt (region->entry));
6972 tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt);
6973 tree local_decls, block, decl;
6976 any_addressable_vars = 0;
6977 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl)
6978 if (TREE_ADDRESSABLE (decl))
6980 any_addressable_vars = 1;
6983 for (block = gimple_block (stmt);
6984 !any_addressable_vars
6986 && TREE_CODE (block) == BLOCK;
6987 block = BLOCK_SUPERCONTEXT (block))
6989 for (local_decls = BLOCK_VARS (block);
6991 local_decls = DECL_CHAIN (local_decls))
6992 if (TREE_ADDRESSABLE (local_decls))
6994 any_addressable_vars = 1;
6997 if (block == gimple_block (parallel_stmt))
7001 if (!any_addressable_vars)
7002 gimple_omp_return_set_nowait (stmt);
7008 remove_exit_barriers (struct omp_region *region)
7010 if (region->type == GIMPLE_OMP_PARALLEL)
7011 remove_exit_barrier (region);
7015 region = region->inner;
7016 remove_exit_barriers (region);
7017 while (region->next)
7019 region = region->next;
7020 remove_exit_barriers (region);
7025 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
7026 calls. These can't be declared as const functions, but
7027 within one parallel body they are constant, so they can be
7028 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
7029 which are declared const. Similarly for task body, except
7030 that in untied task omp_get_thread_num () can change at any task
7031 scheduling point. */
7034 optimize_omp_library_calls (gimple *entry_stmt)
7037 gimple_stmt_iterator gsi;
7038 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
7039 tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree);
7040 tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
7041 tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree);
7042 bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
7043 && find_omp_clause (gimple_omp_task_clauses (entry_stmt),
7044 OMP_CLAUSE_UNTIED) != NULL);
7046 FOR_EACH_BB_FN (bb, cfun)
7047 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7049 gimple *call = gsi_stmt (gsi);
7052 if (is_gimple_call (call)
7053 && (decl = gimple_call_fndecl (call))
7054 && DECL_EXTERNAL (decl)
7055 && TREE_PUBLIC (decl)
7056 && DECL_INITIAL (decl) == NULL)
7060 if (DECL_NAME (decl) == thr_num_id)
7062 /* In #pragma omp task untied omp_get_thread_num () can change
7063 during the execution of the task region. */
7066 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
7068 else if (DECL_NAME (decl) == num_thr_id)
7069 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
7073 if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in)
7074 || gimple_call_num_args (call) != 0)
7077 if (flag_exceptions && !TREE_NOTHROW (decl))
7080 if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE
7081 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)),
7082 TREE_TYPE (TREE_TYPE (built_in))))
7085 gimple_call_set_fndecl (call, built_in);
7090 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
7094 expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *)
7098 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
7099 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t))
7102 if (TREE_CODE (t) == ADDR_EXPR)
7103 recompute_tree_invariant_for_addr_expr (t);
7105 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
7109 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
7112 expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from,
7115 bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to);
7116 from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE,
7117 !after, after ? GSI_CONTINUE_LINKING
7119 gimple *stmt = gimple_build_assign (to, from);
7121 gsi_insert_after (gsi_p, stmt, GSI_CONTINUE_LINKING);
7123 gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT);
7124 if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL)
7125 || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL))
7127 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
7128 gimple_regimplify_operands (stmt, &gsi);
7132 /* Expand the OpenMP parallel or task directive starting at REGION. */
7135 expand_omp_taskreg (struct omp_region *region)
7137 basic_block entry_bb, exit_bb, new_bb;
7138 struct function *child_cfun;
7139 tree child_fn, block, t;
7140 gimple_stmt_iterator gsi;
7141 gimple *entry_stmt, *stmt;
7143 vec<tree, va_gc> *ws_args;
7145 entry_stmt = last_stmt (region->entry);
7146 child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
7147 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
7149 entry_bb = region->entry;
7150 if (gimple_code (entry_stmt) == GIMPLE_OMP_TASK)
7151 exit_bb = region->cont;
7153 exit_bb = region->exit;
7157 && gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL
7158 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt),
7159 OMP_CLAUSE__CILK_FOR_COUNT_) != NULL_TREE);
7162 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
7163 and the inner statement contains the name of the built-in function
7165 ws_args = region->inner->ws_args;
7166 else if (is_combined_parallel (region))
7167 ws_args = region->ws_args;
7171 if (child_cfun->cfg)
7173 /* Due to inlining, it may happen that we have already outlined
7174 the region, in which case all we need to do is make the
7175 sub-graph unreachable and emit the parallel call. */
7176 edge entry_succ_e, exit_succ_e;
7178 entry_succ_e = single_succ_edge (entry_bb);
7180 gsi = gsi_last_bb (entry_bb);
7181 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
7182 || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
7183 gsi_remove (&gsi, true);
7188 exit_succ_e = single_succ_edge (exit_bb);
7189 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
7191 remove_edge_and_dominated_blocks (entry_succ_e);
7195 unsigned srcidx, dstidx, num;
7197 /* If the parallel region needs data sent from the parent
7198 function, then the very first statement (except possible
7199 tree profile counter updates) of the parallel body
7200 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7201 &.OMP_DATA_O is passed as an argument to the child function,
7202 we need to replace it with the argument as seen by the child
7205 In most cases, this will end up being the identity assignment
7206 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7207 a function call that has been inlined, the original PARM_DECL
7208 .OMP_DATA_I may have been converted into a different local
7209 variable. In which case, we need to keep the assignment. */
7210 if (gimple_omp_taskreg_data_arg (entry_stmt))
7212 basic_block entry_succ_bb
7213 = single_succ_p (entry_bb) ? single_succ (entry_bb)
7214 : FALLTHRU_EDGE (entry_bb)->dest;
7216 gimple *parcopy_stmt = NULL;
7218 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
7222 gcc_assert (!gsi_end_p (gsi));
7223 stmt = gsi_stmt (gsi);
7224 if (gimple_code (stmt) != GIMPLE_ASSIGN)
7227 if (gimple_num_ops (stmt) == 2)
7229 tree arg = gimple_assign_rhs1 (stmt);
7231 /* We're ignore the subcode because we're
7232 effectively doing a STRIP_NOPS. */
7234 if (TREE_CODE (arg) == ADDR_EXPR
7235 && TREE_OPERAND (arg, 0)
7236 == gimple_omp_taskreg_data_arg (entry_stmt))
7238 parcopy_stmt = stmt;
7244 gcc_assert (parcopy_stmt != NULL);
7245 arg = DECL_ARGUMENTS (child_fn);
7247 if (!gimple_in_ssa_p (cfun))
7249 if (gimple_assign_lhs (parcopy_stmt) == arg)
7250 gsi_remove (&gsi, true);
7253 /* ?? Is setting the subcode really necessary ?? */
7254 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg));
7255 gimple_assign_set_rhs1 (parcopy_stmt, arg);
7260 tree lhs = gimple_assign_lhs (parcopy_stmt);
7261 gcc_assert (SSA_NAME_VAR (lhs) == arg);
7262 /* We'd like to set the rhs to the default def in the child_fn,
7263 but it's too early to create ssa names in the child_fn.
7264 Instead, we set the rhs to the parm. In
7265 move_sese_region_to_fn, we introduce a default def for the
7266 parm, map the parm to it's default def, and once we encounter
7267 this stmt, replace the parm with the default def. */
7268 gimple_assign_set_rhs1 (parcopy_stmt, arg);
7269 update_stmt (parcopy_stmt);
7273 /* Declare local variables needed in CHILD_CFUN. */
7274 block = DECL_INITIAL (child_fn);
7275 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
7276 /* The gimplifier could record temporaries in parallel/task block
7277 rather than in containing function's local_decls chain,
7278 which would mean cgraph missed finalizing them. Do it now. */
7279 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
7280 if (TREE_CODE (t) == VAR_DECL
7282 && !DECL_EXTERNAL (t))
7283 varpool_node::finalize_decl (t);
7284 DECL_SAVED_TREE (child_fn) = NULL;
7285 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7286 gimple_set_body (child_fn, NULL);
7287 TREE_USED (block) = 1;
7289 /* Reset DECL_CONTEXT on function arguments. */
7290 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
7291 DECL_CONTEXT (t) = child_fn;
7293 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
7294 so that it can be moved to the child function. */
7295 gsi = gsi_last_bb (entry_bb);
7296 stmt = gsi_stmt (gsi);
7297 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
7298 || gimple_code (stmt) == GIMPLE_OMP_TASK));
7299 e = split_block (entry_bb, stmt);
7300 gsi_remove (&gsi, true);
7303 if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
7304 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7307 e2 = make_edge (e->src, BRANCH_EDGE (entry_bb)->dest, EDGE_ABNORMAL);
7308 gcc_assert (e2->dest == region->exit);
7309 remove_edge (BRANCH_EDGE (entry_bb));
7310 set_immediate_dominator (CDI_DOMINATORS, e2->dest, e->src);
7311 gsi = gsi_last_bb (region->exit);
7312 gcc_assert (!gsi_end_p (gsi)
7313 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
7314 gsi_remove (&gsi, true);
7317 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
7320 gsi = gsi_last_bb (exit_bb);
7321 gcc_assert (!gsi_end_p (gsi)
7322 && (gimple_code (gsi_stmt (gsi))
7323 == (e2 ? GIMPLE_OMP_CONTINUE : GIMPLE_OMP_RETURN)));
7324 stmt = gimple_build_return (NULL);
7325 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
7326 gsi_remove (&gsi, true);
7329 /* Move the parallel region into CHILD_CFUN. */
7331 if (gimple_in_ssa_p (cfun))
7333 init_tree_ssa (child_cfun);
7334 init_ssa_operands (child_cfun);
7335 child_cfun->gimple_df->in_ssa_p = true;
7339 block = gimple_block (entry_stmt);
7341 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
7343 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
7346 basic_block dest_bb = e2->dest;
7348 make_edge (new_bb, dest_bb, EDGE_FALLTHRU);
7350 set_immediate_dominator (CDI_DOMINATORS, dest_bb, new_bb);
7352 /* When the OMP expansion process cannot guarantee an up-to-date
7353 loop tree arrange for the child function to fixup loops. */
7354 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
7355 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
7357 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7358 num = vec_safe_length (child_cfun->local_decls);
7359 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
7361 t = (*child_cfun->local_decls)[srcidx];
7362 if (DECL_CONTEXT (t) == cfun->decl)
7364 if (srcidx != dstidx)
7365 (*child_cfun->local_decls)[dstidx] = t;
7369 vec_safe_truncate (child_cfun->local_decls, dstidx);
7371 /* Inform the callgraph about the new function. */
7372 child_cfun->curr_properties = cfun->curr_properties;
7373 child_cfun->has_simduid_loops |= cfun->has_simduid_loops;
7374 child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops;
7375 cgraph_node *node = cgraph_node::get_create (child_fn);
7376 node->parallelized_function = 1;
7377 cgraph_node::add_new_function (child_fn, true);
7379 bool need_asm = DECL_ASSEMBLER_NAME_SET_P (current_function_decl)
7380 && !DECL_ASSEMBLER_NAME_SET_P (child_fn);
7382 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7383 fixed in a following pass. */
7384 push_cfun (child_cfun);
7386 assign_assembler_name_if_neeeded (child_fn);
7389 optimize_omp_library_calls (entry_stmt);
7390 cgraph_edge::rebuild_edges ();
7392 /* Some EH regions might become dead, see PR34608. If
7393 pass_cleanup_cfg isn't the first pass to happen with the
7394 new child, these dead EH edges might cause problems.
7395 Clean them up now. */
7396 if (flag_exceptions)
7399 bool changed = false;
7401 FOR_EACH_BB_FN (bb, cfun)
7402 changed |= gimple_purge_dead_eh_edges (bb);
7404 cleanup_tree_cfg ();
7406 if (gimple_in_ssa_p (cfun))
7407 update_ssa (TODO_update_ssa);
7408 if (flag_checking && !loops_state_satisfies_p (LOOPS_NEED_FIXUP))
7409 verify_loop_structure ();
7412 if (dump_file && !gimple_in_ssa_p (cfun))
7414 omp_any_child_fn_dumped = true;
7415 dump_function_header (dump_file, child_fn, dump_flags);
7416 dump_function_to_file (child_fn, dump_file, dump_flags);
7420 /* Emit a library call to launch the children threads. */
7422 expand_cilk_for_call (new_bb,
7423 as_a <gomp_parallel *> (entry_stmt), ws_args);
7424 else if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
7425 expand_parallel_call (region, new_bb,
7426 as_a <gomp_parallel *> (entry_stmt), ws_args);
7428 expand_task_call (region, new_bb, as_a <gomp_task *> (entry_stmt));
7429 if (gimple_in_ssa_p (cfun))
7430 update_ssa (TODO_update_ssa_only_virtuals);
7433 /* Information about members of an OpenACC collapsed loop nest. */
7435 struct oacc_collapse
7437 tree base; /* Base value. */
7438 tree iters; /* Number of steps. */
7439 tree step; /* step size. */
7442 /* Helper for expand_oacc_for. Determine collapsed loop information.
7443 Fill in COUNTS array. Emit any initialization code before GSI.
7444 Return the calculated outer loop bound of BOUND_TYPE. */
7447 expand_oacc_collapse_init (const struct omp_for_data *fd,
7448 gimple_stmt_iterator *gsi,
7449 oacc_collapse *counts, tree bound_type)
7451 tree total = build_int_cst (bound_type, 1);
7454 gcc_assert (integer_onep (fd->loop.step));
7455 gcc_assert (integer_zerop (fd->loop.n1));
7457 for (ix = 0; ix != fd->collapse; ix++)
7459 const omp_for_data_loop *loop = &fd->loops[ix];
7461 tree iter_type = TREE_TYPE (loop->v);
7462 tree diff_type = iter_type;
7463 tree plus_type = iter_type;
7465 gcc_assert (loop->cond_code == fd->loop.cond_code);
7467 if (POINTER_TYPE_P (iter_type))
7468 plus_type = sizetype;
7469 if (POINTER_TYPE_P (diff_type) || TYPE_UNSIGNED (diff_type))
7470 diff_type = signed_type_for (diff_type);
7474 tree s = loop->step;
7475 bool up = loop->cond_code == LT_EXPR;
7476 tree dir = build_int_cst (diff_type, up ? +1 : -1);
7480 b = force_gimple_operand_gsi (gsi, b, true, NULL_TREE,
7481 true, GSI_SAME_STMT);
7482 e = force_gimple_operand_gsi (gsi, e, true, NULL_TREE,
7483 true, GSI_SAME_STMT);
7485 /* Convert the step, avoiding possible unsigned->signed overflow. */
7486 negating = !up && TYPE_UNSIGNED (TREE_TYPE (s));
7488 s = fold_build1 (NEGATE_EXPR, TREE_TYPE (s), s);
7489 s = fold_convert (diff_type, s);
7491 s = fold_build1 (NEGATE_EXPR, diff_type, s);
7492 s = force_gimple_operand_gsi (gsi, s, true, NULL_TREE,
7493 true, GSI_SAME_STMT);
7495 /* Determine the range, avoiding possible unsigned->signed overflow. */
7496 negating = !up && TYPE_UNSIGNED (iter_type);
7497 expr = fold_build2 (MINUS_EXPR, plus_type,
7498 fold_convert (plus_type, negating ? b : e),
7499 fold_convert (plus_type, negating ? e : b));
7500 expr = fold_convert (diff_type, expr);
7502 expr = fold_build1 (NEGATE_EXPR, diff_type, expr);
7503 tree range = force_gimple_operand_gsi
7504 (gsi, expr, true, NULL_TREE, true, GSI_SAME_STMT);
7506 /* Determine number of iterations. */
7507 expr = fold_build2 (MINUS_EXPR, diff_type, range, dir);
7508 expr = fold_build2 (PLUS_EXPR, diff_type, expr, s);
7509 expr = fold_build2 (TRUNC_DIV_EXPR, diff_type, expr, s);
7511 tree iters = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE,
7512 true, GSI_SAME_STMT);
7514 counts[ix].base = b;
7515 counts[ix].iters = iters;
7516 counts[ix].step = s;
7518 total = fold_build2 (MULT_EXPR, bound_type, total,
7519 fold_convert (bound_type, iters));
7525 /* Emit initializers for collapsed loop members. IVAR is the outer
7526 loop iteration variable, from which collapsed loop iteration values
7527 are calculated. COUNTS array has been initialized by
7528 expand_oacc_collapse_inits. */
7531 expand_oacc_collapse_vars (const struct omp_for_data *fd,
7532 gimple_stmt_iterator *gsi,
7533 const oacc_collapse *counts, tree ivar)
7535 tree ivar_type = TREE_TYPE (ivar);
7537 /* The most rapidly changing iteration variable is the innermost
7539 for (int ix = fd->collapse; ix--;)
7541 const omp_for_data_loop *loop = &fd->loops[ix];
7542 const oacc_collapse *collapse = &counts[ix];
7543 tree iter_type = TREE_TYPE (loop->v);
7544 tree diff_type = TREE_TYPE (collapse->step);
7545 tree plus_type = iter_type;
7546 enum tree_code plus_code = PLUS_EXPR;
7549 if (POINTER_TYPE_P (iter_type))
7551 plus_code = POINTER_PLUS_EXPR;
7552 plus_type = sizetype;
7555 expr = fold_build2 (TRUNC_MOD_EXPR, ivar_type, ivar,
7556 fold_convert (ivar_type, collapse->iters));
7557 expr = fold_build2 (MULT_EXPR, diff_type, fold_convert (diff_type, expr),
7559 expr = fold_build2 (plus_code, iter_type, collapse->base,
7560 fold_convert (plus_type, expr));
7561 expr = force_gimple_operand_gsi (gsi, expr, false, NULL_TREE,
7562 true, GSI_SAME_STMT);
7563 gassign *ass = gimple_build_assign (loop->v, expr);
7564 gsi_insert_before (gsi, ass, GSI_SAME_STMT);
7568 expr = fold_build2 (TRUNC_DIV_EXPR, ivar_type, ivar,
7569 fold_convert (ivar_type, collapse->iters));
7570 ivar = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE,
7571 true, GSI_SAME_STMT);
7577 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7578 of the combined collapse > 1 loop constructs, generate code like:
7579 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7584 count3 = (adj + N32 - N31) / STEP3;
7585 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7590 count2 = (adj + N22 - N21) / STEP2;
7591 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7596 count1 = (adj + N12 - N11) / STEP1;
7597 count = count1 * count2 * count3;
7598 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7600 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7601 of the combined loop constructs, just initialize COUNTS array
7602 from the _looptemp_ clauses. */
7604 /* NOTE: It *could* be better to moosh all of the BBs together,
7605 creating one larger BB with all the computation and the unexpected
7606 jump at the end. I.e.
7608 bool zero3, zero2, zero1, zero;
7611 count3 = (N32 - N31) /[cl] STEP3;
7613 count2 = (N22 - N21) /[cl] STEP2;
7615 count1 = (N12 - N11) /[cl] STEP1;
7616 zero = zero3 || zero2 || zero1;
7617 count = count1 * count2 * count3;
7618 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7620 After all, we expect the zero=false, and thus we expect to have to
7621 evaluate all of the comparison expressions, so short-circuiting
7622 oughtn't be a win. Since the condition isn't protecting a
7623 denominator, we're not concerned about divide-by-zero, so we can
7624 fully evaluate count even if a numerator turned out to be wrong.
7626 It seems like putting this all together would create much better
7627 scheduling opportunities, and less pressure on the chip's branch
7631 expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
7632 basic_block &entry_bb, tree *counts,
7633 basic_block &zero_iter1_bb, int &first_zero_iter1,
7634 basic_block &zero_iter2_bb, int &first_zero_iter2,
7635 basic_block &l2_dom_bb)
7637 tree t, type = TREE_TYPE (fd->loop.v);
7641 /* Collapsed loops need work for expansion into SSA form. */
7642 gcc_assert (!gimple_in_ssa_p (cfun));
7644 if (gimple_omp_for_combined_into_p (fd->for_stmt)
7645 && TREE_CODE (fd->loop.n2) != INTEGER_CST)
7647 gcc_assert (fd->ordered == 0);
7648 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7649 isn't supposed to be handled, as the inner loop doesn't
7651 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7652 OMP_CLAUSE__LOOPTEMP_);
7653 gcc_assert (innerc);
7654 for (i = 0; i < fd->collapse; i++)
7656 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7657 OMP_CLAUSE__LOOPTEMP_);
7658 gcc_assert (innerc);
7660 counts[i] = OMP_CLAUSE_DECL (innerc);
7662 counts[0] = NULL_TREE;
7667 for (i = fd->collapse; i < fd->ordered; i++)
7669 tree itype = TREE_TYPE (fd->loops[i].v);
7670 counts[i] = NULL_TREE;
7671 t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
7672 fold_convert (itype, fd->loops[i].n1),
7673 fold_convert (itype, fd->loops[i].n2));
7674 if (t && integer_zerop (t))
7676 for (i = fd->collapse; i < fd->ordered; i++)
7677 counts[i] = build_int_cst (type, 0);
7681 for (i = 0; i < (fd->ordered ? fd->ordered : fd->collapse); i++)
7683 tree itype = TREE_TYPE (fd->loops[i].v);
7685 if (i >= fd->collapse && counts[i])
7687 if ((SSA_VAR_P (fd->loop.n2) || i >= fd->collapse)
7688 && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
7689 fold_convert (itype, fd->loops[i].n1),
7690 fold_convert (itype, fd->loops[i].n2)))
7691 == NULL_TREE || !integer_onep (t)))
7695 n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1));
7696 n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE,
7697 true, GSI_SAME_STMT);
7698 n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
7699 n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE,
7700 true, GSI_SAME_STMT);
7701 cond_stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
7702 NULL_TREE, NULL_TREE);
7703 gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT);
7704 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
7705 expand_omp_regimplify_p, NULL, NULL)
7706 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
7707 expand_omp_regimplify_p, NULL, NULL))
7709 *gsi = gsi_for_stmt (cond_stmt);
7710 gimple_regimplify_operands (cond_stmt, gsi);
7712 e = split_block (entry_bb, cond_stmt);
7713 basic_block &zero_iter_bb
7714 = i < fd->collapse ? zero_iter1_bb : zero_iter2_bb;
7715 int &first_zero_iter
7716 = i < fd->collapse ? first_zero_iter1 : first_zero_iter2;
7717 if (zero_iter_bb == NULL)
7719 gassign *assign_stmt;
7720 first_zero_iter = i;
7721 zero_iter_bb = create_empty_bb (entry_bb);
7722 add_bb_to_loop (zero_iter_bb, entry_bb->loop_father);
7723 *gsi = gsi_after_labels (zero_iter_bb);
7724 if (i < fd->collapse)
7725 assign_stmt = gimple_build_assign (fd->loop.n2,
7726 build_zero_cst (type));
7729 counts[i] = create_tmp_reg (type, ".count");
7731 = gimple_build_assign (counts[i], build_zero_cst (type));
7733 gsi_insert_before (gsi, assign_stmt, GSI_SAME_STMT);
7734 set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb,
7737 ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
7738 ne->probability = REG_BR_PROB_BASE / 2000 - 1;
7739 e->flags = EDGE_TRUE_VALUE;
7740 e->probability = REG_BR_PROB_BASE - ne->probability;
7741 if (l2_dom_bb == NULL)
7742 l2_dom_bb = entry_bb;
7744 *gsi = gsi_last_bb (entry_bb);
7747 if (POINTER_TYPE_P (itype))
7748 itype = signed_type_for (itype);
7749 t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
7751 t = fold_build2 (PLUS_EXPR, itype,
7752 fold_convert (itype, fd->loops[i].step), t);
7753 t = fold_build2 (PLUS_EXPR, itype, t,
7754 fold_convert (itype, fd->loops[i].n2));
7755 t = fold_build2 (MINUS_EXPR, itype, t,
7756 fold_convert (itype, fd->loops[i].n1));
7757 /* ?? We could probably use CEIL_DIV_EXPR instead of
7758 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7759 generate the same code in the end because generically we
7760 don't know that the values involved must be negative for
7762 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
7763 t = fold_build2 (TRUNC_DIV_EXPR, itype,
7764 fold_build1 (NEGATE_EXPR, itype, t),
7765 fold_build1 (NEGATE_EXPR, itype,
7766 fold_convert (itype,
7767 fd->loops[i].step)));
7769 t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
7770 fold_convert (itype, fd->loops[i].step));
7771 t = fold_convert (type, t);
7772 if (TREE_CODE (t) == INTEGER_CST)
7776 if (i < fd->collapse || i != first_zero_iter2)
7777 counts[i] = create_tmp_reg (type, ".count");
7778 expand_omp_build_assign (gsi, counts[i], t);
7780 if (SSA_VAR_P (fd->loop.n2) && i < fd->collapse)
7785 t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]);
7786 expand_omp_build_assign (gsi, fd->loop.n2, t);
7792 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7794 V3 = N31 + (T % count3) * STEP3;
7796 V2 = N21 + (T % count2) * STEP2;
7798 V1 = N11 + T * STEP1;
7799 if this loop doesn't have an inner loop construct combined with it.
7800 If it does have an inner loop construct combined with it and the
7801 iteration count isn't known constant, store values from counts array
7802 into its _looptemp_ temporaries instead. */
7805 expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
7806 tree *counts, gimple *inner_stmt, tree startvar)
7809 if (gimple_omp_for_combined_p (fd->for_stmt))
7811 /* If fd->loop.n2 is constant, then no propagation of the counts
7812 is needed, they are constant. */
7813 if (TREE_CODE (fd->loop.n2) == INTEGER_CST)
7816 tree clauses = gimple_code (inner_stmt) != GIMPLE_OMP_FOR
7817 ? gimple_omp_taskreg_clauses (inner_stmt)
7818 : gimple_omp_for_clauses (inner_stmt);
7819 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7820 isn't supposed to be handled, as the inner loop doesn't
7822 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
7823 gcc_assert (innerc);
7824 for (i = 0; i < fd->collapse; i++)
7826 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7827 OMP_CLAUSE__LOOPTEMP_);
7828 gcc_assert (innerc);
7831 tree tem = OMP_CLAUSE_DECL (innerc);
7832 tree t = fold_convert (TREE_TYPE (tem), counts[i]);
7833 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
7834 false, GSI_CONTINUE_LINKING);
7835 gassign *stmt = gimple_build_assign (tem, t);
7836 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
7842 tree type = TREE_TYPE (fd->loop.v);
7843 tree tem = create_tmp_reg (type, ".tem");
7844 gassign *stmt = gimple_build_assign (tem, startvar);
7845 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
7847 for (i = fd->collapse - 1; i >= 0; i--)
7849 tree vtype = TREE_TYPE (fd->loops[i].v), itype, t;
7851 if (POINTER_TYPE_P (vtype))
7852 itype = signed_type_for (vtype);
7854 t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]);
7857 t = fold_convert (itype, t);
7858 t = fold_build2 (MULT_EXPR, itype, t,
7859 fold_convert (itype, fd->loops[i].step));
7860 if (POINTER_TYPE_P (vtype))
7861 t = fold_build_pointer_plus (fd->loops[i].n1, t);
7863 t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
7864 t = force_gimple_operand_gsi (gsi, t,
7865 DECL_P (fd->loops[i].v)
7866 && TREE_ADDRESSABLE (fd->loops[i].v),
7868 GSI_CONTINUE_LINKING);
7869 stmt = gimple_build_assign (fd->loops[i].v, t);
7870 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
7873 t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]);
7874 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
7875 false, GSI_CONTINUE_LINKING);
7876 stmt = gimple_build_assign (tem, t);
7877 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
7883 /* Helper function for expand_omp_for_*. Generate code like:
7886 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7890 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7897 extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb,
7898 basic_block body_bb)
7900 basic_block last_bb, bb, collapse_bb = NULL;
7902 gimple_stmt_iterator gsi;
7908 for (i = fd->collapse - 1; i >= 0; i--)
7910 tree vtype = TREE_TYPE (fd->loops[i].v);
7912 bb = create_empty_bb (last_bb);
7913 add_bb_to_loop (bb, last_bb->loop_father);
7914 gsi = gsi_start_bb (bb);
7916 if (i < fd->collapse - 1)
7918 e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
7919 e->probability = REG_BR_PROB_BASE / 8;
7921 t = fd->loops[i + 1].n1;
7922 t = force_gimple_operand_gsi (&gsi, t,
7923 DECL_P (fd->loops[i + 1].v)
7924 && TREE_ADDRESSABLE (fd->loops[i
7927 GSI_CONTINUE_LINKING);
7928 stmt = gimple_build_assign (fd->loops[i + 1].v, t);
7929 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7934 set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
7936 if (POINTER_TYPE_P (vtype))
7937 t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
7939 t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step);
7940 t = force_gimple_operand_gsi (&gsi, t,
7941 DECL_P (fd->loops[i].v)
7942 && TREE_ADDRESSABLE (fd->loops[i].v),
7943 NULL_TREE, false, GSI_CONTINUE_LINKING);
7944 stmt = gimple_build_assign (fd->loops[i].v, t);
7945 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7949 t = fd->loops[i].n2;
7950 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7951 false, GSI_CONTINUE_LINKING);
7952 tree v = fd->loops[i].v;
7953 if (DECL_P (v) && TREE_ADDRESSABLE (v))
7954 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
7955 false, GSI_CONTINUE_LINKING);
7956 t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
7957 stmt = gimple_build_cond_empty (t);
7958 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7959 e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
7960 e->probability = REG_BR_PROB_BASE * 7 / 8;
7963 make_edge (bb, body_bb, EDGE_FALLTHRU);
7971 /* Expand #pragma omp ordered depend(source). */
7974 expand_omp_ordered_source (gimple_stmt_iterator *gsi, struct omp_for_data *fd,
7975 tree *counts, location_t loc)
7977 enum built_in_function source_ix
7978 = fd->iter_type == long_integer_type_node
7979 ? BUILT_IN_GOMP_DOACROSS_POST : BUILT_IN_GOMP_DOACROSS_ULL_POST;
7981 = gimple_build_call (builtin_decl_explicit (source_ix), 1,
7982 build_fold_addr_expr (counts[fd->ordered]));
7983 gimple_set_location (g, loc);
7984 gsi_insert_before (gsi, g, GSI_SAME_STMT);
7987 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7990 expand_omp_ordered_sink (gimple_stmt_iterator *gsi, struct omp_for_data *fd,
7991 tree *counts, tree c, location_t loc)
7993 auto_vec<tree, 10> args;
7994 enum built_in_function sink_ix
7995 = fd->iter_type == long_integer_type_node
7996 ? BUILT_IN_GOMP_DOACROSS_WAIT : BUILT_IN_GOMP_DOACROSS_ULL_WAIT;
7997 tree t, off, coff = NULL_TREE, deps = OMP_CLAUSE_DECL (c), cond = NULL_TREE;
7999 gimple_stmt_iterator gsi2 = *gsi;
8000 bool warned_step = false;
8002 for (i = 0; i < fd->ordered; i++)
8004 off = TREE_PURPOSE (deps);
8005 if (!integer_zerop (off))
8007 gcc_assert (fd->loops[i].cond_code == LT_EXPR
8008 || fd->loops[i].cond_code == GT_EXPR);
8009 bool forward = fd->loops[i].cond_code == LT_EXPR;
8010 if (forward ^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps))
8011 warning_at (loc, 0, "%<depend(sink)%> clause waiting for "
8012 "lexically later iteration");
8015 deps = TREE_CHAIN (deps);
8017 /* If all offsets corresponding to the collapsed loops are zero,
8018 this depend clause can be ignored. FIXME: but there is still a
8019 flush needed. We need to emit one __sync_synchronize () for it
8020 though (perhaps conditionally)? Solve this together with the
8021 conservative dependence folding optimization.
8022 if (i >= fd->collapse)
8025 deps = OMP_CLAUSE_DECL (c);
8027 edge e1 = split_block (gsi_bb (gsi2), gsi_stmt (gsi2));
8028 edge e2 = split_block_after_labels (e1->dest);
8030 *gsi = gsi_after_labels (e1->dest);
8031 for (i = 0; i < fd->ordered; i++)
8033 tree itype = TREE_TYPE (fd->loops[i].v);
8034 if (POINTER_TYPE_P (itype))
8037 deps = TREE_CHAIN (deps);
8038 off = TREE_PURPOSE (deps);
8039 tree s = fold_convert_loc (loc, itype, fd->loops[i].step);
8041 if (integer_zerop (off))
8042 t = boolean_true_node;
8046 tree co = fold_convert_loc (loc, itype, off);
8047 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
8049 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps))
8050 co = fold_build1_loc (loc, NEGATE_EXPR, itype, co);
8051 a = fold_build2_loc (loc, POINTER_PLUS_EXPR,
8052 TREE_TYPE (fd->loops[i].v), fd->loops[i].v,
8055 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps))
8056 a = fold_build2_loc (loc, MINUS_EXPR, TREE_TYPE (fd->loops[i].v),
8057 fd->loops[i].v, co);
8059 a = fold_build2_loc (loc, PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
8060 fd->loops[i].v, co);
8061 if (fd->loops[i].cond_code == LT_EXPR)
8063 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps))
8064 t = fold_build2_loc (loc, GE_EXPR, boolean_type_node, a,
8067 t = fold_build2_loc (loc, LT_EXPR, boolean_type_node, a,
8070 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps))
8071 t = fold_build2_loc (loc, GT_EXPR, boolean_type_node, a,
8074 t = fold_build2_loc (loc, LE_EXPR, boolean_type_node, a,
8078 cond = fold_build2_loc (loc, BIT_AND_EXPR, boolean_type_node, cond, t);
8082 off = fold_convert_loc (loc, itype, off);
8084 if (fd->loops[i].cond_code == LT_EXPR
8085 ? !integer_onep (fd->loops[i].step)
8086 : !integer_minus_onep (fd->loops[i].step))
8088 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
8089 t = fold_build2_loc (loc, TRUNC_MOD_EXPR, itype, off,
8090 fold_build1_loc (loc, NEGATE_EXPR, itype,
8093 t = fold_build2_loc (loc, TRUNC_MOD_EXPR, itype, off, s);
8094 t = fold_build2_loc (loc, EQ_EXPR, boolean_type_node, t,
8095 build_int_cst (itype, 0));
8096 if (integer_zerop (t) && !warned_step)
8098 warning_at (loc, 0, "%<depend(sink)%> refers to iteration never "
8099 "in the iteration space");
8102 cond = fold_build2_loc (loc, BIT_AND_EXPR, boolean_type_node,
8106 if (i <= fd->collapse - 1 && fd->collapse > 1)
8112 t = fold_build2_loc (loc, MINUS_EXPR, TREE_TYPE (fd->loops[i].v),
8113 fd->loops[i].v, fd->loops[i].n1);
8114 t = fold_convert_loc (loc, fd->iter_type, t);
8116 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
8117 off = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, off,
8118 fold_build1_loc (loc, NEGATE_EXPR, itype,
8121 off = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, off, s);
8122 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps))
8123 off = fold_build1_loc (loc, NEGATE_EXPR, itype, off);
8124 off = fold_convert_loc (loc, fd->iter_type, off);
8125 if (i <= fd->collapse - 1 && fd->collapse > 1)
8128 off = fold_build2_loc (loc, PLUS_EXPR, fd->iter_type, coff,
8130 if (i < fd->collapse - 1)
8132 coff = fold_build2_loc (loc, MULT_EXPR, fd->iter_type, off,
8137 off = unshare_expr (off);
8138 t = fold_build2_loc (loc, PLUS_EXPR, fd->iter_type, t, off);
8139 t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
8140 true, GSI_SAME_STMT);
8143 gimple *g = gimple_build_call_vec (builtin_decl_explicit (sink_ix), args);
8144 gimple_set_location (g, loc);
8145 gsi_insert_before (gsi, g, GSI_SAME_STMT);
8147 *gsi = gsi_last_bb (e1->src);
8148 cond = unshare_expr (cond);
8149 cond = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE, false,
8150 GSI_CONTINUE_LINKING);
8151 gsi_insert_after (gsi, gimple_build_cond_empty (cond), GSI_NEW_STMT);
8152 edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
8153 e3->probability = REG_BR_PROB_BASE / 8;
8154 e1->probability = REG_BR_PROB_BASE - e3->probability;
8155 e1->flags = EDGE_TRUE_VALUE;
8156 set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
8158 *gsi = gsi_after_labels (e2->dest);
8161 /* Expand all #pragma omp ordered depend(source) and
8162 #pragma omp ordered depend(sink:...) constructs in the current
8163 #pragma omp for ordered(n) region. */
8166 expand_omp_ordered_source_sink (struct omp_region *region,
8167 struct omp_for_data *fd, tree *counts,
8168 basic_block cont_bb)
8170 struct omp_region *inner;
8172 for (i = fd->collapse - 1; i < fd->ordered; i++)
8173 if (i == fd->collapse - 1 && fd->collapse > 1)
8174 counts[i] = NULL_TREE;
8175 else if (i >= fd->collapse && !cont_bb)
8176 counts[i] = build_zero_cst (fd->iter_type);
8177 else if (!POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v))
8178 && integer_onep (fd->loops[i].step))
8179 counts[i] = NULL_TREE;
8181 counts[i] = create_tmp_var (fd->iter_type, ".orditer");
8183 = build_array_type_nelts (fd->iter_type, fd->ordered - fd->collapse + 1);
8184 counts[fd->ordered] = create_tmp_var (atype, ".orditera");
8185 TREE_ADDRESSABLE (counts[fd->ordered]) = 1;
8187 for (inner = region->inner; inner; inner = inner->next)
8188 if (inner->type == GIMPLE_OMP_ORDERED)
8190 gomp_ordered *ord_stmt = inner->ord_stmt;
8191 gimple_stmt_iterator gsi = gsi_for_stmt (ord_stmt);
8192 location_t loc = gimple_location (ord_stmt);
8194 for (c = gimple_omp_ordered_clauses (ord_stmt);
8195 c; c = OMP_CLAUSE_CHAIN (c))
8196 if (OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_SOURCE)
8199 expand_omp_ordered_source (&gsi, fd, counts, loc);
8200 for (c = gimple_omp_ordered_clauses (ord_stmt);
8201 c; c = OMP_CLAUSE_CHAIN (c))
8202 if (OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_SINK)
8203 expand_omp_ordered_sink (&gsi, fd, counts, c, loc);
8204 gsi_remove (&gsi, true);
8208 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
8212 expand_omp_for_ordered_loops (struct omp_for_data *fd, tree *counts,
8213 basic_block cont_bb, basic_block body_bb,
8214 bool ordered_lastprivate)
8216 if (fd->ordered == fd->collapse)
8221 gimple_stmt_iterator gsi = gsi_after_labels (body_bb);
8222 for (int i = fd->collapse; i < fd->ordered; i++)
8224 tree type = TREE_TYPE (fd->loops[i].v);
8225 tree n1 = fold_convert (type, fd->loops[i].n1);
8226 expand_omp_build_assign (&gsi, fd->loops[i].v, n1);
8227 tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
8228 size_int (i - fd->collapse + 1),
8229 NULL_TREE, NULL_TREE);
8230 expand_omp_build_assign (&gsi, aref, build_zero_cst (fd->iter_type));
8235 for (int i = fd->ordered - 1; i >= fd->collapse; i--)
8237 tree t, type = TREE_TYPE (fd->loops[i].v);
8238 gimple_stmt_iterator gsi = gsi_after_labels (body_bb);
8239 expand_omp_build_assign (&gsi, fd->loops[i].v,
8240 fold_convert (type, fd->loops[i].n1));
8242 expand_omp_build_assign (&gsi, counts[i],
8243 build_zero_cst (fd->iter_type));
8244 tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
8245 size_int (i - fd->collapse + 1),
8246 NULL_TREE, NULL_TREE);
8247 expand_omp_build_assign (&gsi, aref, build_zero_cst (fd->iter_type));
8248 if (!gsi_end_p (gsi))
8251 gsi = gsi_last_bb (body_bb);
8252 edge e1 = split_block (body_bb, gsi_stmt (gsi));
8253 basic_block new_body = e1->dest;
8254 if (body_bb == cont_bb)
8257 basic_block new_header;
8258 if (EDGE_COUNT (cont_bb->preds) > 0)
8260 gsi = gsi_last_bb (cont_bb);
8261 if (POINTER_TYPE_P (type))
8262 t = fold_build_pointer_plus (fd->loops[i].v,
8263 fold_convert (sizetype,
8264 fd->loops[i].step));
8266 t = fold_build2 (PLUS_EXPR, type, fd->loops[i].v,
8267 fold_convert (type, fd->loops[i].step));
8268 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
8271 t = fold_build2 (PLUS_EXPR, fd->iter_type, counts[i],
8272 build_int_cst (fd->iter_type, 1));
8273 expand_omp_build_assign (&gsi, counts[i], t);
8278 t = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[i].v),
8279 fd->loops[i].v, fd->loops[i].n1);
8280 t = fold_convert (fd->iter_type, t);
8281 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
8282 true, GSI_SAME_STMT);
8284 aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
8285 size_int (i - fd->collapse + 1),
8286 NULL_TREE, NULL_TREE);
8287 expand_omp_build_assign (&gsi, aref, t);
8289 e2 = split_block (cont_bb, gsi_stmt (gsi));
8290 new_header = e2->dest;
8293 new_header = cont_bb;
8294 gsi = gsi_after_labels (new_header);
8295 tree v = force_gimple_operand_gsi (&gsi, fd->loops[i].v, true, NULL_TREE,
8296 true, GSI_SAME_STMT);
8298 = force_gimple_operand_gsi (&gsi, fold_convert (type, fd->loops[i].n2),
8299 true, NULL_TREE, true, GSI_SAME_STMT);
8300 t = build2 (fd->loops[i].cond_code, boolean_type_node, v, n2);
8301 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_NEW_STMT);
8302 edge e3 = split_block (new_header, gsi_stmt (gsi));
8305 make_edge (body_bb, new_header, EDGE_FALLTHRU);
8306 e3->flags = EDGE_FALSE_VALUE;
8307 e3->probability = REG_BR_PROB_BASE / 8;
8308 e1 = make_edge (new_header, new_body, EDGE_TRUE_VALUE);
8309 e1->probability = REG_BR_PROB_BASE - e3->probability;
8311 set_immediate_dominator (CDI_DOMINATORS, new_header, body_bb);
8312 set_immediate_dominator (CDI_DOMINATORS, new_body, new_header);
8316 struct loop *loop = alloc_loop ();
8317 loop->header = new_header;
8318 loop->latch = e2->src;
8319 add_loop (loop, body_bb->loop_father);
8323 /* If there are any lastprivate clauses and it is possible some loops
8324 might have zero iterations, ensure all the decls are initialized,
8325 otherwise we could crash evaluating C++ class iterators with lastprivate
8327 bool need_inits = false;
8328 for (int i = fd->collapse; ordered_lastprivate && i < fd->ordered; i++)
8331 tree type = TREE_TYPE (fd->loops[i].v);
8332 gimple_stmt_iterator gsi = gsi_after_labels (body_bb);
8333 expand_omp_build_assign (&gsi, fd->loops[i].v,
8334 fold_convert (type, fd->loops[i].n1));
8338 tree type = TREE_TYPE (fd->loops[i].v);
8339 tree this_cond = fold_build2 (fd->loops[i].cond_code,
8341 fold_convert (type, fd->loops[i].n1),
8342 fold_convert (type, fd->loops[i].n2));
8343 if (!integer_onep (this_cond))
8351 /* A subroutine of expand_omp_for. Generate code for a parallel
8352 loop with any schedule. Given parameters:
8354 for (V = N1; V cond N2; V += STEP) BODY;
8356 where COND is "<" or ">", we generate pseudocode
8358 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
8359 if (more) goto L0; else goto L3;
8366 if (V cond iend) goto L1; else goto L2;
8368 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8371 If this is a combined omp parallel loop, instead of the call to
8372 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
8373 If this is gimple_omp_for_combined_p loop, then instead of assigning
8374 V and iend in L0 we assign the first two _looptemp_ clause decls of the
8375 inner GIMPLE_OMP_FOR and V += STEP; and
8376 if (V cond iend) goto L1; else goto L2; are removed.
8378 For collapsed loops, given parameters:
8380 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8381 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8382 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8385 we generate pseudocode
8387 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
8392 count3 = (adj + N32 - N31) / STEP3;
8393 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
8398 count2 = (adj + N22 - N21) / STEP2;
8399 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
8404 count1 = (adj + N12 - N11) / STEP1;
8405 count = count1 * count2 * count3;
8410 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
8411 if (more) goto L0; else goto L3;
8415 V3 = N31 + (T % count3) * STEP3;
8417 V2 = N21 + (T % count2) * STEP2;
8419 V1 = N11 + T * STEP1;
8424 if (V < iend) goto L10; else goto L2;
8427 if (V3 cond3 N32) goto L1; else goto L11;
8431 if (V2 cond2 N22) goto L1; else goto L12;
8437 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8443 expand_omp_for_generic (struct omp_region *region,
8444 struct omp_for_data *fd,
8445 enum built_in_function start_fn,
8446 enum built_in_function next_fn,
8449 tree type, istart0, iend0, iend;
8450 tree t, vmain, vback, bias = NULL_TREE;
8451 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
8452 basic_block l2_bb = NULL, l3_bb = NULL;
8453 gimple_stmt_iterator gsi;
8454 gassign *assign_stmt;
8455 bool in_combined_parallel = is_combined_parallel (region);
8456 bool broken_loop = region->cont == NULL;
8458 tree *counts = NULL;
8460 bool ordered_lastprivate = false;
8462 gcc_assert (!broken_loop || !in_combined_parallel);
8463 gcc_assert (fd->iter_type == long_integer_type_node
8464 || !in_combined_parallel);
8466 entry_bb = region->entry;
8467 cont_bb = region->cont;
8469 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
8470 gcc_assert (broken_loop
8471 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
8472 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
8473 l1_bb = single_succ (l0_bb);
8476 l2_bb = create_empty_bb (cont_bb);
8477 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb
8478 || (single_succ_edge (BRANCH_EDGE (cont_bb)->dest)->dest
8480 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
8484 l3_bb = BRANCH_EDGE (entry_bb)->dest;
8485 exit_bb = region->exit;
8487 gsi = gsi_last_bb (entry_bb);
8489 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
8491 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi)),
8492 OMP_CLAUSE_LASTPRIVATE))
8493 ordered_lastprivate = false;
8494 if (fd->collapse > 1 || fd->ordered)
8496 int first_zero_iter1 = -1, first_zero_iter2 = -1;
8497 basic_block zero_iter1_bb = NULL, zero_iter2_bb = NULL, l2_dom_bb = NULL;
8499 counts = XALLOCAVEC (tree, fd->ordered ? fd->ordered + 1 : fd->collapse);
8500 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
8501 zero_iter1_bb, first_zero_iter1,
8502 zero_iter2_bb, first_zero_iter2, l2_dom_bb);
8506 /* Some counts[i] vars might be uninitialized if
8507 some loop has zero iterations. But the body shouldn't
8508 be executed in that case, so just avoid uninit warnings. */
8509 for (i = first_zero_iter1;
8510 i < (fd->ordered ? fd->ordered : fd->collapse); i++)
8511 if (SSA_VAR_P (counts[i]))
8512 TREE_NO_WARNING (counts[i]) = 1;
8514 e = split_block (entry_bb, gsi_stmt (gsi));
8516 make_edge (zero_iter1_bb, entry_bb, EDGE_FALLTHRU);
8517 gsi = gsi_last_bb (entry_bb);
8518 set_immediate_dominator (CDI_DOMINATORS, entry_bb,
8519 get_immediate_dominator (CDI_DOMINATORS,
8524 /* Some counts[i] vars might be uninitialized if
8525 some loop has zero iterations. But the body shouldn't
8526 be executed in that case, so just avoid uninit warnings. */
8527 for (i = first_zero_iter2; i < fd->ordered; i++)
8528 if (SSA_VAR_P (counts[i]))
8529 TREE_NO_WARNING (counts[i]) = 1;
8531 make_edge (zero_iter2_bb, entry_bb, EDGE_FALLTHRU);
8535 e = split_block (entry_bb, gsi_stmt (gsi));
8537 make_edge (zero_iter2_bb, entry_bb, EDGE_FALLTHRU);
8538 gsi = gsi_last_bb (entry_bb);
8539 set_immediate_dominator (CDI_DOMINATORS, entry_bb,
8540 get_immediate_dominator
8541 (CDI_DOMINATORS, zero_iter2_bb));
8544 if (fd->collapse == 1)
8546 counts[0] = fd->loop.n2;
8547 fd->loop = fd->loops[0];
8551 type = TREE_TYPE (fd->loop.v);
8552 istart0 = create_tmp_var (fd->iter_type, ".istart0");
8553 iend0 = create_tmp_var (fd->iter_type, ".iend0");
8554 TREE_ADDRESSABLE (istart0) = 1;
8555 TREE_ADDRESSABLE (iend0) = 1;
8557 /* See if we need to bias by LLONG_MIN. */
8558 if (fd->iter_type == long_long_unsigned_type_node
8559 && TREE_CODE (type) == INTEGER_TYPE
8560 && !TYPE_UNSIGNED (type)
8561 && fd->ordered == 0)
8565 if (fd->loop.cond_code == LT_EXPR)
8568 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
8572 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
8575 if (TREE_CODE (n1) != INTEGER_CST
8576 || TREE_CODE (n2) != INTEGER_CST
8577 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
8578 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
8581 gimple_stmt_iterator gsif = gsi;
8584 tree arr = NULL_TREE;
8585 if (in_combined_parallel)
8587 gcc_assert (fd->ordered == 0);
8588 /* In a combined parallel loop, emit a call to
8589 GOMP_loop_foo_next. */
8590 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
8591 build_fold_addr_expr (istart0),
8592 build_fold_addr_expr (iend0));
8596 tree t0, t1, t2, t3, t4;
8597 /* If this is not a combined parallel loop, emit a call to
8598 GOMP_loop_foo_start in ENTRY_BB. */
8599 t4 = build_fold_addr_expr (iend0);
8600 t3 = build_fold_addr_expr (istart0);
8603 t0 = build_int_cst (unsigned_type_node,
8604 fd->ordered - fd->collapse + 1);
8605 arr = create_tmp_var (build_array_type_nelts (fd->iter_type,
8607 - fd->collapse + 1),
8609 DECL_NAMELESS (arr) = 1;
8610 TREE_ADDRESSABLE (arr) = 1;
8611 TREE_STATIC (arr) = 1;
8612 vec<constructor_elt, va_gc> *v;
8613 vec_alloc (v, fd->ordered - fd->collapse + 1);
8616 for (idx = 0; idx < fd->ordered - fd->collapse + 1; idx++)
8619 if (idx == 0 && fd->collapse > 1)
8622 c = counts[idx + fd->collapse - 1];
8623 tree purpose = size_int (idx);
8624 CONSTRUCTOR_APPEND_ELT (v, purpose, c);
8625 if (TREE_CODE (c) != INTEGER_CST)
8626 TREE_STATIC (arr) = 0;
8629 DECL_INITIAL (arr) = build_constructor (TREE_TYPE (arr), v);
8630 if (!TREE_STATIC (arr))
8631 force_gimple_operand_gsi (&gsi, build1 (DECL_EXPR,
8632 void_type_node, arr),
8633 true, NULL_TREE, true, GSI_SAME_STMT);
8634 t1 = build_fold_addr_expr (arr);
8639 t2 = fold_convert (fd->iter_type, fd->loop.step);
8642 if (gimple_omp_for_combined_into_p (fd->for_stmt))
8645 = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
8646 OMP_CLAUSE__LOOPTEMP_);
8647 gcc_assert (innerc);
8648 t0 = OMP_CLAUSE_DECL (innerc);
8649 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
8650 OMP_CLAUSE__LOOPTEMP_);
8651 gcc_assert (innerc);
8652 t1 = OMP_CLAUSE_DECL (innerc);
8654 if (POINTER_TYPE_P (TREE_TYPE (t0))
8655 && TYPE_PRECISION (TREE_TYPE (t0))
8656 != TYPE_PRECISION (fd->iter_type))
8658 /* Avoid casting pointers to integer of a different size. */
8659 tree itype = signed_type_for (type);
8660 t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
8661 t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
8665 t1 = fold_convert (fd->iter_type, t1);
8666 t0 = fold_convert (fd->iter_type, t0);
8670 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
8671 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
8674 if (fd->iter_type == long_integer_type_node || fd->ordered)
8678 t = fold_convert (fd->iter_type, fd->chunk_size);
8679 t = omp_adjust_chunk_size (t, fd->simd_schedule);
8681 t = build_call_expr (builtin_decl_explicit (start_fn),
8682 5, t0, t1, t, t3, t4);
8684 t = build_call_expr (builtin_decl_explicit (start_fn),
8685 6, t0, t1, t2, t, t3, t4);
8687 else if (fd->ordered)
8688 t = build_call_expr (builtin_decl_explicit (start_fn),
8691 t = build_call_expr (builtin_decl_explicit (start_fn),
8692 5, t0, t1, t2, t3, t4);
8700 /* The GOMP_loop_ull_*start functions have additional boolean
8701 argument, true for < loops and false for > loops.
8702 In Fortran, the C bool type can be different from
8703 boolean_type_node. */
8704 bfn_decl = builtin_decl_explicit (start_fn);
8705 c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl));
8706 t5 = build_int_cst (c_bool_type,
8707 fd->loop.cond_code == LT_EXPR ? 1 : 0);
8710 tree bfn_decl = builtin_decl_explicit (start_fn);
8711 t = fold_convert (fd->iter_type, fd->chunk_size);
8712 t = omp_adjust_chunk_size (t, fd->simd_schedule);
8713 t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
8716 t = build_call_expr (builtin_decl_explicit (start_fn),
8717 6, t5, t0, t1, t2, t3, t4);
8720 if (TREE_TYPE (t) != boolean_type_node)
8721 t = fold_build2 (NE_EXPR, boolean_type_node,
8722 t, build_int_cst (TREE_TYPE (t), 0));
8723 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
8724 true, GSI_SAME_STMT);
8725 if (arr && !TREE_STATIC (arr))
8727 tree clobber = build_constructor (TREE_TYPE (arr), NULL);
8728 TREE_THIS_VOLATILE (clobber) = 1;
8729 gsi_insert_before (&gsi, gimple_build_assign (arr, clobber),
8732 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
8734 /* Remove the GIMPLE_OMP_FOR statement. */
8735 gsi_remove (&gsi, true);
8737 if (gsi_end_p (gsif))
8738 gsif = gsi_after_labels (gsi_bb (gsif));
8741 /* Iteration setup for sequential loop goes in L0_BB. */
8742 tree startvar = fd->loop.v;
8743 tree endvar = NULL_TREE;
8745 if (gimple_omp_for_combined_p (fd->for_stmt))
8747 gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR
8748 && gimple_omp_for_kind (inner_stmt)
8749 == GF_OMP_FOR_KIND_SIMD);
8750 tree innerc = find_omp_clause (gimple_omp_for_clauses (inner_stmt),
8751 OMP_CLAUSE__LOOPTEMP_);
8752 gcc_assert (innerc);
8753 startvar = OMP_CLAUSE_DECL (innerc);
8754 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
8755 OMP_CLAUSE__LOOPTEMP_);
8756 gcc_assert (innerc);
8757 endvar = OMP_CLAUSE_DECL (innerc);
8760 gsi = gsi_start_bb (l0_bb);
8762 if (fd->ordered && fd->collapse == 1)
8763 t = fold_build2 (MULT_EXPR, fd->iter_type, t,
8764 fold_convert (fd->iter_type, fd->loop.step));
8766 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
8767 if (fd->ordered && fd->collapse == 1)
8769 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
8770 t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (startvar),
8771 fd->loop.n1, fold_convert (sizetype, t));
8774 t = fold_convert (TREE_TYPE (startvar), t);
8775 t = fold_build2 (PLUS_EXPR, TREE_TYPE (startvar),
8781 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
8782 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
8783 t = fold_convert (TREE_TYPE (startvar), t);
8785 t = force_gimple_operand_gsi (&gsi, t,
8787 && TREE_ADDRESSABLE (startvar),
8788 NULL_TREE, false, GSI_CONTINUE_LINKING);
8789 assign_stmt = gimple_build_assign (startvar, t);
8790 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
8793 if (fd->ordered && fd->collapse == 1)
8794 t = fold_build2 (MULT_EXPR, fd->iter_type, t,
8795 fold_convert (fd->iter_type, fd->loop.step));
8797 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
8798 if (fd->ordered && fd->collapse == 1)
8800 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
8801 t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (startvar),
8802 fd->loop.n1, fold_convert (sizetype, t));
8805 t = fold_convert (TREE_TYPE (startvar), t);
8806 t = fold_build2 (PLUS_EXPR, TREE_TYPE (startvar),
8812 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
8813 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
8814 t = fold_convert (TREE_TYPE (startvar), t);
8816 iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
8817 false, GSI_CONTINUE_LINKING);
8820 assign_stmt = gimple_build_assign (endvar, iend);
8821 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
8822 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (iend)))
8823 assign_stmt = gimple_build_assign (fd->loop.v, iend);
8825 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, iend);
8826 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
8828 /* Handle linear clause adjustments. */
8829 tree itercnt = NULL_TREE;
8830 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_FOR)
8831 for (tree c = gimple_omp_for_clauses (fd->for_stmt);
8832 c; c = OMP_CLAUSE_CHAIN (c))
8833 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
8834 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
8836 tree d = OMP_CLAUSE_DECL (c);
8837 bool is_ref = is_reference (d);
8838 tree t = d, a, dest;
8840 t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
8841 tree type = TREE_TYPE (t);
8842 if (POINTER_TYPE_P (type))
8844 dest = unshare_expr (t);
8845 tree v = create_tmp_var (TREE_TYPE (t), NULL);
8846 expand_omp_build_assign (&gsif, v, t);
8847 if (itercnt == NULL_TREE)
8850 tree n1 = fd->loop.n1;
8851 if (POINTER_TYPE_P (TREE_TYPE (itercnt)))
8854 = fold_convert (signed_type_for (TREE_TYPE (itercnt)),
8856 n1 = fold_convert (TREE_TYPE (itercnt), n1);
8858 itercnt = fold_build2 (MINUS_EXPR, TREE_TYPE (itercnt),
8860 itercnt = fold_build2 (EXACT_DIV_EXPR, TREE_TYPE (itercnt),
8861 itercnt, fd->loop.step);
8862 itercnt = force_gimple_operand_gsi (&gsi, itercnt, true,
8864 GSI_CONTINUE_LINKING);
8866 a = fold_build2 (MULT_EXPR, type,
8867 fold_convert (type, itercnt),
8868 fold_convert (type, OMP_CLAUSE_LINEAR_STEP (c)));
8869 t = fold_build2 (type == TREE_TYPE (t) ? PLUS_EXPR
8870 : POINTER_PLUS_EXPR, TREE_TYPE (t), v, a);
8871 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
8872 false, GSI_CONTINUE_LINKING);
8873 assign_stmt = gimple_build_assign (dest, t);
8874 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
8876 if (fd->collapse > 1)
8877 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
8881 /* Until now, counts array contained number of iterations or
8882 variable containing it for ith loop. From now on, we need
8883 those counts only for collapsed loops, and only for the 2nd
8884 till the last collapsed one. Move those one element earlier,
8885 we'll use counts[fd->collapse - 1] for the first source/sink
8886 iteration counter and so on and counts[fd->ordered]
8887 as the array holding the current counter values for
8889 if (fd->collapse > 1)
8890 memmove (counts, counts + 1, (fd->collapse - 1) * sizeof (counts[0]));
8894 for (i = fd->collapse; i < fd->ordered; i++)
8896 tree type = TREE_TYPE (fd->loops[i].v);
8898 = fold_build2 (fd->loops[i].cond_code, boolean_type_node,
8899 fold_convert (type, fd->loops[i].n1),
8900 fold_convert (type, fd->loops[i].n2));
8901 if (!integer_onep (this_cond))
8904 if (i < fd->ordered)
8907 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
8908 add_bb_to_loop (cont_bb, l1_bb->loop_father);
8909 gimple_stmt_iterator gsi = gsi_after_labels (cont_bb);
8910 gimple *g = gimple_build_omp_continue (fd->loop.v, fd->loop.v);
8911 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
8912 make_edge (cont_bb, l3_bb, EDGE_FALLTHRU);
8913 make_edge (cont_bb, l1_bb, 0);
8914 l2_bb = create_empty_bb (cont_bb);
8915 broken_loop = false;
8918 expand_omp_ordered_source_sink (region, fd, counts, cont_bb);
8919 cont_bb = expand_omp_for_ordered_loops (fd, counts, cont_bb, l1_bb,
8920 ordered_lastprivate);
8921 if (counts[fd->collapse - 1])
8923 gcc_assert (fd->collapse == 1);
8924 gsi = gsi_last_bb (l0_bb);
8925 expand_omp_build_assign (&gsi, counts[fd->collapse - 1],
8927 gsi = gsi_last_bb (cont_bb);
8928 t = fold_build2 (PLUS_EXPR, fd->iter_type, counts[fd->collapse - 1],
8929 build_int_cst (fd->iter_type, 1));
8930 expand_omp_build_assign (&gsi, counts[fd->collapse - 1], t);
8931 tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
8932 size_zero_node, NULL_TREE, NULL_TREE);
8933 expand_omp_build_assign (&gsi, aref, counts[fd->collapse - 1]);
8934 t = counts[fd->collapse - 1];
8936 else if (fd->collapse > 1)
8940 t = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[0].v),
8941 fd->loops[0].v, fd->loops[0].n1);
8942 t = fold_convert (fd->iter_type, t);
8944 gsi = gsi_last_bb (l0_bb);
8945 tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
8946 size_zero_node, NULL_TREE, NULL_TREE);
8947 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
8948 false, GSI_CONTINUE_LINKING);
8949 expand_omp_build_assign (&gsi, aref, t, true);
8954 /* Code to control the increment and predicate for the sequential
8955 loop goes in the CONT_BB. */
8956 gsi = gsi_last_bb (cont_bb);
8957 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
8958 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
8959 vmain = gimple_omp_continue_control_use (cont_stmt);
8960 vback = gimple_omp_continue_control_def (cont_stmt);
8962 if (!gimple_omp_for_combined_p (fd->for_stmt))
8964 if (POINTER_TYPE_P (type))
8965 t = fold_build_pointer_plus (vmain, fd->loop.step);
8967 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
8968 t = force_gimple_operand_gsi (&gsi, t,
8970 && TREE_ADDRESSABLE (vback),
8971 NULL_TREE, true, GSI_SAME_STMT);
8972 assign_stmt = gimple_build_assign (vback, t);
8973 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
8975 if (fd->ordered && counts[fd->collapse - 1] == NULL_TREE)
8977 if (fd->collapse > 1)
8981 t = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[0].v),
8982 fd->loops[0].v, fd->loops[0].n1);
8983 t = fold_convert (fd->iter_type, t);
8985 tree aref = build4 (ARRAY_REF, fd->iter_type,
8986 counts[fd->ordered], size_zero_node,
8987 NULL_TREE, NULL_TREE);
8988 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
8989 true, GSI_SAME_STMT);
8990 expand_omp_build_assign (&gsi, aref, t);
8993 t = build2 (fd->loop.cond_code, boolean_type_node,
8994 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
8996 gcond *cond_stmt = gimple_build_cond_empty (t);
8997 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
9000 /* Remove GIMPLE_OMP_CONTINUE. */
9001 gsi_remove (&gsi, true);
9003 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
9004 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, l1_bb);
9006 /* Emit code to get the next parallel iteration in L2_BB. */
9007 gsi = gsi_start_bb (l2_bb);
9009 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
9010 build_fold_addr_expr (istart0),
9011 build_fold_addr_expr (iend0));
9012 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
9013 false, GSI_CONTINUE_LINKING);
9014 if (TREE_TYPE (t) != boolean_type_node)
9015 t = fold_build2 (NE_EXPR, boolean_type_node,
9016 t, build_int_cst (TREE_TYPE (t), 0));
9017 gcond *cond_stmt = gimple_build_cond_empty (t);
9018 gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
9021 /* Add the loop cleanup function. */
9022 gsi = gsi_last_bb (exit_bb);
9023 if (gimple_omp_return_nowait_p (gsi_stmt (gsi)))
9024 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
9025 else if (gimple_omp_return_lhs (gsi_stmt (gsi)))
9026 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
9028 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
9029 gcall *call_stmt = gimple_build_call (t, 0);
9030 if (gimple_omp_return_lhs (gsi_stmt (gsi)))
9031 gimple_call_set_lhs (call_stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
9032 gsi_insert_after (&gsi, call_stmt, GSI_SAME_STMT);
9035 tree arr = counts[fd->ordered];
9036 tree clobber = build_constructor (TREE_TYPE (arr), NULL);
9037 TREE_THIS_VOLATILE (clobber) = 1;
9038 gsi_insert_after (&gsi, gimple_build_assign (arr, clobber),
9041 gsi_remove (&gsi, true);
9043 /* Connect the new blocks. */
9044 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
9045 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
9051 e = find_edge (cont_bb, l3_bb);
9052 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
9054 phis = phi_nodes (l3_bb);
9055 for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi))
9057 gimple *phi = gsi_stmt (gsi);
9058 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
9059 PHI_ARG_DEF_FROM_EDGE (phi, e));
9063 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
9064 e = find_edge (cont_bb, l1_bb);
9067 e = BRANCH_EDGE (cont_bb);
9068 gcc_assert (single_succ (e->dest) == l1_bb);
9070 if (gimple_omp_for_combined_p (fd->for_stmt))
9075 else if (fd->collapse > 1)
9078 e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
9081 e->flags = EDGE_TRUE_VALUE;
9084 e->probability = REG_BR_PROB_BASE * 7 / 8;
9085 find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8;
9089 e = find_edge (cont_bb, l2_bb);
9090 e->flags = EDGE_FALLTHRU;
9092 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
9094 if (gimple_in_ssa_p (cfun))
9096 /* Add phis to the outer loop that connect to the phis in the inner,
9097 original loop, and move the loop entry value of the inner phi to
9098 the loop entry value of the outer phi. */
9100 for (psi = gsi_start_phis (l3_bb); !gsi_end_p (psi); gsi_next (&psi))
9102 source_location locus;
9104 gphi *exit_phi = psi.phi ();
9106 edge l2_to_l3 = find_edge (l2_bb, l3_bb);
9107 tree exit_res = PHI_ARG_DEF_FROM_EDGE (exit_phi, l2_to_l3);
9109 basic_block latch = BRANCH_EDGE (cont_bb)->dest;
9110 edge latch_to_l1 = find_edge (latch, l1_bb);
9112 = find_phi_with_arg_on_edge (exit_res, latch_to_l1);
9114 tree t = gimple_phi_result (exit_phi);
9115 tree new_res = copy_ssa_name (t, NULL);
9116 nphi = create_phi_node (new_res, l0_bb);
9118 edge l0_to_l1 = find_edge (l0_bb, l1_bb);
9119 t = PHI_ARG_DEF_FROM_EDGE (inner_phi, l0_to_l1);
9120 locus = gimple_phi_arg_location_from_edge (inner_phi, l0_to_l1);
9121 edge entry_to_l0 = find_edge (entry_bb, l0_bb);
9122 add_phi_arg (nphi, t, entry_to_l0, locus);
9124 edge l2_to_l0 = find_edge (l2_bb, l0_bb);
9125 add_phi_arg (nphi, exit_res, l2_to_l0, UNKNOWN_LOCATION);
9127 add_phi_arg (inner_phi, new_res, l0_to_l1, UNKNOWN_LOCATION);
9131 set_immediate_dominator (CDI_DOMINATORS, l2_bb,
9132 recompute_dominator (CDI_DOMINATORS, l2_bb));
9133 set_immediate_dominator (CDI_DOMINATORS, l3_bb,
9134 recompute_dominator (CDI_DOMINATORS, l3_bb));
9135 set_immediate_dominator (CDI_DOMINATORS, l0_bb,
9136 recompute_dominator (CDI_DOMINATORS, l0_bb));
9137 set_immediate_dominator (CDI_DOMINATORS, l1_bb,
9138 recompute_dominator (CDI_DOMINATORS, l1_bb));
9140 /* We enter expand_omp_for_generic with a loop. This original loop may
9141 have its own loop struct, or it may be part of an outer loop struct
9142 (which may be the fake loop). */
9143 struct loop *outer_loop = entry_bb->loop_father;
9144 bool orig_loop_has_loop_struct = l1_bb->loop_father != outer_loop;
9146 add_bb_to_loop (l2_bb, outer_loop);
9148 /* We've added a new loop around the original loop. Allocate the
9149 corresponding loop struct. */
9150 struct loop *new_loop = alloc_loop ();
9151 new_loop->header = l0_bb;
9152 new_loop->latch = l2_bb;
9153 add_loop (new_loop, outer_loop);
9155 /* Allocate a loop structure for the original loop unless we already
9157 if (!orig_loop_has_loop_struct
9158 && !gimple_omp_for_combined_p (fd->for_stmt))
9160 struct loop *orig_loop = alloc_loop ();
9161 orig_loop->header = l1_bb;
9162 /* The loop may have multiple latches. */
9163 add_loop (orig_loop, new_loop);
9169 /* A subroutine of expand_omp_for. Generate code for a parallel
9170 loop with static schedule and no specified chunk size. Given
9173 for (V = N1; V cond N2; V += STEP) BODY;
9175 where COND is "<" or ">", we generate pseudocode
9177 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9182 if ((__typeof (V)) -1 > 0 && cond is >)
9183 n = -(adj + N2 - N1) / -STEP;
9185 n = (adj + N2 - N1) / STEP;
9188 if (threadid < tt) goto L3; else goto L4;
9193 s0 = q * threadid + tt;
9196 if (s0 >= e0) goto L2; else goto L0;
9202 if (V cond e) goto L1;
9207 expand_omp_for_static_nochunk (struct omp_region *region,
9208 struct omp_for_data *fd,
9211 tree n, q, s0, e0, e, t, tt, nthreads, threadid;
9212 tree type, itype, vmain, vback;
9213 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
9214 basic_block body_bb, cont_bb, collapse_bb = NULL;
9216 gimple_stmt_iterator gsi;
9218 bool broken_loop = region->cont == NULL;
9219 tree *counts = NULL;
9222 itype = type = TREE_TYPE (fd->loop.v);
9223 if (POINTER_TYPE_P (type))
9224 itype = signed_type_for (type);
9226 entry_bb = region->entry;
9227 cont_bb = region->cont;
9228 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
9229 fin_bb = BRANCH_EDGE (entry_bb)->dest;
9230 gcc_assert (broken_loop
9231 || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
9232 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
9233 body_bb = single_succ (seq_start_bb);
9236 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb
9237 || single_succ (BRANCH_EDGE (cont_bb)->dest) == body_bb);
9238 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
9240 exit_bb = region->exit;
9242 /* Iteration space partitioning goes in ENTRY_BB. */
9243 gsi = gsi_last_bb (entry_bb);
9244 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
9246 if (fd->collapse > 1)
9248 int first_zero_iter = -1, dummy = -1;
9249 basic_block l2_dom_bb = NULL, dummy_bb = NULL;
9251 counts = XALLOCAVEC (tree, fd->collapse);
9252 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
9253 fin_bb, first_zero_iter,
9254 dummy_bb, dummy, l2_dom_bb);
9257 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
9258 t = integer_one_node;
9260 t = fold_binary (fd->loop.cond_code, boolean_type_node,
9261 fold_convert (type, fd->loop.n1),
9262 fold_convert (type, fd->loop.n2));
9263 if (fd->collapse == 1
9264 && TYPE_UNSIGNED (type)
9265 && (t == NULL_TREE || !integer_onep (t)))
9267 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
9268 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
9269 true, GSI_SAME_STMT);
9270 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
9271 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
9272 true, GSI_SAME_STMT);
9273 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
9274 NULL_TREE, NULL_TREE);
9275 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
9276 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
9277 expand_omp_regimplify_p, NULL, NULL)
9278 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
9279 expand_omp_regimplify_p, NULL, NULL))
9281 gsi = gsi_for_stmt (cond_stmt);
9282 gimple_regimplify_operands (cond_stmt, &gsi);
9284 ep = split_block (entry_bb, cond_stmt);
9285 ep->flags = EDGE_TRUE_VALUE;
9286 entry_bb = ep->dest;
9287 ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
9288 ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
9289 ep->probability = REG_BR_PROB_BASE / 2000 - 1;
9290 if (gimple_in_ssa_p (cfun))
9292 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
9293 for (gphi_iterator gpi = gsi_start_phis (fin_bb);
9294 !gsi_end_p (gpi); gsi_next (&gpi))
9296 gphi *phi = gpi.phi ();
9297 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
9298 ep, UNKNOWN_LOCATION);
9301 gsi = gsi_last_bb (entry_bb);
9304 switch (gimple_omp_for_kind (fd->for_stmt))
9306 case GF_OMP_FOR_KIND_FOR:
9307 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
9308 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
9310 case GF_OMP_FOR_KIND_DISTRIBUTE:
9311 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS);
9312 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
9317 nthreads = build_call_expr (nthreads, 0);
9318 nthreads = fold_convert (itype, nthreads);
9319 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE,
9320 true, GSI_SAME_STMT);
9321 threadid = build_call_expr (threadid, 0);
9322 threadid = fold_convert (itype, threadid);
9323 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
9324 true, GSI_SAME_STMT);
9328 step = fd->loop.step;
9329 if (gimple_omp_for_combined_into_p (fd->for_stmt))
9331 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
9332 OMP_CLAUSE__LOOPTEMP_);
9333 gcc_assert (innerc);
9334 n1 = OMP_CLAUSE_DECL (innerc);
9335 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
9336 OMP_CLAUSE__LOOPTEMP_);
9337 gcc_assert (innerc);
9338 n2 = OMP_CLAUSE_DECL (innerc);
9340 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
9341 true, NULL_TREE, true, GSI_SAME_STMT);
9342 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
9343 true, NULL_TREE, true, GSI_SAME_STMT);
9344 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
9345 true, NULL_TREE, true, GSI_SAME_STMT);
9347 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
9348 t = fold_build2 (PLUS_EXPR, itype, step, t);
9349 t = fold_build2 (PLUS_EXPR, itype, t, n2);
9350 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
9351 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
9352 t = fold_build2 (TRUNC_DIV_EXPR, itype,
9353 fold_build1 (NEGATE_EXPR, itype, t),
9354 fold_build1 (NEGATE_EXPR, itype, step));
9356 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
9357 t = fold_convert (itype, t);
9358 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
9360 q = create_tmp_reg (itype, "q");
9361 t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads);
9362 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
9363 gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT);
9365 tt = create_tmp_reg (itype, "tt");
9366 t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads);
9367 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
9368 gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT);
9370 t = build2 (LT_EXPR, boolean_type_node, threadid, tt);
9371 gcond *cond_stmt = gimple_build_cond_empty (t);
9372 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
9374 second_bb = split_block (entry_bb, cond_stmt)->dest;
9375 gsi = gsi_last_bb (second_bb);
9376 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
9378 gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)),
9380 gassign *assign_stmt
9381 = gimple_build_assign (q, PLUS_EXPR, q, build_int_cst (itype, 1));
9382 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
9384 third_bb = split_block (second_bb, assign_stmt)->dest;
9385 gsi = gsi_last_bb (third_bb);
9386 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
9388 t = build2 (MULT_EXPR, itype, q, threadid);
9389 t = build2 (PLUS_EXPR, itype, t, tt);
9390 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
9392 t = fold_build2 (PLUS_EXPR, itype, s0, q);
9393 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
9395 t = build2 (GE_EXPR, boolean_type_node, s0, e0);
9396 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
9398 /* Remove the GIMPLE_OMP_FOR statement. */
9399 gsi_remove (&gsi, true);
9401 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9402 gsi = gsi_start_bb (seq_start_bb);
9404 tree startvar = fd->loop.v;
9405 tree endvar = NULL_TREE;
9407 if (gimple_omp_for_combined_p (fd->for_stmt))
9409 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
9410 ? gimple_omp_parallel_clauses (inner_stmt)
9411 : gimple_omp_for_clauses (inner_stmt);
9412 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
9413 gcc_assert (innerc);
9414 startvar = OMP_CLAUSE_DECL (innerc);
9415 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
9416 OMP_CLAUSE__LOOPTEMP_);
9417 gcc_assert (innerc);
9418 endvar = OMP_CLAUSE_DECL (innerc);
9419 if (fd->collapse > 1 && TREE_CODE (fd->loop.n2) != INTEGER_CST
9420 && gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
9423 for (i = 1; i < fd->collapse; i++)
9425 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
9426 OMP_CLAUSE__LOOPTEMP_);
9427 gcc_assert (innerc);
9429 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
9430 OMP_CLAUSE__LOOPTEMP_);
9433 /* If needed (distribute parallel for with lastprivate),
9434 propagate down the total number of iterations. */
9435 tree t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc)),
9437 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, false,
9438 GSI_CONTINUE_LINKING);
9439 assign_stmt = gimple_build_assign (OMP_CLAUSE_DECL (innerc), t);
9440 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
9444 t = fold_convert (itype, s0);
9445 t = fold_build2 (MULT_EXPR, itype, t, step);
9446 if (POINTER_TYPE_P (type))
9447 t = fold_build_pointer_plus (n1, t);
9449 t = fold_build2 (PLUS_EXPR, type, t, n1);
9450 t = fold_convert (TREE_TYPE (startvar), t);
9451 t = force_gimple_operand_gsi (&gsi, t,
9453 && TREE_ADDRESSABLE (startvar),
9454 NULL_TREE, false, GSI_CONTINUE_LINKING);
9455 assign_stmt = gimple_build_assign (startvar, t);
9456 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
9458 t = fold_convert (itype, e0);
9459 t = fold_build2 (MULT_EXPR, itype, t, step);
9460 if (POINTER_TYPE_P (type))
9461 t = fold_build_pointer_plus (n1, t);
9463 t = fold_build2 (PLUS_EXPR, type, t, n1);
9464 t = fold_convert (TREE_TYPE (startvar), t);
9465 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
9466 false, GSI_CONTINUE_LINKING);
9469 assign_stmt = gimple_build_assign (endvar, e);
9470 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
9471 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
9472 assign_stmt = gimple_build_assign (fd->loop.v, e);
9474 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
9475 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
9477 /* Handle linear clause adjustments. */
9478 tree itercnt = NULL_TREE;
9479 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_FOR)
9480 for (tree c = gimple_omp_for_clauses (fd->for_stmt);
9481 c; c = OMP_CLAUSE_CHAIN (c))
9482 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
9483 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
9485 tree d = OMP_CLAUSE_DECL (c);
9486 bool is_ref = is_reference (d);
9487 tree t = d, a, dest;
9489 t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
9490 if (itercnt == NULL_TREE)
9492 if (gimple_omp_for_combined_into_p (fd->for_stmt))
9494 itercnt = fold_build2 (MINUS_EXPR, itype,
9495 fold_convert (itype, n1),
9496 fold_convert (itype, fd->loop.n1));
9497 itercnt = fold_build2 (EXACT_DIV_EXPR, itype, itercnt, step);
9498 itercnt = fold_build2 (PLUS_EXPR, itype, itercnt, s0);
9499 itercnt = force_gimple_operand_gsi (&gsi, itercnt, true,
9501 GSI_CONTINUE_LINKING);
9506 tree type = TREE_TYPE (t);
9507 if (POINTER_TYPE_P (type))
9509 a = fold_build2 (MULT_EXPR, type,
9510 fold_convert (type, itercnt),
9511 fold_convert (type, OMP_CLAUSE_LINEAR_STEP (c)));
9512 dest = unshare_expr (t);
9513 t = fold_build2 (type == TREE_TYPE (t) ? PLUS_EXPR
9514 : POINTER_PLUS_EXPR, TREE_TYPE (t), t, a);
9515 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
9516 false, GSI_CONTINUE_LINKING);
9517 assign_stmt = gimple_build_assign (dest, t);
9518 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
9520 if (fd->collapse > 1)
9521 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
9525 /* The code controlling the sequential loop replaces the
9526 GIMPLE_OMP_CONTINUE. */
9527 gsi = gsi_last_bb (cont_bb);
9528 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
9529 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
9530 vmain = gimple_omp_continue_control_use (cont_stmt);
9531 vback = gimple_omp_continue_control_def (cont_stmt);
9533 if (!gimple_omp_for_combined_p (fd->for_stmt))
9535 if (POINTER_TYPE_P (type))
9536 t = fold_build_pointer_plus (vmain, step);
9538 t = fold_build2 (PLUS_EXPR, type, vmain, step);
9539 t = force_gimple_operand_gsi (&gsi, t,
9541 && TREE_ADDRESSABLE (vback),
9542 NULL_TREE, true, GSI_SAME_STMT);
9543 assign_stmt = gimple_build_assign (vback, t);
9544 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
9546 t = build2 (fd->loop.cond_code, boolean_type_node,
9547 DECL_P (vback) && TREE_ADDRESSABLE (vback)
9549 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
9552 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9553 gsi_remove (&gsi, true);
9555 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
9556 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
9559 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9560 gsi = gsi_last_bb (exit_bb);
9561 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
9563 t = gimple_omp_return_lhs (gsi_stmt (gsi));
9564 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
9566 gsi_remove (&gsi, true);
9568 /* Connect all the blocks. */
9569 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
9570 ep->probability = REG_BR_PROB_BASE / 4 * 3;
9571 ep = find_edge (entry_bb, second_bb);
9572 ep->flags = EDGE_TRUE_VALUE;
9573 ep->probability = REG_BR_PROB_BASE / 4;
9574 find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
9575 find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
9579 ep = find_edge (cont_bb, body_bb);
9582 ep = BRANCH_EDGE (cont_bb);
9583 gcc_assert (single_succ (ep->dest) == body_bb);
9585 if (gimple_omp_for_combined_p (fd->for_stmt))
9590 else if (fd->collapse > 1)
9593 ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
9596 ep->flags = EDGE_TRUE_VALUE;
9597 find_edge (cont_bb, fin_bb)->flags
9598 = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
9601 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
9602 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
9603 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb);
9605 set_immediate_dominator (CDI_DOMINATORS, body_bb,
9606 recompute_dominator (CDI_DOMINATORS, body_bb));
9607 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
9608 recompute_dominator (CDI_DOMINATORS, fin_bb));
9610 struct loop *loop = body_bb->loop_father;
9611 if (loop != entry_bb->loop_father)
9613 gcc_assert (broken_loop || loop->header == body_bb);
9614 gcc_assert (broken_loop
9615 || loop->latch == region->cont
9616 || single_pred (loop->latch) == region->cont);
9620 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
9622 loop = alloc_loop ();
9623 loop->header = body_bb;
9624 if (collapse_bb == NULL)
9625 loop->latch = cont_bb;
9626 add_loop (loop, body_bb->loop_father);
9630 /* Return phi in E->DEST with ARG on edge E. */
9633 find_phi_with_arg_on_edge (tree arg, edge e)
9635 basic_block bb = e->dest;
9637 for (gphi_iterator gpi = gsi_start_phis (bb);
9641 gphi *phi = gpi.phi ();
9642 if (PHI_ARG_DEF_FROM_EDGE (phi, e) == arg)
9649 /* A subroutine of expand_omp_for. Generate code for a parallel
9650 loop with static schedule and a specified chunk size. Given
9653 for (V = N1; V cond N2; V += STEP) BODY;
9655 where COND is "<" or ">", we generate pseudocode
9657 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9662 if ((__typeof (V)) -1 > 0 && cond is >)
9663 n = -(adj + N2 - N1) / -STEP;
9665 n = (adj + N2 - N1) / STEP;
9667 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9668 here so that V is defined
9669 if the loop is not entered
9671 s0 = (trip * nthreads + threadid) * CHUNK;
9672 e0 = min(s0 + CHUNK, n);
9673 if (s0 < n) goto L1; else goto L4;
9680 if (V cond e) goto L2; else goto L3;
9688 expand_omp_for_static_chunk (struct omp_region *region,
9689 struct omp_for_data *fd, gimple *inner_stmt)
9691 tree n, s0, e0, e, t;
9692 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
9693 tree type, itype, vmain, vback, vextra;
9694 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
9695 basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb;
9696 gimple_stmt_iterator gsi;
9698 bool broken_loop = region->cont == NULL;
9699 tree *counts = NULL;
9702 itype = type = TREE_TYPE (fd->loop.v);
9703 if (POINTER_TYPE_P (type))
9704 itype = signed_type_for (type);
9706 entry_bb = region->entry;
9707 se = split_block (entry_bb, last_stmt (entry_bb));
9709 iter_part_bb = se->dest;
9710 cont_bb = region->cont;
9711 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
9712 fin_bb = BRANCH_EDGE (iter_part_bb)->dest;
9713 gcc_assert (broken_loop
9714 || fin_bb == FALLTHRU_EDGE (cont_bb)->dest);
9715 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
9716 body_bb = single_succ (seq_start_bb);
9719 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb
9720 || single_succ (BRANCH_EDGE (cont_bb)->dest) == body_bb);
9721 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
9722 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
9724 exit_bb = region->exit;
9726 /* Trip and adjustment setup goes in ENTRY_BB. */
9727 gsi = gsi_last_bb (entry_bb);
9728 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
9730 if (fd->collapse > 1)
9732 int first_zero_iter = -1, dummy = -1;
9733 basic_block l2_dom_bb = NULL, dummy_bb = NULL;
9735 counts = XALLOCAVEC (tree, fd->collapse);
9736 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
9737 fin_bb, first_zero_iter,
9738 dummy_bb, dummy, l2_dom_bb);
9741 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
9742 t = integer_one_node;
9744 t = fold_binary (fd->loop.cond_code, boolean_type_node,
9745 fold_convert (type, fd->loop.n1),
9746 fold_convert (type, fd->loop.n2));
9747 if (fd->collapse == 1
9748 && TYPE_UNSIGNED (type)
9749 && (t == NULL_TREE || !integer_onep (t)))
9751 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
9752 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
9753 true, GSI_SAME_STMT);
9754 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
9755 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
9756 true, GSI_SAME_STMT);
9757 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
9758 NULL_TREE, NULL_TREE);
9759 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
9760 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
9761 expand_omp_regimplify_p, NULL, NULL)
9762 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
9763 expand_omp_regimplify_p, NULL, NULL))
9765 gsi = gsi_for_stmt (cond_stmt);
9766 gimple_regimplify_operands (cond_stmt, &gsi);
9768 se = split_block (entry_bb, cond_stmt);
9769 se->flags = EDGE_TRUE_VALUE;
9770 entry_bb = se->dest;
9771 se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
9772 se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
9773 se->probability = REG_BR_PROB_BASE / 2000 - 1;
9774 if (gimple_in_ssa_p (cfun))
9776 int dest_idx = find_edge (iter_part_bb, fin_bb)->dest_idx;
9777 for (gphi_iterator gpi = gsi_start_phis (fin_bb);
9778 !gsi_end_p (gpi); gsi_next (&gpi))
9780 gphi *phi = gpi.phi ();
9781 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
9782 se, UNKNOWN_LOCATION);
9785 gsi = gsi_last_bb (entry_bb);
9788 switch (gimple_omp_for_kind (fd->for_stmt))
9790 case GF_OMP_FOR_KIND_FOR:
9791 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
9792 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
9794 case GF_OMP_FOR_KIND_DISTRIBUTE:
9795 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS);
9796 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
9801 nthreads = build_call_expr (nthreads, 0);
9802 nthreads = fold_convert (itype, nthreads);
9803 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE,
9804 true, GSI_SAME_STMT);
9805 threadid = build_call_expr (threadid, 0);
9806 threadid = fold_convert (itype, threadid);
9807 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
9808 true, GSI_SAME_STMT);
9812 step = fd->loop.step;
9813 if (gimple_omp_for_combined_into_p (fd->for_stmt))
9815 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
9816 OMP_CLAUSE__LOOPTEMP_);
9817 gcc_assert (innerc);
9818 n1 = OMP_CLAUSE_DECL (innerc);
9819 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
9820 OMP_CLAUSE__LOOPTEMP_);
9821 gcc_assert (innerc);
9822 n2 = OMP_CLAUSE_DECL (innerc);
9824 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
9825 true, NULL_TREE, true, GSI_SAME_STMT);
9826 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
9827 true, NULL_TREE, true, GSI_SAME_STMT);
9828 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
9829 true, NULL_TREE, true, GSI_SAME_STMT);
9830 tree chunk_size = fold_convert (itype, fd->chunk_size);
9831 chunk_size = omp_adjust_chunk_size (chunk_size, fd->simd_schedule);
9833 = force_gimple_operand_gsi (&gsi, chunk_size, true, NULL_TREE, true,
9836 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
9837 t = fold_build2 (PLUS_EXPR, itype, step, t);
9838 t = fold_build2 (PLUS_EXPR, itype, t, n2);
9839 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
9840 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
9841 t = fold_build2 (TRUNC_DIV_EXPR, itype,
9842 fold_build1 (NEGATE_EXPR, itype, t),
9843 fold_build1 (NEGATE_EXPR, itype, step));
9845 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
9846 t = fold_convert (itype, t);
9847 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
9848 true, GSI_SAME_STMT);
9850 trip_var = create_tmp_reg (itype, ".trip");
9851 if (gimple_in_ssa_p (cfun))
9853 trip_init = make_ssa_name (trip_var);
9854 trip_main = make_ssa_name (trip_var);
9855 trip_back = make_ssa_name (trip_var);
9859 trip_init = trip_var;
9860 trip_main = trip_var;
9861 trip_back = trip_var;
9864 gassign *assign_stmt
9865 = gimple_build_assign (trip_init, build_int_cst (itype, 0));
9866 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
9868 t = fold_build2 (MULT_EXPR, itype, threadid, chunk_size);
9869 t = fold_build2 (MULT_EXPR, itype, t, step);
9870 if (POINTER_TYPE_P (type))
9871 t = fold_build_pointer_plus (n1, t);
9873 t = fold_build2 (PLUS_EXPR, type, t, n1);
9874 vextra = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
9875 true, GSI_SAME_STMT);
9877 /* Remove the GIMPLE_OMP_FOR. */
9878 gsi_remove (&gsi, true);
9880 gimple_stmt_iterator gsif = gsi;
9882 /* Iteration space partitioning goes in ITER_PART_BB. */
9883 gsi = gsi_last_bb (iter_part_bb);
9885 t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads);
9886 t = fold_build2 (PLUS_EXPR, itype, t, threadid);
9887 t = fold_build2 (MULT_EXPR, itype, t, chunk_size);
9888 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
9889 false, GSI_CONTINUE_LINKING);
9891 t = fold_build2 (PLUS_EXPR, itype, s0, chunk_size);
9892 t = fold_build2 (MIN_EXPR, itype, t, n);
9893 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
9894 false, GSI_CONTINUE_LINKING);
9896 t = build2 (LT_EXPR, boolean_type_node, s0, n);
9897 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING);
9899 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9900 gsi = gsi_start_bb (seq_start_bb);
9902 tree startvar = fd->loop.v;
9903 tree endvar = NULL_TREE;
9905 if (gimple_omp_for_combined_p (fd->for_stmt))
9907 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
9908 ? gimple_omp_parallel_clauses (inner_stmt)
9909 : gimple_omp_for_clauses (inner_stmt);
9910 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
9911 gcc_assert (innerc);
9912 startvar = OMP_CLAUSE_DECL (innerc);
9913 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
9914 OMP_CLAUSE__LOOPTEMP_);
9915 gcc_assert (innerc);
9916 endvar = OMP_CLAUSE_DECL (innerc);
9917 if (fd->collapse > 1 && TREE_CODE (fd->loop.n2) != INTEGER_CST
9918 && gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
9921 for (i = 1; i < fd->collapse; i++)
9923 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
9924 OMP_CLAUSE__LOOPTEMP_);
9925 gcc_assert (innerc);
9927 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
9928 OMP_CLAUSE__LOOPTEMP_);
9931 /* If needed (distribute parallel for with lastprivate),
9932 propagate down the total number of iterations. */
9933 tree t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc)),
9935 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, false,
9936 GSI_CONTINUE_LINKING);
9937 assign_stmt = gimple_build_assign (OMP_CLAUSE_DECL (innerc), t);
9938 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
9943 t = fold_convert (itype, s0);
9944 t = fold_build2 (MULT_EXPR, itype, t, step);
9945 if (POINTER_TYPE_P (type))
9946 t = fold_build_pointer_plus (n1, t);
9948 t = fold_build2 (PLUS_EXPR, type, t, n1);
9949 t = fold_convert (TREE_TYPE (startvar), t);
9950 t = force_gimple_operand_gsi (&gsi, t,
9952 && TREE_ADDRESSABLE (startvar),
9953 NULL_TREE, false, GSI_CONTINUE_LINKING);
9954 assign_stmt = gimple_build_assign (startvar, t);
9955 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
9957 t = fold_convert (itype, e0);
9958 t = fold_build2 (MULT_EXPR, itype, t, step);
9959 if (POINTER_TYPE_P (type))
9960 t = fold_build_pointer_plus (n1, t);
9962 t = fold_build2 (PLUS_EXPR, type, t, n1);
9963 t = fold_convert (TREE_TYPE (startvar), t);
9964 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
9965 false, GSI_CONTINUE_LINKING);
9968 assign_stmt = gimple_build_assign (endvar, e);
9969 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
9970 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
9971 assign_stmt = gimple_build_assign (fd->loop.v, e);
9973 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
9974 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
9976 /* Handle linear clause adjustments. */
9977 tree itercnt = NULL_TREE, itercntbias = NULL_TREE;
9978 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_FOR)
9979 for (tree c = gimple_omp_for_clauses (fd->for_stmt);
9980 c; c = OMP_CLAUSE_CHAIN (c))
9981 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
9982 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
9984 tree d = OMP_CLAUSE_DECL (c);
9985 bool is_ref = is_reference (d);
9986 tree t = d, a, dest;
9988 t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
9989 tree type = TREE_TYPE (t);
9990 if (POINTER_TYPE_P (type))
9992 dest = unshare_expr (t);
9993 tree v = create_tmp_var (TREE_TYPE (t), NULL);
9994 expand_omp_build_assign (&gsif, v, t);
9995 if (itercnt == NULL_TREE)
9997 if (gimple_omp_for_combined_into_p (fd->for_stmt))
10000 = fold_build2 (MINUS_EXPR, itype, fold_convert (itype, n1),
10001 fold_convert (itype, fd->loop.n1));
10002 itercntbias = fold_build2 (EXACT_DIV_EXPR, itype,
10003 itercntbias, step);
10005 = force_gimple_operand_gsi (&gsif, itercntbias, true,
10008 itercnt = fold_build2 (PLUS_EXPR, itype, itercntbias, s0);
10009 itercnt = force_gimple_operand_gsi (&gsi, itercnt, true,
10011 GSI_CONTINUE_LINKING);
10016 a = fold_build2 (MULT_EXPR, type,
10017 fold_convert (type, itercnt),
10018 fold_convert (type, OMP_CLAUSE_LINEAR_STEP (c)));
10019 t = fold_build2 (type == TREE_TYPE (t) ? PLUS_EXPR
10020 : POINTER_PLUS_EXPR, TREE_TYPE (t), v, a);
10021 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
10022 false, GSI_CONTINUE_LINKING);
10023 assign_stmt = gimple_build_assign (dest, t);
10024 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
10026 if (fd->collapse > 1)
10027 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
10031 /* The code controlling the sequential loop goes in CONT_BB,
10032 replacing the GIMPLE_OMP_CONTINUE. */
10033 gsi = gsi_last_bb (cont_bb);
10034 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
10035 vmain = gimple_omp_continue_control_use (cont_stmt);
10036 vback = gimple_omp_continue_control_def (cont_stmt);
10038 if (!gimple_omp_for_combined_p (fd->for_stmt))
10040 if (POINTER_TYPE_P (type))
10041 t = fold_build_pointer_plus (vmain, step);
10043 t = fold_build2 (PLUS_EXPR, type, vmain, step);
10044 if (DECL_P (vback) && TREE_ADDRESSABLE (vback))
10045 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
10046 true, GSI_SAME_STMT);
10047 assign_stmt = gimple_build_assign (vback, t);
10048 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
10050 if (tree_int_cst_equal (fd->chunk_size, integer_one_node))
10051 t = build2 (EQ_EXPR, boolean_type_node,
10052 build_int_cst (itype, 0),
10053 build_int_cst (itype, 1));
10055 t = build2 (fd->loop.cond_code, boolean_type_node,
10056 DECL_P (vback) && TREE_ADDRESSABLE (vback)
10058 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
10061 /* Remove GIMPLE_OMP_CONTINUE. */
10062 gsi_remove (&gsi, true);
10064 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
10065 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
10067 /* Trip update code goes into TRIP_UPDATE_BB. */
10068 gsi = gsi_start_bb (trip_update_bb);
10070 t = build_int_cst (itype, 1);
10071 t = build2 (PLUS_EXPR, itype, trip_main, t);
10072 assign_stmt = gimple_build_assign (trip_back, t);
10073 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
10076 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
10077 gsi = gsi_last_bb (exit_bb);
10078 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
10080 t = gimple_omp_return_lhs (gsi_stmt (gsi));
10081 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
10083 gsi_remove (&gsi, true);
10085 /* Connect the new blocks. */
10086 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
10087 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
10091 se = find_edge (cont_bb, body_bb);
10094 se = BRANCH_EDGE (cont_bb);
10095 gcc_assert (single_succ (se->dest) == body_bb);
10097 if (gimple_omp_for_combined_p (fd->for_stmt))
10102 else if (fd->collapse > 1)
10105 se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
10108 se->flags = EDGE_TRUE_VALUE;
10109 find_edge (cont_bb, trip_update_bb)->flags
10110 = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
10112 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb);
10115 if (gimple_in_ssa_p (cfun))
10123 gcc_assert (fd->collapse == 1 && !broken_loop);
10125 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
10126 remove arguments of the phi nodes in fin_bb. We need to create
10127 appropriate phi nodes in iter_part_bb instead. */
10128 se = find_edge (iter_part_bb, fin_bb);
10129 re = single_succ_edge (trip_update_bb);
10130 vec<edge_var_map> *head = redirect_edge_var_map_vector (re);
10131 ene = single_succ_edge (entry_bb);
10133 psi = gsi_start_phis (fin_bb);
10134 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
10135 gsi_next (&psi), ++i)
10138 source_location locus;
10141 t = gimple_phi_result (phi);
10142 gcc_assert (t == redirect_edge_var_map_result (vm));
10144 if (!single_pred_p (fin_bb))
10145 t = copy_ssa_name (t, phi);
10147 nphi = create_phi_node (t, iter_part_bb);
10149 t = PHI_ARG_DEF_FROM_EDGE (phi, se);
10150 locus = gimple_phi_arg_location_from_edge (phi, se);
10152 /* A special case -- fd->loop.v is not yet computed in
10153 iter_part_bb, we need to use vextra instead. */
10154 if (t == fd->loop.v)
10156 add_phi_arg (nphi, t, ene, locus);
10157 locus = redirect_edge_var_map_location (vm);
10158 tree back_arg = redirect_edge_var_map_def (vm);
10159 add_phi_arg (nphi, back_arg, re, locus);
10160 edge ce = find_edge (cont_bb, body_bb);
10163 ce = BRANCH_EDGE (cont_bb);
10164 gcc_assert (single_succ (ce->dest) == body_bb);
10165 ce = single_succ_edge (ce->dest);
10167 gphi *inner_loop_phi = find_phi_with_arg_on_edge (back_arg, ce);
10168 gcc_assert (inner_loop_phi != NULL);
10169 add_phi_arg (inner_loop_phi, gimple_phi_result (nphi),
10170 find_edge (seq_start_bb, body_bb), locus);
10172 if (!single_pred_p (fin_bb))
10173 add_phi_arg (phi, gimple_phi_result (nphi), se, locus);
10175 gcc_assert (gsi_end_p (psi) && (head == NULL || i == head->length ()));
10176 redirect_edge_var_map_clear (re);
10177 if (single_pred_p (fin_bb))
10180 psi = gsi_start_phis (fin_bb);
10181 if (gsi_end_p (psi))
10183 remove_phi_node (&psi, false);
10186 /* Make phi node for trip. */
10187 phi = create_phi_node (trip_main, iter_part_bb);
10188 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb),
10190 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb),
10195 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
10196 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
10197 recompute_dominator (CDI_DOMINATORS, iter_part_bb));
10198 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
10199 recompute_dominator (CDI_DOMINATORS, fin_bb));
10200 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
10201 recompute_dominator (CDI_DOMINATORS, seq_start_bb));
10202 set_immediate_dominator (CDI_DOMINATORS, body_bb,
10203 recompute_dominator (CDI_DOMINATORS, body_bb));
10207 struct loop *loop = body_bb->loop_father;
10208 struct loop *trip_loop = alloc_loop ();
10209 trip_loop->header = iter_part_bb;
10210 trip_loop->latch = trip_update_bb;
10211 add_loop (trip_loop, iter_part_bb->loop_father);
10213 if (loop != entry_bb->loop_father)
10215 gcc_assert (loop->header == body_bb);
10216 gcc_assert (loop->latch == region->cont
10217 || single_pred (loop->latch) == region->cont);
10218 trip_loop->inner = loop;
10222 if (!gimple_omp_for_combined_p (fd->for_stmt))
10224 loop = alloc_loop ();
10225 loop->header = body_bb;
10226 if (collapse_bb == NULL)
10227 loop->latch = cont_bb;
10228 add_loop (loop, trip_loop);
10233 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
10235 for (V = N1; V cond N2; V += STEP) BODY;
10237 where COND is "<" or ">" or "!=", we generate pseudocode
10239 for (ind_var = low; ind_var < high; ind_var++)
10241 V = n1 + (ind_var * STEP)
10246 In the above pseudocode, low and high are function parameters of the
10247 child function. In the function below, we are inserting a temp.
10248 variable that will be making a call to two OMP functions that will not be
10249 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
10250 with _Cilk_for). These functions are replaced with low and high
10251 by the function that handles taskreg. */
10255 expand_cilk_for (struct omp_region *region, struct omp_for_data *fd)
10257 bool broken_loop = region->cont == NULL;
10258 basic_block entry_bb = region->entry;
10259 basic_block cont_bb = region->cont;
10261 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
10262 gcc_assert (broken_loop
10263 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
10264 basic_block l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
10265 basic_block l1_bb, l2_bb;
10269 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
10270 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
10271 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
10272 l2_bb = BRANCH_EDGE (entry_bb)->dest;
10276 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
10277 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
10278 l2_bb = single_succ (l1_bb);
10280 basic_block exit_bb = region->exit;
10281 basic_block l2_dom_bb = NULL;
10283 gimple_stmt_iterator gsi = gsi_last_bb (entry_bb);
10285 /* Below statements until the "tree high_val = ..." are pseudo statements
10286 used to pass information to be used by expand_omp_taskreg.
10287 low_val and high_val will be replaced by the __low and __high
10288 parameter from the child function.
10290 The call_exprs part is a place-holder, it is mainly used
10291 to distinctly identify to the top-level part that this is
10292 where we should put low and high (reasoning given in header
10296 = gimple_omp_parallel_child_fn (
10297 as_a <gomp_parallel *> (last_stmt (region->outer->entry)));
10298 tree t, low_val = NULL_TREE, high_val = NULL_TREE;
10299 for (t = DECL_ARGUMENTS (child_fndecl); t; t = TREE_CHAIN (t))
10301 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__high"))
10303 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__low"))
10306 gcc_assert (low_val && high_val);
10308 tree type = TREE_TYPE (low_val);
10309 tree ind_var = create_tmp_reg (type, "__cilk_ind_var");
10310 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
10312 /* Not needed in SSA form right now. */
10313 gcc_assert (!gimple_in_ssa_p (cfun));
10314 if (l2_dom_bb == NULL)
10318 tree n2 = high_val;
10320 gimple *stmt = gimple_build_assign (ind_var, n1);
10322 /* Replace the GIMPLE_OMP_FOR statement. */
10323 gsi_replace (&gsi, stmt, true);
10327 /* Code to control the increment goes in the CONT_BB. */
10328 gsi = gsi_last_bb (cont_bb);
10329 stmt = gsi_stmt (gsi);
10330 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
10331 stmt = gimple_build_assign (ind_var, PLUS_EXPR, ind_var,
10332 build_one_cst (type));
10334 /* Replace GIMPLE_OMP_CONTINUE. */
10335 gsi_replace (&gsi, stmt, true);
10338 /* Emit the condition in L1_BB. */
10339 gsi = gsi_after_labels (l1_bb);
10340 t = fold_build2 (MULT_EXPR, TREE_TYPE (fd->loop.step),
10341 fold_convert (TREE_TYPE (fd->loop.step), ind_var),
10343 if (POINTER_TYPE_P (TREE_TYPE (fd->loop.n1)))
10344 t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (fd->loop.n1),
10345 fd->loop.n1, fold_convert (sizetype, t));
10347 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loop.n1),
10348 fd->loop.n1, fold_convert (TREE_TYPE (fd->loop.n1), t));
10349 t = fold_convert (TREE_TYPE (fd->loop.v), t);
10350 expand_omp_build_assign (&gsi, fd->loop.v, t);
10352 /* The condition is always '<' since the runtime will fill in the low
10353 and high values. */
10354 stmt = gimple_build_cond (LT_EXPR, ind_var, n2, NULL_TREE, NULL_TREE);
10355 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
10357 /* Remove GIMPLE_OMP_RETURN. */
10358 gsi = gsi_last_bb (exit_bb);
10359 gsi_remove (&gsi, true);
10361 /* Connect the new blocks. */
10362 remove_edge (FALLTHRU_EDGE (entry_bb));
10367 remove_edge (BRANCH_EDGE (entry_bb));
10368 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
10370 e = BRANCH_EDGE (l1_bb);
10371 ne = FALLTHRU_EDGE (l1_bb);
10372 e->flags = EDGE_TRUE_VALUE;
10376 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
10378 ne = single_succ_edge (l1_bb);
10379 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
10382 ne->flags = EDGE_FALSE_VALUE;
10383 e->probability = REG_BR_PROB_BASE * 7 / 8;
10384 ne->probability = REG_BR_PROB_BASE / 8;
10386 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
10387 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
10388 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
10392 struct loop *loop = alloc_loop ();
10393 loop->header = l1_bb;
10394 loop->latch = cont_bb;
10395 add_loop (loop, l1_bb->loop_father);
10396 loop->safelen = INT_MAX;
10399 /* Pick the correct library function based on the precision of the
10400 induction variable type. */
10401 tree lib_fun = NULL_TREE;
10402 if (TYPE_PRECISION (type) == 32)
10403 lib_fun = cilk_for_32_fndecl;
10404 else if (TYPE_PRECISION (type) == 64)
10405 lib_fun = cilk_for_64_fndecl;
10407 gcc_unreachable ();
10409 gcc_assert (fd->sched_kind == OMP_CLAUSE_SCHEDULE_CILKFOR);
10411 /* WS_ARGS contains the library function flavor to call:
10412 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
10413 user-defined grain value. If the user does not define one, then zero
10414 is passed in by the parser. */
10415 vec_alloc (region->ws_args, 2);
10416 region->ws_args->quick_push (lib_fun);
10417 region->ws_args->quick_push (fd->chunk_size);
10420 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
10421 loop. Given parameters:
10423 for (V = N1; V cond N2; V += STEP) BODY;
10425 where COND is "<" or ">", we generate pseudocode
10433 if (V cond N2) goto L0; else goto L2;
10436 For collapsed loops, given parameters:
10438 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
10439 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
10440 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
10443 we generate pseudocode
10449 count3 = (adj + N32 - N31) / STEP3;
10454 count2 = (adj + N22 - N21) / STEP2;
10459 count1 = (adj + N12 - N11) / STEP1;
10460 count = count1 * count2 * count3;
10470 V2 += (V3 cond3 N32) ? 0 : STEP2;
10471 V3 = (V3 cond3 N32) ? V3 : N31;
10472 V1 += (V2 cond2 N22) ? 0 : STEP1;
10473 V2 = (V2 cond2 N22) ? V2 : N21;
10475 if (V < count) goto L0; else goto L2;
10481 expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
10484 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb;
10485 gimple_stmt_iterator gsi;
10488 bool broken_loop = region->cont == NULL;
10490 tree *counts = NULL;
10492 tree safelen = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
10493 OMP_CLAUSE_SAFELEN);
10494 tree simduid = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
10495 OMP_CLAUSE__SIMDUID_);
10498 type = TREE_TYPE (fd->loop.v);
10499 entry_bb = region->entry;
10500 cont_bb = region->cont;
10501 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
10502 gcc_assert (broken_loop
10503 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
10504 l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
10507 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
10508 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
10509 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
10510 l2_bb = BRANCH_EDGE (entry_bb)->dest;
10514 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
10515 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
10516 l2_bb = single_succ (l1_bb);
10518 exit_bb = region->exit;
10521 gsi = gsi_last_bb (entry_bb);
10523 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
10524 /* Not needed in SSA form right now. */
10525 gcc_assert (!gimple_in_ssa_p (cfun));
10526 if (fd->collapse > 1)
10528 int first_zero_iter = -1, dummy = -1;
10529 basic_block zero_iter_bb = l2_bb, dummy_bb = NULL;
10531 counts = XALLOCAVEC (tree, fd->collapse);
10532 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
10533 zero_iter_bb, first_zero_iter,
10534 dummy_bb, dummy, l2_dom_bb);
10536 if (l2_dom_bb == NULL)
10541 if (gimple_omp_for_combined_into_p (fd->for_stmt))
10543 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
10544 OMP_CLAUSE__LOOPTEMP_);
10545 gcc_assert (innerc);
10546 n1 = OMP_CLAUSE_DECL (innerc);
10547 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
10548 OMP_CLAUSE__LOOPTEMP_);
10549 gcc_assert (innerc);
10550 n2 = OMP_CLAUSE_DECL (innerc);
10551 expand_omp_build_assign (&gsi, fd->loop.v,
10552 fold_convert (type, n1));
10553 if (fd->collapse > 1)
10556 expand_omp_for_init_vars (fd, &gsi, counts, NULL, n1);
10562 expand_omp_build_assign (&gsi, fd->loop.v,
10563 fold_convert (type, fd->loop.n1));
10564 if (fd->collapse > 1)
10565 for (i = 0; i < fd->collapse; i++)
10567 tree itype = TREE_TYPE (fd->loops[i].v);
10568 if (POINTER_TYPE_P (itype))
10569 itype = signed_type_for (itype);
10570 t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1);
10571 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
10575 /* Remove the GIMPLE_OMP_FOR statement. */
10576 gsi_remove (&gsi, true);
10580 /* Code to control the increment goes in the CONT_BB. */
10581 gsi = gsi_last_bb (cont_bb);
10582 stmt = gsi_stmt (gsi);
10583 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
10585 if (POINTER_TYPE_P (type))
10586 t = fold_build_pointer_plus (fd->loop.v, fd->loop.step);
10588 t = fold_build2 (PLUS_EXPR, type, fd->loop.v, fd->loop.step);
10589 expand_omp_build_assign (&gsi, fd->loop.v, t);
10591 if (fd->collapse > 1)
10593 i = fd->collapse - 1;
10594 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
10596 t = fold_convert (sizetype, fd->loops[i].step);
10597 t = fold_build_pointer_plus (fd->loops[i].v, t);
10601 t = fold_convert (TREE_TYPE (fd->loops[i].v),
10602 fd->loops[i].step);
10603 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
10604 fd->loops[i].v, t);
10606 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
10608 for (i = fd->collapse - 1; i > 0; i--)
10610 tree itype = TREE_TYPE (fd->loops[i].v);
10611 tree itype2 = TREE_TYPE (fd->loops[i - 1].v);
10612 if (POINTER_TYPE_P (itype2))
10613 itype2 = signed_type_for (itype2);
10614 t = build3 (COND_EXPR, itype2,
10615 build2 (fd->loops[i].cond_code, boolean_type_node,
10617 fold_convert (itype, fd->loops[i].n2)),
10618 build_int_cst (itype2, 0),
10619 fold_convert (itype2, fd->loops[i - 1].step));
10620 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i - 1].v)))
10621 t = fold_build_pointer_plus (fd->loops[i - 1].v, t);
10623 t = fold_build2 (PLUS_EXPR, itype2, fd->loops[i - 1].v, t);
10624 expand_omp_build_assign (&gsi, fd->loops[i - 1].v, t);
10626 t = build3 (COND_EXPR, itype,
10627 build2 (fd->loops[i].cond_code, boolean_type_node,
10629 fold_convert (itype, fd->loops[i].n2)),
10631 fold_convert (itype, fd->loops[i].n1));
10632 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
10636 /* Remove GIMPLE_OMP_CONTINUE. */
10637 gsi_remove (&gsi, true);
10640 /* Emit the condition in L1_BB. */
10641 gsi = gsi_start_bb (l1_bb);
10643 t = fold_convert (type, n2);
10644 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
10645 false, GSI_CONTINUE_LINKING);
10646 tree v = fd->loop.v;
10647 if (DECL_P (v) && TREE_ADDRESSABLE (v))
10648 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
10649 false, GSI_CONTINUE_LINKING);
10650 t = build2 (fd->loop.cond_code, boolean_type_node, v, t);
10651 cond_stmt = gimple_build_cond_empty (t);
10652 gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
10653 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p,
10655 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p,
10658 gsi = gsi_for_stmt (cond_stmt);
10659 gimple_regimplify_operands (cond_stmt, &gsi);
10662 /* Remove GIMPLE_OMP_RETURN. */
10663 gsi = gsi_last_bb (exit_bb);
10664 gsi_remove (&gsi, true);
10666 /* Connect the new blocks. */
10667 remove_edge (FALLTHRU_EDGE (entry_bb));
10671 remove_edge (BRANCH_EDGE (entry_bb));
10672 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
10674 e = BRANCH_EDGE (l1_bb);
10675 ne = FALLTHRU_EDGE (l1_bb);
10676 e->flags = EDGE_TRUE_VALUE;
10680 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
10682 ne = single_succ_edge (l1_bb);
10683 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
10686 ne->flags = EDGE_FALSE_VALUE;
10687 e->probability = REG_BR_PROB_BASE * 7 / 8;
10688 ne->probability = REG_BR_PROB_BASE / 8;
10690 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
10691 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
10692 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
10696 struct loop *loop = alloc_loop ();
10697 loop->header = l1_bb;
10698 loop->latch = cont_bb;
10699 add_loop (loop, l1_bb->loop_father);
10700 if (safelen == NULL_TREE)
10701 loop->safelen = INT_MAX;
10704 safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen);
10705 if (TREE_CODE (safelen) != INTEGER_CST)
10707 else if (!tree_fits_uhwi_p (safelen)
10708 || tree_to_uhwi (safelen) > INT_MAX)
10709 loop->safelen = INT_MAX;
10711 loop->safelen = tree_to_uhwi (safelen);
10712 if (loop->safelen == 1)
10717 loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
10718 cfun->has_simduid_loops = true;
10720 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10722 if ((flag_tree_loop_vectorize
10723 || (!global_options_set.x_flag_tree_loop_vectorize
10724 && !global_options_set.x_flag_tree_vectorize))
10725 && flag_tree_loop_optimize
10726 && loop->safelen > 1)
10728 loop->force_vectorize = true;
10729 cfun->has_force_vectorize_loops = true;
10733 cfun->has_simduid_loops = true;
10736 /* Taskloop construct is represented after gimplification with
10737 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10738 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10739 which should just compute all the needed loop temporaries
10740 for GIMPLE_OMP_TASK. */
10743 expand_omp_taskloop_for_outer (struct omp_region *region,
10744 struct omp_for_data *fd,
10745 gimple *inner_stmt)
10747 tree type, bias = NULL_TREE;
10748 basic_block entry_bb, cont_bb, exit_bb;
10749 gimple_stmt_iterator gsi;
10750 gassign *assign_stmt;
10751 tree *counts = NULL;
10754 gcc_assert (inner_stmt);
10755 gcc_assert (region->cont);
10756 gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_TASK
10757 && gimple_omp_task_taskloop_p (inner_stmt));
10758 type = TREE_TYPE (fd->loop.v);
10760 /* See if we need to bias by LLONG_MIN. */
10761 if (fd->iter_type == long_long_unsigned_type_node
10762 && TREE_CODE (type) == INTEGER_TYPE
10763 && !TYPE_UNSIGNED (type))
10767 if (fd->loop.cond_code == LT_EXPR)
10770 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
10774 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
10777 if (TREE_CODE (n1) != INTEGER_CST
10778 || TREE_CODE (n2) != INTEGER_CST
10779 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
10780 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
10783 entry_bb = region->entry;
10784 cont_bb = region->cont;
10785 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
10786 gcc_assert (BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
10787 exit_bb = region->exit;
10789 gsi = gsi_last_bb (entry_bb);
10790 gimple *for_stmt = gsi_stmt (gsi);
10791 gcc_assert (gimple_code (for_stmt) == GIMPLE_OMP_FOR);
10792 if (fd->collapse > 1)
10794 int first_zero_iter = -1, dummy = -1;
10795 basic_block zero_iter_bb = NULL, dummy_bb = NULL, l2_dom_bb = NULL;
10797 counts = XALLOCAVEC (tree, fd->collapse);
10798 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
10799 zero_iter_bb, first_zero_iter,
10800 dummy_bb, dummy, l2_dom_bb);
10804 /* Some counts[i] vars might be uninitialized if
10805 some loop has zero iterations. But the body shouldn't
10806 be executed in that case, so just avoid uninit warnings. */
10807 for (i = first_zero_iter; i < fd->collapse; i++)
10808 if (SSA_VAR_P (counts[i]))
10809 TREE_NO_WARNING (counts[i]) = 1;
10811 edge e = split_block (entry_bb, gsi_stmt (gsi));
10812 entry_bb = e->dest;
10813 make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU);
10814 gsi = gsi_last_bb (entry_bb);
10815 set_immediate_dominator (CDI_DOMINATORS, entry_bb,
10816 get_immediate_dominator (CDI_DOMINATORS,
10824 if (POINTER_TYPE_P (TREE_TYPE (t0))
10825 && TYPE_PRECISION (TREE_TYPE (t0))
10826 != TYPE_PRECISION (fd->iter_type))
10828 /* Avoid casting pointers to integer of a different size. */
10829 tree itype = signed_type_for (type);
10830 t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
10831 t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
10835 t1 = fold_convert (fd->iter_type, t1);
10836 t0 = fold_convert (fd->iter_type, t0);
10840 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
10841 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
10844 tree innerc = find_omp_clause (gimple_omp_task_clauses (inner_stmt),
10845 OMP_CLAUSE__LOOPTEMP_);
10846 gcc_assert (innerc);
10847 tree startvar = OMP_CLAUSE_DECL (innerc);
10848 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), OMP_CLAUSE__LOOPTEMP_);
10849 gcc_assert (innerc);
10850 tree endvar = OMP_CLAUSE_DECL (innerc);
10851 if (fd->collapse > 1 && TREE_CODE (fd->loop.n2) != INTEGER_CST)
10853 gcc_assert (innerc);
10854 for (i = 1; i < fd->collapse; i++)
10856 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
10857 OMP_CLAUSE__LOOPTEMP_);
10858 gcc_assert (innerc);
10860 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
10861 OMP_CLAUSE__LOOPTEMP_);
10864 /* If needed (inner taskloop has lastprivate clause), propagate
10865 down the total number of iterations. */
10866 tree t = force_gimple_operand_gsi (&gsi, fd->loop.n2, false,
10868 GSI_CONTINUE_LINKING);
10869 assign_stmt = gimple_build_assign (OMP_CLAUSE_DECL (innerc), t);
10870 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
10874 t0 = force_gimple_operand_gsi (&gsi, t0, false, NULL_TREE, false,
10875 GSI_CONTINUE_LINKING);
10876 assign_stmt = gimple_build_assign (startvar, t0);
10877 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
10879 t1 = force_gimple_operand_gsi (&gsi, t1, false, NULL_TREE, false,
10880 GSI_CONTINUE_LINKING);
10881 assign_stmt = gimple_build_assign (endvar, t1);
10882 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
10883 if (fd->collapse > 1)
10884 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
10886 /* Remove the GIMPLE_OMP_FOR statement. */
10887 gsi = gsi_for_stmt (for_stmt);
10888 gsi_remove (&gsi, true);
10890 gsi = gsi_last_bb (cont_bb);
10891 gsi_remove (&gsi, true);
10893 gsi = gsi_last_bb (exit_bb);
10894 gsi_remove (&gsi, true);
10896 FALLTHRU_EDGE (entry_bb)->probability = REG_BR_PROB_BASE;
10897 remove_edge (BRANCH_EDGE (entry_bb));
10898 FALLTHRU_EDGE (cont_bb)->probability = REG_BR_PROB_BASE;
10899 remove_edge (BRANCH_EDGE (cont_bb));
10900 set_immediate_dominator (CDI_DOMINATORS, exit_bb, cont_bb);
10901 set_immediate_dominator (CDI_DOMINATORS, region->entry,
10902 recompute_dominator (CDI_DOMINATORS, region->entry));
10905 /* Taskloop construct is represented after gimplification with
10906 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10907 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10908 GOMP_taskloop{,_ull} function arranges for each task to be given just
10909 a single range of iterations. */
10912 expand_omp_taskloop_for_inner (struct omp_region *region,
10913 struct omp_for_data *fd,
10914 gimple *inner_stmt)
10916 tree e, t, type, itype, vmain, vback, bias = NULL_TREE;
10917 basic_block entry_bb, exit_bb, body_bb, cont_bb, collapse_bb = NULL;
10918 basic_block fin_bb;
10919 gimple_stmt_iterator gsi;
10921 bool broken_loop = region->cont == NULL;
10922 tree *counts = NULL;
10925 itype = type = TREE_TYPE (fd->loop.v);
10926 if (POINTER_TYPE_P (type))
10927 itype = signed_type_for (type);
10929 /* See if we need to bias by LLONG_MIN. */
10930 if (fd->iter_type == long_long_unsigned_type_node
10931 && TREE_CODE (type) == INTEGER_TYPE
10932 && !TYPE_UNSIGNED (type))
10936 if (fd->loop.cond_code == LT_EXPR)
10939 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
10943 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
10946 if (TREE_CODE (n1) != INTEGER_CST
10947 || TREE_CODE (n2) != INTEGER_CST
10948 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
10949 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
10952 entry_bb = region->entry;
10953 cont_bb = region->cont;
10954 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
10955 fin_bb = BRANCH_EDGE (entry_bb)->dest;
10956 gcc_assert (broken_loop
10957 || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
10958 body_bb = FALLTHRU_EDGE (entry_bb)->dest;
10961 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
10962 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
10964 exit_bb = region->exit;
10966 /* Iteration space partitioning goes in ENTRY_BB. */
10967 gsi = gsi_last_bb (entry_bb);
10968 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
10970 if (fd->collapse > 1)
10972 int first_zero_iter = -1, dummy = -1;
10973 basic_block l2_dom_bb = NULL, dummy_bb = NULL;
10975 counts = XALLOCAVEC (tree, fd->collapse);
10976 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
10977 fin_bb, first_zero_iter,
10978 dummy_bb, dummy, l2_dom_bb);
10982 t = integer_one_node;
10984 step = fd->loop.step;
10985 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
10986 OMP_CLAUSE__LOOPTEMP_);
10987 gcc_assert (innerc);
10988 n1 = OMP_CLAUSE_DECL (innerc);
10989 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), OMP_CLAUSE__LOOPTEMP_);
10990 gcc_assert (innerc);
10991 n2 = OMP_CLAUSE_DECL (innerc);
10994 n1 = fold_build2 (PLUS_EXPR, fd->iter_type, n1, bias);
10995 n2 = fold_build2 (PLUS_EXPR, fd->iter_type, n2, bias);
10997 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
10998 true, NULL_TREE, true, GSI_SAME_STMT);
10999 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
11000 true, NULL_TREE, true, GSI_SAME_STMT);
11001 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
11002 true, NULL_TREE, true, GSI_SAME_STMT);
11004 tree startvar = fd->loop.v;
11005 tree endvar = NULL_TREE;
11007 if (gimple_omp_for_combined_p (fd->for_stmt))
11009 tree clauses = gimple_omp_for_clauses (inner_stmt);
11010 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
11011 gcc_assert (innerc);
11012 startvar = OMP_CLAUSE_DECL (innerc);
11013 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
11014 OMP_CLAUSE__LOOPTEMP_);
11015 gcc_assert (innerc);
11016 endvar = OMP_CLAUSE_DECL (innerc);
11018 t = fold_convert (TREE_TYPE (startvar), n1);
11019 t = force_gimple_operand_gsi (&gsi, t,
11021 && TREE_ADDRESSABLE (startvar),
11022 NULL_TREE, false, GSI_CONTINUE_LINKING);
11023 gimple *assign_stmt = gimple_build_assign (startvar, t);
11024 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
11026 t = fold_convert (TREE_TYPE (startvar), n2);
11027 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
11028 false, GSI_CONTINUE_LINKING);
11031 assign_stmt = gimple_build_assign (endvar, e);
11032 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
11033 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
11034 assign_stmt = gimple_build_assign (fd->loop.v, e);
11036 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
11037 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
11039 if (fd->collapse > 1)
11040 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
11044 /* The code controlling the sequential loop replaces the
11045 GIMPLE_OMP_CONTINUE. */
11046 gsi = gsi_last_bb (cont_bb);
11047 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
11048 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
11049 vmain = gimple_omp_continue_control_use (cont_stmt);
11050 vback = gimple_omp_continue_control_def (cont_stmt);
11052 if (!gimple_omp_for_combined_p (fd->for_stmt))
11054 if (POINTER_TYPE_P (type))
11055 t = fold_build_pointer_plus (vmain, step);
11057 t = fold_build2 (PLUS_EXPR, type, vmain, step);
11058 t = force_gimple_operand_gsi (&gsi, t,
11060 && TREE_ADDRESSABLE (vback),
11061 NULL_TREE, true, GSI_SAME_STMT);
11062 assign_stmt = gimple_build_assign (vback, t);
11063 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
11065 t = build2 (fd->loop.cond_code, boolean_type_node,
11066 DECL_P (vback) && TREE_ADDRESSABLE (vback)
11068 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
11071 /* Remove the GIMPLE_OMP_CONTINUE statement. */
11072 gsi_remove (&gsi, true);
11074 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
11075 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
11078 /* Remove the GIMPLE_OMP_FOR statement. */
11079 gsi = gsi_for_stmt (fd->for_stmt);
11080 gsi_remove (&gsi, true);
11082 /* Remove the GIMPLE_OMP_RETURN statement. */
11083 gsi = gsi_last_bb (exit_bb);
11084 gsi_remove (&gsi, true);
11086 FALLTHRU_EDGE (entry_bb)->probability = REG_BR_PROB_BASE;
11088 remove_edge (BRANCH_EDGE (entry_bb));
11091 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb));
11092 region->outer->cont = NULL;
11095 /* Connect all the blocks. */
11098 ep = find_edge (cont_bb, body_bb);
11099 if (gimple_omp_for_combined_p (fd->for_stmt))
11104 else if (fd->collapse > 1)
11107 ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
11110 ep->flags = EDGE_TRUE_VALUE;
11111 find_edge (cont_bb, fin_bb)->flags
11112 = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
11115 set_immediate_dominator (CDI_DOMINATORS, body_bb,
11116 recompute_dominator (CDI_DOMINATORS, body_bb));
11118 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
11119 recompute_dominator (CDI_DOMINATORS, fin_bb));
11121 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
11123 struct loop *loop = alloc_loop ();
11124 loop->header = body_bb;
11125 if (collapse_bb == NULL)
11126 loop->latch = cont_bb;
11127 add_loop (loop, body_bb->loop_father);
11131 /* A subroutine of expand_omp_for. Generate code for an OpenACC
11132 partitioned loop. The lowering here is abstracted, in that the
11133 loop parameters are passed through internal functions, which are
11134 further lowered by oacc_device_lower, once we get to the target
11135 compiler. The loop is of the form:
11137 for (V = B; V LTGT E; V += S) {BODY}
11139 where LTGT is < or >. We may have a specified chunking size, CHUNKING
11140 (constant 0 for no chunking) and we will have a GWV partitioning
11141 mask, specifying dimensions over which the loop is to be
11142 partitioned (see note below). We generate code that looks like:
11144 <entry_bb> [incoming FALL->body, BRANCH->exit]
11145 typedef signedintify (typeof (V)) T; // underlying signed integral type
11148 T DIR = LTGT == '<' ? +1 : -1;
11149 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
11150 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
11152 <head_bb> [created by splitting end of entry_bb]
11153 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
11154 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
11155 if (!(offset LTGT bound)) goto bottom_bb;
11157 <body_bb> [incoming]
11161 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
11163 if (offset LTGT bound) goto body_bb; [*]
11165 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
11167 if (chunk < chunk_max) goto head_bb;
11169 <exit_bb> [incoming]
11170 V = B + ((range -/+ 1) / S +/- 1) * S [*]
11172 [*] Needed if V live at end of loop
11174 Note: CHUNKING & GWV mask are specified explicitly here. This is a
11175 transition, and will be specified by a more general mechanism shortly.
11179 expand_oacc_for (struct omp_region *region, struct omp_for_data *fd)
11181 tree v = fd->loop.v;
11182 enum tree_code cond_code = fd->loop.cond_code;
11183 enum tree_code plus_code = PLUS_EXPR;
11185 tree chunk_size = integer_minus_one_node;
11186 tree gwv = integer_zero_node;
11187 tree iter_type = TREE_TYPE (v);
11188 tree diff_type = iter_type;
11189 tree plus_type = iter_type;
11190 struct oacc_collapse *counts = NULL;
11192 gcc_checking_assert (gimple_omp_for_kind (fd->for_stmt)
11193 == GF_OMP_FOR_KIND_OACC_LOOP);
11194 gcc_assert (!gimple_omp_for_combined_into_p (fd->for_stmt));
11195 gcc_assert (cond_code == LT_EXPR || cond_code == GT_EXPR);
11197 if (POINTER_TYPE_P (iter_type))
11199 plus_code = POINTER_PLUS_EXPR;
11200 plus_type = sizetype;
11202 if (POINTER_TYPE_P (diff_type) || TYPE_UNSIGNED (diff_type))
11203 diff_type = signed_type_for (diff_type);
11205 basic_block entry_bb = region->entry; /* BB ending in OMP_FOR */
11206 basic_block exit_bb = region->exit; /* BB ending in OMP_RETURN */
11207 basic_block cont_bb = region->cont; /* BB ending in OMP_CONTINUE */
11208 basic_block bottom_bb = NULL;
11210 /* entry_bb has two sucessors; the branch edge is to the exit
11211 block, fallthrough edge to body. */
11212 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2
11213 && BRANCH_EDGE (entry_bb)->dest == exit_bb);
11215 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
11216 body_bb, or to a block whose only successor is the body_bb. Its
11217 fallthrough successor is the final block (same as the branch
11218 successor of the entry_bb). */
11221 basic_block body_bb = FALLTHRU_EDGE (entry_bb)->dest;
11222 basic_block bed = BRANCH_EDGE (cont_bb)->dest;
11224 gcc_assert (FALLTHRU_EDGE (cont_bb)->dest == exit_bb);
11225 gcc_assert (bed == body_bb || single_succ_edge (bed)->dest == body_bb);
11228 gcc_assert (!gimple_in_ssa_p (cfun));
11230 /* The exit block only has entry_bb and cont_bb as predecessors. */
11231 gcc_assert (EDGE_COUNT (exit_bb->preds) == 1 + (cont_bb != NULL));
11234 tree chunk_max = NULL_TREE;
11235 tree bound, offset;
11236 tree step = create_tmp_var (diff_type, ".step");
11237 bool up = cond_code == LT_EXPR;
11238 tree dir = build_int_cst (diff_type, up ? +1 : -1);
11239 bool chunking = !gimple_in_ssa_p (cfun);;
11242 /* SSA instances. */
11243 tree offset_incr = NULL_TREE;
11244 tree offset_init = NULL_TREE;
11246 gimple_stmt_iterator gsi;
11252 edge split, be, fte;
11254 /* Split the end of entry_bb to create head_bb. */
11255 split = split_block (entry_bb, last_stmt (entry_bb));
11256 basic_block head_bb = split->dest;
11257 entry_bb = split->src;
11259 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
11260 gsi = gsi_last_bb (entry_bb);
11261 gomp_for *for_stmt = as_a <gomp_for *> (gsi_stmt (gsi));
11262 loc = gimple_location (for_stmt);
11264 if (gimple_in_ssa_p (cfun))
11266 offset_init = gimple_omp_for_index (for_stmt, 0);
11267 gcc_assert (integer_zerop (fd->loop.n1));
11268 /* The SSA parallelizer does gang parallelism. */
11269 gwv = build_int_cst (integer_type_node, GOMP_DIM_MASK (GOMP_DIM_GANG));
11272 if (fd->collapse > 1)
11274 counts = XALLOCAVEC (struct oacc_collapse, fd->collapse);
11275 tree total = expand_oacc_collapse_init (fd, &gsi, counts,
11276 TREE_TYPE (fd->loop.n2));
11278 if (SSA_VAR_P (fd->loop.n2))
11280 total = force_gimple_operand_gsi (&gsi, total, false, NULL_TREE,
11281 true, GSI_SAME_STMT);
11282 ass = gimple_build_assign (fd->loop.n2, total);
11283 gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
11288 tree b = fd->loop.n1;
11289 tree e = fd->loop.n2;
11290 tree s = fd->loop.step;
11292 b = force_gimple_operand_gsi (&gsi, b, true, NULL_TREE, true, GSI_SAME_STMT);
11293 e = force_gimple_operand_gsi (&gsi, e, true, NULL_TREE, true, GSI_SAME_STMT);
11295 /* Convert the step, avoiding possible unsigned->signed overflow. */
11296 negating = !up && TYPE_UNSIGNED (TREE_TYPE (s));
11298 s = fold_build1 (NEGATE_EXPR, TREE_TYPE (s), s);
11299 s = fold_convert (diff_type, s);
11301 s = fold_build1 (NEGATE_EXPR, diff_type, s);
11302 s = force_gimple_operand_gsi (&gsi, s, true, NULL_TREE, true, GSI_SAME_STMT);
11305 chunk_size = integer_zero_node;
11306 expr = fold_convert (diff_type, chunk_size);
11307 chunk_size = force_gimple_operand_gsi (&gsi, expr, true,
11308 NULL_TREE, true, GSI_SAME_STMT);
11309 /* Determine the range, avoiding possible unsigned->signed overflow. */
11310 negating = !up && TYPE_UNSIGNED (iter_type);
11311 expr = fold_build2 (MINUS_EXPR, plus_type,
11312 fold_convert (plus_type, negating ? b : e),
11313 fold_convert (plus_type, negating ? e : b));
11314 expr = fold_convert (diff_type, expr);
11316 expr = fold_build1 (NEGATE_EXPR, diff_type, expr);
11317 tree range = force_gimple_operand_gsi (&gsi, expr, true,
11318 NULL_TREE, true, GSI_SAME_STMT);
11320 chunk_no = build_int_cst (diff_type, 0);
11323 gcc_assert (!gimple_in_ssa_p (cfun));
11326 chunk_max = create_tmp_var (diff_type, ".chunk_max");
11327 chunk_no = create_tmp_var (diff_type, ".chunk_no");
11329 ass = gimple_build_assign (chunk_no, expr);
11330 gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
11332 call = gimple_build_call_internal (IFN_GOACC_LOOP, 6,
11333 build_int_cst (integer_type_node,
11334 IFN_GOACC_LOOP_CHUNKS),
11335 dir, range, s, chunk_size, gwv);
11336 gimple_call_set_lhs (call, chunk_max);
11337 gimple_set_location (call, loc);
11338 gsi_insert_before (&gsi, call, GSI_SAME_STMT);
11341 chunk_size = chunk_no;
11343 call = gimple_build_call_internal (IFN_GOACC_LOOP, 6,
11344 build_int_cst (integer_type_node,
11345 IFN_GOACC_LOOP_STEP),
11346 dir, range, s, chunk_size, gwv);
11347 gimple_call_set_lhs (call, step);
11348 gimple_set_location (call, loc);
11349 gsi_insert_before (&gsi, call, GSI_SAME_STMT);
11351 /* Remove the GIMPLE_OMP_FOR. */
11352 gsi_remove (&gsi, true);
11354 /* Fixup edges from head_bb */
11355 be = BRANCH_EDGE (head_bb);
11356 fte = FALLTHRU_EDGE (head_bb);
11357 be->flags |= EDGE_FALSE_VALUE;
11358 fte->flags ^= EDGE_FALLTHRU | EDGE_TRUE_VALUE;
11360 basic_block body_bb = fte->dest;
11362 if (gimple_in_ssa_p (cfun))
11364 gsi = gsi_last_bb (cont_bb);
11365 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
11367 offset = gimple_omp_continue_control_use (cont_stmt);
11368 offset_incr = gimple_omp_continue_control_def (cont_stmt);
11372 offset = create_tmp_var (diff_type, ".offset");
11373 offset_init = offset_incr = offset;
11375 bound = create_tmp_var (TREE_TYPE (offset), ".bound");
11377 /* Loop offset & bound go into head_bb. */
11378 gsi = gsi_start_bb (head_bb);
11380 call = gimple_build_call_internal (IFN_GOACC_LOOP, 7,
11381 build_int_cst (integer_type_node,
11382 IFN_GOACC_LOOP_OFFSET),
11384 chunk_size, gwv, chunk_no);
11385 gimple_call_set_lhs (call, offset_init);
11386 gimple_set_location (call, loc);
11387 gsi_insert_after (&gsi, call, GSI_CONTINUE_LINKING);
11389 call = gimple_build_call_internal (IFN_GOACC_LOOP, 7,
11390 build_int_cst (integer_type_node,
11391 IFN_GOACC_LOOP_BOUND),
11393 chunk_size, gwv, offset_init);
11394 gimple_call_set_lhs (call, bound);
11395 gimple_set_location (call, loc);
11396 gsi_insert_after (&gsi, call, GSI_CONTINUE_LINKING);
11398 expr = build2 (cond_code, boolean_type_node, offset_init, bound);
11399 gsi_insert_after (&gsi, gimple_build_cond_empty (expr),
11400 GSI_CONTINUE_LINKING);
11402 /* V assignment goes into body_bb. */
11403 if (!gimple_in_ssa_p (cfun))
11405 gsi = gsi_start_bb (body_bb);
11407 expr = build2 (plus_code, iter_type, b,
11408 fold_convert (plus_type, offset));
11409 expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
11410 true, GSI_SAME_STMT);
11411 ass = gimple_build_assign (v, expr);
11412 gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
11413 if (fd->collapse > 1)
11414 expand_oacc_collapse_vars (fd, &gsi, counts, v);
11417 /* Loop increment goes into cont_bb. If this is not a loop, we
11418 will have spawned threads as if it was, and each one will
11419 execute one iteration. The specification is not explicit about
11420 whether such constructs are ill-formed or not, and they can
11421 occur, especially when noreturn routines are involved. */
11424 gsi = gsi_last_bb (cont_bb);
11425 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
11426 loc = gimple_location (cont_stmt);
11428 /* Increment offset. */
11429 if (gimple_in_ssa_p (cfun))
11430 expr= build2 (plus_code, iter_type, offset,
11431 fold_convert (plus_type, step));
11433 expr = build2 (PLUS_EXPR, diff_type, offset, step);
11434 expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
11435 true, GSI_SAME_STMT);
11436 ass = gimple_build_assign (offset_incr, expr);
11437 gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
11438 expr = build2 (cond_code, boolean_type_node, offset_incr, bound);
11439 gsi_insert_before (&gsi, gimple_build_cond_empty (expr), GSI_SAME_STMT);
11441 /* Remove the GIMPLE_OMP_CONTINUE. */
11442 gsi_remove (&gsi, true);
11444 /* Fixup edges from cont_bb */
11445 be = BRANCH_EDGE (cont_bb);
11446 fte = FALLTHRU_EDGE (cont_bb);
11447 be->flags |= EDGE_TRUE_VALUE;
11448 fte->flags ^= EDGE_FALLTHRU | EDGE_FALSE_VALUE;
11452 /* Split the beginning of exit_bb to make bottom_bb. We
11453 need to insert a nop at the start, because splitting is
11454 after a stmt, not before. */
11455 gsi = gsi_start_bb (exit_bb);
11456 stmt = gimple_build_nop ();
11457 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
11458 split = split_block (exit_bb, stmt);
11459 bottom_bb = split->src;
11460 exit_bb = split->dest;
11461 gsi = gsi_last_bb (bottom_bb);
11463 /* Chunk increment and test goes into bottom_bb. */
11464 expr = build2 (PLUS_EXPR, diff_type, chunk_no,
11465 build_int_cst (diff_type, 1));
11466 ass = gimple_build_assign (chunk_no, expr);
11467 gsi_insert_after (&gsi, ass, GSI_CONTINUE_LINKING);
11469 /* Chunk test at end of bottom_bb. */
11470 expr = build2 (LT_EXPR, boolean_type_node, chunk_no, chunk_max);
11471 gsi_insert_after (&gsi, gimple_build_cond_empty (expr),
11472 GSI_CONTINUE_LINKING);
11474 /* Fixup edges from bottom_bb. */
11475 split->flags ^= EDGE_FALLTHRU | EDGE_FALSE_VALUE;
11476 make_edge (bottom_bb, head_bb, EDGE_TRUE_VALUE);
11480 gsi = gsi_last_bb (exit_bb);
11481 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
11482 loc = gimple_location (gsi_stmt (gsi));
11484 if (!gimple_in_ssa_p (cfun))
11486 /* Insert the final value of V, in case it is live. This is the
11487 value for the only thread that survives past the join. */
11488 expr = fold_build2 (MINUS_EXPR, diff_type, range, dir);
11489 expr = fold_build2 (PLUS_EXPR, diff_type, expr, s);
11490 expr = fold_build2 (TRUNC_DIV_EXPR, diff_type, expr, s);
11491 expr = fold_build2 (MULT_EXPR, diff_type, expr, s);
11492 expr = build2 (plus_code, iter_type, b, fold_convert (plus_type, expr));
11493 expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
11494 true, GSI_SAME_STMT);
11495 ass = gimple_build_assign (v, expr);
11496 gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
11499 /* Remove the OMP_RETURN. */
11500 gsi_remove (&gsi, true);
11504 /* We now have one or two nested loops. Update the loop
11506 struct loop *parent = entry_bb->loop_father;
11507 struct loop *body = body_bb->loop_father;
11511 struct loop *chunk_loop = alloc_loop ();
11512 chunk_loop->header = head_bb;
11513 chunk_loop->latch = bottom_bb;
11514 add_loop (chunk_loop, parent);
11515 parent = chunk_loop;
11517 else if (parent != body)
11519 gcc_assert (body->header == body_bb);
11520 gcc_assert (body->latch == cont_bb
11521 || single_pred (body->latch) == cont_bb);
11527 struct loop *body_loop = alloc_loop ();
11528 body_loop->header = body_bb;
11529 body_loop->latch = cont_bb;
11530 add_loop (body_loop, parent);
11535 /* Expand the OMP loop defined by REGION. */
11538 expand_omp_for (struct omp_region *region, gimple *inner_stmt)
11540 struct omp_for_data fd;
11541 struct omp_for_data_loop *loops;
11544 = (struct omp_for_data_loop *)
11545 alloca (gimple_omp_for_collapse (last_stmt (region->entry))
11546 * sizeof (struct omp_for_data_loop));
11547 extract_omp_for_data (as_a <gomp_for *> (last_stmt (region->entry)),
11549 region->sched_kind = fd.sched_kind;
11550 region->sched_modifiers = fd.sched_modifiers;
11552 gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
11553 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
11554 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
11557 gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
11558 BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
11559 FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
11562 /* If there isn't a continue then this is a degerate case where
11563 the introduction of abnormal edges during lowering will prevent
11564 original loops from being detected. Fix that up. */
11565 loops_state_set (LOOPS_NEED_FIXUP);
11567 if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD)
11568 expand_omp_simd (region, &fd);
11569 else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_CILKFOR)
11570 expand_cilk_for (region, &fd);
11571 else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
11573 gcc_assert (!inner_stmt);
11574 expand_oacc_for (region, &fd);
11576 else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_TASKLOOP)
11578 if (gimple_omp_for_combined_into_p (fd.for_stmt))
11579 expand_omp_taskloop_for_inner (region, &fd, inner_stmt);
11581 expand_omp_taskloop_for_outer (region, &fd, inner_stmt);
11583 else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
11584 && !fd.have_ordered)
11586 if (fd.chunk_size == NULL)
11587 expand_omp_for_static_nochunk (region, &fd, inner_stmt);
11589 expand_omp_for_static_chunk (region, &fd, inner_stmt);
11593 int fn_index, start_ix, next_ix;
11595 gcc_assert (gimple_omp_for_kind (fd.for_stmt)
11596 == GF_OMP_FOR_KIND_FOR);
11597 if (fd.chunk_size == NULL
11598 && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
11599 fd.chunk_size = integer_zero_node;
11600 gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
11601 switch (fd.sched_kind)
11603 case OMP_CLAUSE_SCHEDULE_RUNTIME:
11606 case OMP_CLAUSE_SCHEDULE_DYNAMIC:
11607 case OMP_CLAUSE_SCHEDULE_GUIDED:
11608 if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_NONMONOTONIC)
11610 && !fd.have_ordered)
11612 fn_index = 3 + fd.sched_kind;
11617 fn_index = fd.sched_kind;
11621 fn_index += fd.have_ordered * 6;
11623 start_ix = ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START) + fn_index;
11625 start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
11626 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
11627 if (fd.iter_type == long_long_unsigned_type_node)
11629 start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11630 - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
11631 next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11632 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
11634 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
11635 (enum built_in_function) next_ix, inner_stmt);
11638 if (gimple_in_ssa_p (cfun))
11639 update_ssa (TODO_update_ssa_only_virtuals);
11643 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11645 v = GOMP_sections_start (n);
11662 v = GOMP_sections_next ();
11667 If this is a combined parallel sections, replace the call to
11668 GOMP_sections_start with call to GOMP_sections_next. */
11671 expand_omp_sections (struct omp_region *region)
11673 tree t, u, vin = NULL, vmain, vnext, l2;
11675 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
11676 gimple_stmt_iterator si, switch_si;
11677 gomp_sections *sections_stmt;
11679 gomp_continue *cont;
11682 struct omp_region *inner;
11684 bool exit_reachable = region->cont != NULL;
11686 gcc_assert (region->exit != NULL);
11687 entry_bb = region->entry;
11688 l0_bb = single_succ (entry_bb);
11689 l1_bb = region->cont;
11690 l2_bb = region->exit;
11691 if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb)
11692 l2 = gimple_block_label (l2_bb);
11695 /* This can happen if there are reductions. */
11696 len = EDGE_COUNT (l0_bb->succs);
11697 gcc_assert (len > 0);
11698 e = EDGE_SUCC (l0_bb, len - 1);
11699 si = gsi_last_bb (e->dest);
11702 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
11703 l2 = gimple_block_label (e->dest);
11705 FOR_EACH_EDGE (e, ei, l0_bb->succs)
11707 si = gsi_last_bb (e->dest);
11709 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
11711 l2 = gimple_block_label (e->dest);
11716 if (exit_reachable)
11717 default_bb = create_empty_bb (l1_bb->prev_bb);
11719 default_bb = create_empty_bb (l0_bb);
11721 /* We will build a switch() with enough cases for all the
11722 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11723 and a default case to abort if something goes wrong. */
11724 len = EDGE_COUNT (l0_bb->succs);
11726 /* Use vec::quick_push on label_vec throughout, since we know the size
11728 auto_vec<tree> label_vec (len);
11730 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11731 GIMPLE_OMP_SECTIONS statement. */
11732 si = gsi_last_bb (entry_bb);
11733 sections_stmt = as_a <gomp_sections *> (gsi_stmt (si));
11734 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
11735 vin = gimple_omp_sections_control (sections_stmt);
11736 if (!is_combined_parallel (region))
11738 /* If we are not inside a combined parallel+sections region,
11739 call GOMP_sections_start. */
11740 t = build_int_cst (unsigned_type_node, len - 1);
11741 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
11742 stmt = gimple_build_call (u, 1, t);
11746 /* Otherwise, call GOMP_sections_next. */
11747 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
11748 stmt = gimple_build_call (u, 0);
11750 gimple_call_set_lhs (stmt, vin);
11751 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
11752 gsi_remove (&si, true);
11754 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11756 switch_si = gsi_last_bb (l0_bb);
11757 gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH);
11758 if (exit_reachable)
11760 cont = as_a <gomp_continue *> (last_stmt (l1_bb));
11761 gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE);
11762 vmain = gimple_omp_continue_control_use (cont);
11763 vnext = gimple_omp_continue_control_def (cont);
11771 t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2);
11772 label_vec.quick_push (t);
11775 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11776 for (inner = region->inner, casei = 1;
11778 inner = inner->next, i++, casei++)
11780 basic_block s_entry_bb, s_exit_bb;
11782 /* Skip optional reduction region. */
11783 if (inner->type == GIMPLE_OMP_ATOMIC_LOAD)
11790 s_entry_bb = inner->entry;
11791 s_exit_bb = inner->exit;
11793 t = gimple_block_label (s_entry_bb);
11794 u = build_int_cst (unsigned_type_node, casei);
11795 u = build_case_label (u, NULL, t);
11796 label_vec.quick_push (u);
11798 si = gsi_last_bb (s_entry_bb);
11799 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION);
11800 gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si)));
11801 gsi_remove (&si, true);
11802 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
11804 if (s_exit_bb == NULL)
11807 si = gsi_last_bb (s_exit_bb);
11808 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
11809 gsi_remove (&si, true);
11811 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
11814 /* Error handling code goes in DEFAULT_BB. */
11815 t = gimple_block_label (default_bb);
11816 u = build_case_label (NULL, NULL, t);
11817 make_edge (l0_bb, default_bb, 0);
11818 add_bb_to_loop (default_bb, current_loops->tree_root);
11820 stmt = gimple_build_switch (vmain, u, label_vec);
11821 gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT);
11822 gsi_remove (&switch_si, true);
11824 si = gsi_start_bb (default_bb);
11825 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
11826 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
11828 if (exit_reachable)
11832 /* Code to get the next section goes in L1_BB. */
11833 si = gsi_last_bb (l1_bb);
11834 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
11836 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
11837 stmt = gimple_build_call (bfn_decl, 0);
11838 gimple_call_set_lhs (stmt, vnext);
11839 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
11840 gsi_remove (&si, true);
11842 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
11845 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11846 si = gsi_last_bb (l2_bb);
11847 if (gimple_omp_return_nowait_p (gsi_stmt (si)))
11848 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT);
11849 else if (gimple_omp_return_lhs (gsi_stmt (si)))
11850 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL);
11852 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END);
11853 stmt = gimple_build_call (t, 0);
11854 if (gimple_omp_return_lhs (gsi_stmt (si)))
11855 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si)));
11856 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
11857 gsi_remove (&si, true);
11859 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
11863 /* Expand code for an OpenMP single directive. We've already expanded
11864 much of the code, here we simply place the GOMP_barrier call. */
11867 expand_omp_single (struct omp_region *region)
11869 basic_block entry_bb, exit_bb;
11870 gimple_stmt_iterator si;
11872 entry_bb = region->entry;
11873 exit_bb = region->exit;
11875 si = gsi_last_bb (entry_bb);
11876 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE);
11877 gsi_remove (&si, true);
11878 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
11880 si = gsi_last_bb (exit_bb);
11881 if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
11883 tree t = gimple_omp_return_lhs (gsi_stmt (si));
11884 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT);
11886 gsi_remove (&si, true);
11887 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
11891 /* Generic expansion for OpenMP synchronization directives: master,
11892 ordered and critical. All we need to do here is remove the entry
11893 and exit markers for REGION. */
11896 expand_omp_synch (struct omp_region *region)
11898 basic_block entry_bb, exit_bb;
11899 gimple_stmt_iterator si;
11901 entry_bb = region->entry;
11902 exit_bb = region->exit;
11904 si = gsi_last_bb (entry_bb);
11905 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
11906 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
11907 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
11908 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
11909 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
11910 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS);
11911 gsi_remove (&si, true);
11912 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
11916 si = gsi_last_bb (exit_bb);
11917 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
11918 gsi_remove (&si, true);
11919 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
11923 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11924 operation as a normal volatile load. */
11927 expand_omp_atomic_load (basic_block load_bb, tree addr,
11928 tree loaded_val, int index)
11930 enum built_in_function tmpbase;
11931 gimple_stmt_iterator gsi;
11932 basic_block store_bb;
11935 tree decl, call, type, itype;
11937 gsi = gsi_last_bb (load_bb);
11938 stmt = gsi_stmt (gsi);
11939 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
11940 loc = gimple_location (stmt);
11942 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11943 is smaller than word size, then expand_atomic_load assumes that the load
11944 is atomic. We could avoid the builtin entirely in this case. */
11946 tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
11947 decl = builtin_decl_explicit (tmpbase);
11948 if (decl == NULL_TREE)
11951 type = TREE_TYPE (loaded_val);
11952 itype = TREE_TYPE (TREE_TYPE (decl));
11954 call = build_call_expr_loc (loc, decl, 2, addr,
11955 build_int_cst (NULL,
11956 gimple_omp_atomic_seq_cst_p (stmt)
11958 : MEMMODEL_RELAXED));
11959 if (!useless_type_conversion_p (type, itype))
11960 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
11961 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
11963 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
11964 gsi_remove (&gsi, true);
11966 store_bb = single_succ (load_bb);
11967 gsi = gsi_last_bb (store_bb);
11968 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
11969 gsi_remove (&gsi, true);
11971 if (gimple_in_ssa_p (cfun))
11972 update_ssa (TODO_update_ssa_no_phi);
11977 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11978 operation as a normal volatile store. */
11981 expand_omp_atomic_store (basic_block load_bb, tree addr,
11982 tree loaded_val, tree stored_val, int index)
11984 enum built_in_function tmpbase;
11985 gimple_stmt_iterator gsi;
11986 basic_block store_bb = single_succ (load_bb);
11989 tree decl, call, type, itype;
11990 machine_mode imode;
11993 gsi = gsi_last_bb (load_bb);
11994 stmt = gsi_stmt (gsi);
11995 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
11997 /* If the load value is needed, then this isn't a store but an exchange. */
11998 exchange = gimple_omp_atomic_need_value_p (stmt);
12000 gsi = gsi_last_bb (store_bb);
12001 stmt = gsi_stmt (gsi);
12002 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE);
12003 loc = gimple_location (stmt);
12005 /* ??? If the target does not implement atomic_store_optab[mode], and mode
12006 is smaller than word size, then expand_atomic_store assumes that the store
12007 is atomic. We could avoid the builtin entirely in this case. */
12009 tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N);
12010 tmpbase = (enum built_in_function) ((int) tmpbase + index + 1);
12011 decl = builtin_decl_explicit (tmpbase);
12012 if (decl == NULL_TREE)
12015 type = TREE_TYPE (stored_val);
12017 /* Dig out the type of the function's second argument. */
12018 itype = TREE_TYPE (decl);
12019 itype = TYPE_ARG_TYPES (itype);
12020 itype = TREE_CHAIN (itype);
12021 itype = TREE_VALUE (itype);
12022 imode = TYPE_MODE (itype);
12024 if (exchange && !can_atomic_exchange_p (imode, true))
12027 if (!useless_type_conversion_p (itype, type))
12028 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
12029 call = build_call_expr_loc (loc, decl, 3, addr, stored_val,
12030 build_int_cst (NULL,
12031 gimple_omp_atomic_seq_cst_p (stmt)
12033 : MEMMODEL_RELAXED));
12036 if (!useless_type_conversion_p (type, itype))
12037 call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
12038 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
12041 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
12042 gsi_remove (&gsi, true);
12044 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
12045 gsi = gsi_last_bb (load_bb);
12046 gsi_remove (&gsi, true);
12048 if (gimple_in_ssa_p (cfun))
12049 update_ssa (TODO_update_ssa_no_phi);
12054 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
12055 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
12056 size of the data type, and thus usable to find the index of the builtin
12057 decl. Returns false if the expression is not of the proper form. */
12060 expand_omp_atomic_fetch_op (basic_block load_bb,
12061 tree addr, tree loaded_val,
12062 tree stored_val, int index)
12064 enum built_in_function oldbase, newbase, tmpbase;
12065 tree decl, itype, call;
12067 basic_block store_bb = single_succ (load_bb);
12068 gimple_stmt_iterator gsi;
12071 enum tree_code code;
12072 bool need_old, need_new;
12073 machine_mode imode;
12076 /* We expect to find the following sequences:
12079 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
12082 val = tmp OP something; (or: something OP tmp)
12083 GIMPLE_OMP_STORE (val)
12085 ???FIXME: Allow a more flexible sequence.
12086 Perhaps use data flow to pick the statements.
12090 gsi = gsi_after_labels (store_bb);
12091 stmt = gsi_stmt (gsi);
12092 loc = gimple_location (stmt);
12093 if (!is_gimple_assign (stmt))
12096 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
12098 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
12099 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb));
12100 seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb));
12101 gcc_checking_assert (!need_old || !need_new);
12103 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
12106 /* Check for one of the supported fetch-op operations. */
12107 code = gimple_assign_rhs_code (stmt);
12111 case POINTER_PLUS_EXPR:
12112 oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N;
12113 newbase = BUILT_IN_ATOMIC_ADD_FETCH_N;
12116 oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N;
12117 newbase = BUILT_IN_ATOMIC_SUB_FETCH_N;
12120 oldbase = BUILT_IN_ATOMIC_FETCH_AND_N;
12121 newbase = BUILT_IN_ATOMIC_AND_FETCH_N;
12124 oldbase = BUILT_IN_ATOMIC_FETCH_OR_N;
12125 newbase = BUILT_IN_ATOMIC_OR_FETCH_N;
12128 oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N;
12129 newbase = BUILT_IN_ATOMIC_XOR_FETCH_N;
12135 /* Make sure the expression is of the proper form. */
12136 if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
12137 rhs = gimple_assign_rhs2 (stmt);
12138 else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
12139 && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
12140 rhs = gimple_assign_rhs1 (stmt);
12144 tmpbase = ((enum built_in_function)
12145 ((need_new ? newbase : oldbase) + index + 1));
12146 decl = builtin_decl_explicit (tmpbase);
12147 if (decl == NULL_TREE)
12149 itype = TREE_TYPE (TREE_TYPE (decl));
12150 imode = TYPE_MODE (itype);
12152 /* We could test all of the various optabs involved, but the fact of the
12153 matter is that (with the exception of i486 vs i586 and xadd) all targets
12154 that support any atomic operaton optab also implements compare-and-swap.
12155 Let optabs.c take care of expanding any compare-and-swap loop. */
12156 if (!can_compare_and_swap_p (imode, true))
12159 gsi = gsi_last_bb (load_bb);
12160 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
12162 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
12163 It only requires that the operation happen atomically. Thus we can
12164 use the RELAXED memory model. */
12165 call = build_call_expr_loc (loc, decl, 3, addr,
12166 fold_convert_loc (loc, itype, rhs),
12167 build_int_cst (NULL,
12168 seq_cst ? MEMMODEL_SEQ_CST
12169 : MEMMODEL_RELAXED));
12171 if (need_old || need_new)
12173 lhs = need_old ? loaded_val : stored_val;
12174 call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
12175 call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call);
12178 call = fold_convert_loc (loc, void_type_node, call);
12179 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
12180 gsi_remove (&gsi, true);
12182 gsi = gsi_last_bb (store_bb);
12183 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
12184 gsi_remove (&gsi, true);
12185 gsi = gsi_last_bb (store_bb);
12186 stmt = gsi_stmt (gsi);
12187 gsi_remove (&gsi, true);
12189 if (gimple_in_ssa_p (cfun))
12191 release_defs (stmt);
12192 update_ssa (TODO_update_ssa_no_phi);
12198 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12202 newval = rhs; // with oldval replacing *addr in rhs
12203 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
12204 if (oldval != newval)
12207 INDEX is log2 of the size of the data type, and thus usable to find the
12208 index of the builtin decl. */
12211 expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
12212 tree addr, tree loaded_val, tree stored_val,
12215 tree loadedi, storedi, initial, new_storedi, old_vali;
12216 tree type, itype, cmpxchg, iaddr;
12217 gimple_stmt_iterator si;
12218 basic_block loop_header = single_succ (load_bb);
12219 gimple *phi, *stmt;
12221 enum built_in_function fncode;
12223 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
12224 order to use the RELAXED memory model effectively. */
12225 fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
12227 cmpxchg = builtin_decl_explicit (fncode);
12228 if (cmpxchg == NULL_TREE)
12230 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
12231 itype = TREE_TYPE (TREE_TYPE (cmpxchg));
12233 if (!can_compare_and_swap_p (TYPE_MODE (itype), true))
12236 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
12237 si = gsi_last_bb (load_bb);
12238 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
12240 /* For floating-point values, we'll need to view-convert them to integers
12241 so that we can perform the atomic compare and swap. Simplify the
12242 following code by always setting up the "i"ntegral variables. */
12243 if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
12247 iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode,
12250 = force_gimple_operand_gsi (&si,
12251 fold_convert (TREE_TYPE (iaddr), addr),
12252 false, NULL_TREE, true, GSI_SAME_STMT);
12253 stmt = gimple_build_assign (iaddr, iaddr_val);
12254 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
12255 loadedi = create_tmp_var (itype);
12256 if (gimple_in_ssa_p (cfun))
12257 loadedi = make_ssa_name (loadedi);
12262 loadedi = loaded_val;
12265 fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
12266 tree loaddecl = builtin_decl_explicit (fncode);
12269 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr)),
12270 build_call_expr (loaddecl, 2, iaddr,
12271 build_int_cst (NULL_TREE,
12272 MEMMODEL_RELAXED)));
12274 initial = build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)), iaddr,
12275 build_int_cst (TREE_TYPE (iaddr), 0));
12278 = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true,
12281 /* Move the value to the LOADEDI temporary. */
12282 if (gimple_in_ssa_p (cfun))
12284 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
12285 phi = create_phi_node (loadedi, loop_header);
12286 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
12290 gsi_insert_before (&si,
12291 gimple_build_assign (loadedi, initial),
12293 if (loadedi != loaded_val)
12295 gimple_stmt_iterator gsi2;
12298 x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
12299 gsi2 = gsi_start_bb (loop_header);
12300 if (gimple_in_ssa_p (cfun))
12303 x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
12304 true, GSI_SAME_STMT);
12305 stmt = gimple_build_assign (loaded_val, x);
12306 gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
12310 x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
12311 force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
12312 true, GSI_SAME_STMT);
12315 gsi_remove (&si, true);
12317 si = gsi_last_bb (store_bb);
12318 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
12321 storedi = stored_val;
12324 force_gimple_operand_gsi (&si,
12325 build1 (VIEW_CONVERT_EXPR, itype,
12326 stored_val), true, NULL_TREE, true,
12329 /* Build the compare&swap statement. */
12330 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
12331 new_storedi = force_gimple_operand_gsi (&si,
12332 fold_convert (TREE_TYPE (loadedi),
12335 true, GSI_SAME_STMT);
12337 if (gimple_in_ssa_p (cfun))
12338 old_vali = loadedi;
12341 old_vali = create_tmp_var (TREE_TYPE (loadedi));
12342 stmt = gimple_build_assign (old_vali, loadedi);
12343 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
12345 stmt = gimple_build_assign (loadedi, new_storedi);
12346 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
12349 /* Note that we always perform the comparison as an integer, even for
12350 floating point. This allows the atomic operation to properly
12351 succeed even with NaNs and -0.0. */
12352 stmt = gimple_build_cond_empty
12353 (build2 (NE_EXPR, boolean_type_node,
12354 new_storedi, old_vali));
12355 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
12358 e = single_succ_edge (store_bb);
12359 e->flags &= ~EDGE_FALLTHRU;
12360 e->flags |= EDGE_FALSE_VALUE;
12362 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
12364 /* Copy the new value to loadedi (we already did that before the condition
12365 if we are not in SSA). */
12366 if (gimple_in_ssa_p (cfun))
12368 phi = gimple_seq_first_stmt (phi_nodes (loop_header));
12369 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
12372 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
12373 gsi_remove (&si, true);
12375 struct loop *loop = alloc_loop ();
12376 loop->header = loop_header;
12377 loop->latch = store_bb;
12378 add_loop (loop, loop_header->loop_father);
12380 if (gimple_in_ssa_p (cfun))
12381 update_ssa (TODO_update_ssa_no_phi);
12386 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12388 GOMP_atomic_start ();
12390 GOMP_atomic_end ();
12392 The result is not globally atomic, but works so long as all parallel
12393 references are within #pragma omp atomic directives. According to
12394 responses received from omp@openmp.org, appears to be within spec.
12395 Which makes sense, since that's how several other compilers handle
12396 this situation as well.
12397 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
12398 expanding. STORED_VAL is the operand of the matching
12399 GIMPLE_OMP_ATOMIC_STORE.
12402 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
12403 loaded_val = *addr;
12406 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
12407 *addr = stored_val;
12411 expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
12412 tree addr, tree loaded_val, tree stored_val)
12414 gimple_stmt_iterator si;
12418 si = gsi_last_bb (load_bb);
12419 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
12421 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
12422 t = build_call_expr (t, 0);
12423 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
12425 stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr));
12426 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
12427 gsi_remove (&si, true);
12429 si = gsi_last_bb (store_bb);
12430 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
12432 stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)),
12434 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
12436 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
12437 t = build_call_expr (t, 0);
12438 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
12439 gsi_remove (&si, true);
12441 if (gimple_in_ssa_p (cfun))
12442 update_ssa (TODO_update_ssa_no_phi);
12446 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
12447 using expand_omp_atomic_fetch_op. If it failed, we try to
12448 call expand_omp_atomic_pipeline, and if it fails too, the
12449 ultimate fallback is wrapping the operation in a mutex
12450 (expand_omp_atomic_mutex). REGION is the atomic region built
12451 by build_omp_regions_1(). */
12454 expand_omp_atomic (struct omp_region *region)
12456 basic_block load_bb = region->entry, store_bb = region->exit;
12457 gomp_atomic_load *load = as_a <gomp_atomic_load *> (last_stmt (load_bb));
12458 gomp_atomic_store *store = as_a <gomp_atomic_store *> (last_stmt (store_bb));
12459 tree loaded_val = gimple_omp_atomic_load_lhs (load);
12460 tree addr = gimple_omp_atomic_load_rhs (load);
12461 tree stored_val = gimple_omp_atomic_store_val (store);
12462 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
12463 HOST_WIDE_INT index;
12465 /* Make sure the type is one of the supported sizes. */
12466 index = tree_to_uhwi (TYPE_SIZE_UNIT (type));
12467 index = exact_log2 (index);
12468 if (index >= 0 && index <= 4)
12470 unsigned int align = TYPE_ALIGN_UNIT (type);
12472 /* __sync builtins require strict data alignment. */
12473 if (exact_log2 (align) >= index)
12476 if (loaded_val == stored_val
12477 && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
12478 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
12479 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
12480 && expand_omp_atomic_load (load_bb, addr, loaded_val, index))
12483 /* Atomic store. */
12484 if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
12485 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
12486 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
12487 && store_bb == single_succ (load_bb)
12488 && first_stmt (store_bb) == store
12489 && expand_omp_atomic_store (load_bb, addr, loaded_val,
12490 stored_val, index))
12493 /* When possible, use specialized atomic update functions. */
12494 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
12495 && store_bb == single_succ (load_bb)
12496 && expand_omp_atomic_fetch_op (load_bb, addr,
12497 loaded_val, stored_val, index))
12500 /* If we don't have specialized __sync builtins, try and implement
12501 as a compare and swap loop. */
12502 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
12503 loaded_val, stored_val, index))
12508 /* The ultimate fallback is wrapping the operation in a mutex. */
12509 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
12513 /* Encode an oacc launch argument. This matches the GOMP_LAUNCH_PACK
12514 macro on gomp-constants.h. We do not check for overflow. */
12517 oacc_launch_pack (unsigned code, tree device, unsigned op)
12521 res = build_int_cst (unsigned_type_node, GOMP_LAUNCH_PACK (code, 0, op));
12524 device = fold_build2 (LSHIFT_EXPR, unsigned_type_node,
12525 device, build_int_cst (unsigned_type_node,
12526 GOMP_LAUNCH_DEVICE_SHIFT));
12527 res = fold_build2 (BIT_IOR_EXPR, unsigned_type_node, res, device);
12532 /* Look for compute grid dimension clauses and convert to an attribute
12533 attached to FN. This permits the target-side code to (a) massage
12534 the dimensions, (b) emit that data and (c) optimize. Non-constant
12535 dimensions are pushed onto ARGS.
12537 The attribute value is a TREE_LIST. A set of dimensions is
12538 represented as a list of INTEGER_CST. Those that are runtime
12539 exprs are represented as an INTEGER_CST of zero.
12541 TOOO. Normally the attribute will just contain a single such list. If
12542 however it contains a list of lists, this will represent the use of
12543 device_type. Each member of the outer list is an assoc list of
12544 dimensions, keyed by the device type. The first entry will be the
12545 default. Well, that's the plan. */
12547 #define OACC_FN_ATTRIB "oacc function"
12549 /* Replace any existing oacc fn attribute with updated dimensions. */
12552 replace_oacc_fn_attrib (tree fn, tree dims)
12554 tree ident = get_identifier (OACC_FN_ATTRIB);
12555 tree attribs = DECL_ATTRIBUTES (fn);
12557 /* If we happen to be present as the first attrib, drop it. */
12558 if (attribs && TREE_PURPOSE (attribs) == ident)
12559 attribs = TREE_CHAIN (attribs);
12560 DECL_ATTRIBUTES (fn) = tree_cons (ident, dims, attribs);
12563 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12564 function attribute. Push any that are non-constant onto the ARGS
12565 list, along with an appropriate GOMP_LAUNCH_DIM tag. IS_KERNEL is
12566 true, if these are for a kernels region offload function. */
12569 set_oacc_fn_attrib (tree fn, tree clauses, bool is_kernel, vec<tree> *args)
12571 /* Must match GOMP_DIM ordering. */
12572 static const omp_clause_code ids[]
12573 = { OMP_CLAUSE_NUM_GANGS, OMP_CLAUSE_NUM_WORKERS,
12574 OMP_CLAUSE_VECTOR_LENGTH };
12576 tree dims[GOMP_DIM_MAX];
12577 tree attr = NULL_TREE;
12578 unsigned non_const = 0;
12580 for (ix = GOMP_DIM_MAX; ix--;)
12582 tree clause = find_omp_clause (clauses, ids[ix]);
12583 tree dim = NULL_TREE;
12586 dim = OMP_CLAUSE_EXPR (clause, ids[ix]);
12588 if (dim && TREE_CODE (dim) != INTEGER_CST)
12590 dim = integer_zero_node;
12591 non_const |= GOMP_DIM_MASK (ix);
12593 attr = tree_cons (NULL_TREE, dim, attr);
12594 /* Note kernelness with TREE_PUBLIC. */
12596 TREE_PUBLIC (attr) = 1;
12599 replace_oacc_fn_attrib (fn, attr);
12603 /* Push a dynamic argument set. */
12604 args->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM,
12605 NULL_TREE, non_const));
12606 for (unsigned ix = 0; ix != GOMP_DIM_MAX; ix++)
12607 if (non_const & GOMP_DIM_MASK (ix))
12608 args->safe_push (dims[ix]);
12612 /* Process the routine's dimension clauess to generate an attribute
12613 value. Issue diagnostics as appropriate. We default to SEQ
12614 (OpenACC 2.5 clarifies this). All dimensions have a size of zero
12615 (dynamic). TREE_PURPOSE is set to indicate whether that dimension
12616 can have a loop partitioned on it. non-zero indicates
12617 yes, zero indicates no. By construction once a non-zero has been
12618 reached, further inner dimensions must also be non-zero. We set
12619 TREE_VALUE to zero for the dimensions that may be partitioned and
12620 1 for the other ones -- if a loop is (erroneously) spawned at
12621 an outer level, we don't want to try and partition it. */
12624 build_oacc_routine_dims (tree clauses)
12626 /* Must match GOMP_DIM ordering. */
12627 static const omp_clause_code ids[] =
12628 {OMP_CLAUSE_GANG, OMP_CLAUSE_WORKER, OMP_CLAUSE_VECTOR, OMP_CLAUSE_SEQ};
12632 for (; clauses; clauses = OMP_CLAUSE_CHAIN (clauses))
12633 for (ix = GOMP_DIM_MAX + 1; ix--;)
12634 if (OMP_CLAUSE_CODE (clauses) == ids[ix])
12637 error_at (OMP_CLAUSE_LOCATION (clauses),
12638 "multiple loop axes specified for routine");
12643 /* Default to SEQ. */
12645 level = GOMP_DIM_MAX;
12647 tree dims = NULL_TREE;
12649 for (ix = GOMP_DIM_MAX; ix--;)
12650 dims = tree_cons (build_int_cst (boolean_type_node, ix >= level),
12651 build_int_cst (integer_type_node, ix < level), dims);
12656 /* Retrieve the oacc function attrib and return it. Non-oacc
12657 functions will return NULL. */
12660 get_oacc_fn_attrib (tree fn)
12662 return lookup_attribute (OACC_FN_ATTRIB, DECL_ATTRIBUTES (fn));
12665 /* Return true if this oacc fn attrib is for a kernels offload
12666 region. We use the TREE_PUBLIC flag of each dimension -- only
12667 need to check the first one. */
12670 oacc_fn_attrib_kernels_p (tree attr)
12672 return TREE_PUBLIC (TREE_VALUE (attr));
12675 /* Return level at which oacc routine may spawn a partitioned loop, or
12676 -1 if it is not a routine (i.e. is an offload fn). */
12679 oacc_fn_attrib_level (tree attr)
12681 tree pos = TREE_VALUE (attr);
12683 if (!TREE_PURPOSE (pos))
12687 for (ix = 0; ix != GOMP_DIM_MAX;
12688 ix++, pos = TREE_CHAIN (pos))
12689 if (!integer_zerop (TREE_PURPOSE (pos)))
12695 /* Extract an oacc execution dimension from FN. FN must be an
12696 offloaded function or routine that has already had its execution
12697 dimensions lowered to the target-specific values. */
12700 get_oacc_fn_dim_size (tree fn, int axis)
12702 tree attrs = get_oacc_fn_attrib (fn);
12704 gcc_assert (axis < GOMP_DIM_MAX);
12706 tree dims = TREE_VALUE (attrs);
12708 dims = TREE_CHAIN (dims);
12710 int size = TREE_INT_CST_LOW (TREE_VALUE (dims));
12715 /* Extract the dimension axis from an IFN_GOACC_DIM_POS or
12716 IFN_GOACC_DIM_SIZE call. */
12719 get_oacc_ifn_dim_arg (const gimple *stmt)
12721 gcc_checking_assert (gimple_call_internal_fn (stmt) == IFN_GOACC_DIM_SIZE
12722 || gimple_call_internal_fn (stmt) == IFN_GOACC_DIM_POS);
12723 tree arg = gimple_call_arg (stmt, 0);
12724 HOST_WIDE_INT axis = TREE_INT_CST_LOW (arg);
12726 gcc_checking_assert (axis >= 0 && axis < GOMP_DIM_MAX);
12730 /* Mark the loops inside the kernels region starting at REGION_ENTRY and ending
12734 mark_loops_in_oacc_kernels_region (basic_block region_entry,
12735 basic_block region_exit)
12737 struct loop *outer = region_entry->loop_father;
12738 gcc_assert (region_exit == NULL || outer == region_exit->loop_father);
12740 /* Don't parallelize the kernels region if it contains more than one outer
12742 unsigned int nr_outer_loops = 0;
12743 struct loop *single_outer = NULL;
12744 for (struct loop *loop = outer->inner; loop != NULL; loop = loop->next)
12746 gcc_assert (loop_outer (loop) == outer);
12748 if (!dominated_by_p (CDI_DOMINATORS, loop->header, region_entry))
12751 if (region_exit != NULL
12752 && dominated_by_p (CDI_DOMINATORS, loop->header, region_exit))
12756 single_outer = loop;
12758 if (nr_outer_loops != 1)
12761 for (struct loop *loop = single_outer->inner; loop != NULL; loop = loop->inner)
12765 /* Mark the loops in the region. */
12766 for (struct loop *loop = single_outer; loop != NULL; loop = loop->inner)
12767 loop->in_oacc_kernels_region = true;
12770 /* Types used to pass grid and wortkgroup sizes to kernel invocation. */
12772 struct GTY(()) grid_launch_attributes_trees
12774 tree kernel_dim_array_type;
12775 tree kernel_lattrs_dimnum_decl;
12776 tree kernel_lattrs_grid_decl;
12777 tree kernel_lattrs_group_decl;
12778 tree kernel_launch_attributes_type;
12781 static GTY(()) struct grid_launch_attributes_trees *grid_attr_trees;
12783 /* Create types used to pass kernel launch attributes to target. */
12786 grid_create_kernel_launch_attr_types (void)
12788 if (grid_attr_trees)
12790 grid_attr_trees = ggc_alloc <grid_launch_attributes_trees> ();
12792 tree dim_arr_index_type
12793 = build_index_type (build_int_cst (integer_type_node, 2));
12794 grid_attr_trees->kernel_dim_array_type
12795 = build_array_type (uint32_type_node, dim_arr_index_type);
12797 grid_attr_trees->kernel_launch_attributes_type = make_node (RECORD_TYPE);
12798 grid_attr_trees->kernel_lattrs_dimnum_decl
12799 = build_decl (BUILTINS_LOCATION, FIELD_DECL, get_identifier ("ndim"),
12801 DECL_CHAIN (grid_attr_trees->kernel_lattrs_dimnum_decl) = NULL_TREE;
12803 grid_attr_trees->kernel_lattrs_grid_decl
12804 = build_decl (BUILTINS_LOCATION, FIELD_DECL, get_identifier ("grid_size"),
12805 grid_attr_trees->kernel_dim_array_type);
12806 DECL_CHAIN (grid_attr_trees->kernel_lattrs_grid_decl)
12807 = grid_attr_trees->kernel_lattrs_dimnum_decl;
12808 grid_attr_trees->kernel_lattrs_group_decl
12809 = build_decl (BUILTINS_LOCATION, FIELD_DECL, get_identifier ("group_size"),
12810 grid_attr_trees->kernel_dim_array_type);
12811 DECL_CHAIN (grid_attr_trees->kernel_lattrs_group_decl)
12812 = grid_attr_trees->kernel_lattrs_grid_decl;
12813 finish_builtin_struct (grid_attr_trees->kernel_launch_attributes_type,
12814 "__gomp_kernel_launch_attributes",
12815 grid_attr_trees->kernel_lattrs_group_decl, NULL_TREE);
12818 /* Insert before the current statement in GSI a store of VALUE to INDEX of
12819 array (of type kernel_dim_array_type) FLD_DECL of RANGE_VAR. VALUE must be
12820 of type uint32_type_node. */
12823 grid_insert_store_range_dim (gimple_stmt_iterator *gsi, tree range_var,
12824 tree fld_decl, int index, tree value)
12826 tree ref = build4 (ARRAY_REF, uint32_type_node,
12827 build3 (COMPONENT_REF,
12828 grid_attr_trees->kernel_dim_array_type,
12829 range_var, fld_decl, NULL_TREE),
12830 build_int_cst (integer_type_node, index),
12831 NULL_TREE, NULL_TREE);
12832 gsi_insert_before (gsi, gimple_build_assign (ref, value), GSI_SAME_STMT);
12835 /* Return a tree representation of a pointer to a structure with grid and
12836 work-group size information. Statements filling that information will be
12837 inserted before GSI, TGT_STMT is the target statement which has the
12838 necessary information in it. */
12841 grid_get_kernel_launch_attributes (gimple_stmt_iterator *gsi,
12842 gomp_target *tgt_stmt)
12844 grid_create_kernel_launch_attr_types ();
12845 tree u32_one = build_one_cst (uint32_type_node);
12846 tree lattrs = create_tmp_var (grid_attr_trees->kernel_launch_attributes_type,
12847 "__kernel_launch_attrs");
12849 unsigned max_dim = 0;
12850 for (tree clause = gimple_omp_target_clauses (tgt_stmt);
12852 clause = OMP_CLAUSE_CHAIN (clause))
12854 if (OMP_CLAUSE_CODE (clause) != OMP_CLAUSE__GRIDDIM_)
12857 unsigned dim = OMP_CLAUSE__GRIDDIM__DIMENSION (clause);
12858 max_dim = MAX (dim, max_dim);
12860 grid_insert_store_range_dim (gsi, lattrs,
12861 grid_attr_trees->kernel_lattrs_grid_decl,
12862 dim, OMP_CLAUSE__GRIDDIM__SIZE (clause));
12863 grid_insert_store_range_dim (gsi, lattrs,
12864 grid_attr_trees->kernel_lattrs_group_decl,
12865 dim, OMP_CLAUSE__GRIDDIM__GROUP (clause));
12868 tree dimref = build3 (COMPONENT_REF, uint32_type_node, lattrs,
12869 grid_attr_trees->kernel_lattrs_dimnum_decl, NULL_TREE);
12870 /* At this moment we cannot gridify a loop with a collapse clause. */
12871 /* TODO: Adjust when we support bigger collapse. */
12872 gcc_assert (max_dim == 0);
12873 gsi_insert_before (gsi, gimple_build_assign (dimref, u32_one), GSI_SAME_STMT);
12874 TREE_ADDRESSABLE (lattrs) = 1;
12875 return build_fold_addr_expr (lattrs);
12878 /* Build target argument identifier from the DEVICE identifier, value
12879 identifier ID and whether the element also has a SUBSEQUENT_PARAM. */
12882 get_target_argument_identifier_1 (int device, bool subseqent_param, int id)
12884 tree t = build_int_cst (integer_type_node, device);
12885 if (subseqent_param)
12886 t = fold_build2 (BIT_IOR_EXPR, integer_type_node, t,
12887 build_int_cst (integer_type_node,
12888 GOMP_TARGET_ARG_SUBSEQUENT_PARAM));
12889 t = fold_build2 (BIT_IOR_EXPR, integer_type_node, t,
12890 build_int_cst (integer_type_node, id));
12894 /* Like above but return it in type that can be directly stored as an element
12895 of the argument array. */
12898 get_target_argument_identifier (int device, bool subseqent_param, int id)
12900 tree t = get_target_argument_identifier_1 (device, subseqent_param, id);
12901 return fold_convert (ptr_type_node, t);
12904 /* Return a target argument consisting of DEVICE identifier, value identifier
12905 ID, and the actual VALUE. */
12908 get_target_argument_value (gimple_stmt_iterator *gsi, int device, int id,
12911 tree t = fold_build2 (LSHIFT_EXPR, integer_type_node,
12912 fold_convert (integer_type_node, value),
12913 build_int_cst (unsigned_type_node,
12914 GOMP_TARGET_ARG_VALUE_SHIFT));
12915 t = fold_build2 (BIT_IOR_EXPR, integer_type_node, t,
12916 get_target_argument_identifier_1 (device, false, id));
12917 t = fold_convert (ptr_type_node, t);
12918 return force_gimple_operand_gsi (gsi, t, true, NULL, true, GSI_SAME_STMT);
12921 /* If VALUE is an integer constant greater than -2^15 and smaller than 2^15,
12922 push one argument to ARGS with both the DEVICE, ID and VALUE embedded in it,
12923 otherwise push an identifier (with DEVICE and ID) and the VALUE in two
12927 push_target_argument_according_to_value (gimple_stmt_iterator *gsi, int device,
12928 int id, tree value, vec <tree> *args)
12930 if (tree_fits_shwi_p (value)
12931 && tree_to_shwi (value) > -(1 << 15)
12932 && tree_to_shwi (value) < (1 << 15))
12933 args->quick_push (get_target_argument_value (gsi, device, id, value));
12936 args->quick_push (get_target_argument_identifier (device, true, id));
12937 value = fold_convert (ptr_type_node, value);
12938 value = force_gimple_operand_gsi (gsi, value, true, NULL, true,
12940 args->quick_push (value);
12944 /* Create an array of arguments that is then passed to GOMP_target. */
12947 get_target_arguments (gimple_stmt_iterator *gsi, gomp_target *tgt_stmt)
12949 auto_vec <tree, 6> args;
12950 tree clauses = gimple_omp_target_clauses (tgt_stmt);
12951 tree t, c = find_omp_clause (clauses, OMP_CLAUSE_NUM_TEAMS);
12953 t = OMP_CLAUSE_NUM_TEAMS_EXPR (c);
12955 t = integer_minus_one_node;
12956 push_target_argument_according_to_value (gsi, GOMP_TARGET_ARG_DEVICE_ALL,
12957 GOMP_TARGET_ARG_NUM_TEAMS, t, &args);
12959 c = find_omp_clause (clauses, OMP_CLAUSE_THREAD_LIMIT);
12961 t = OMP_CLAUSE_THREAD_LIMIT_EXPR (c);
12963 t = integer_minus_one_node;
12964 push_target_argument_according_to_value (gsi, GOMP_TARGET_ARG_DEVICE_ALL,
12965 GOMP_TARGET_ARG_THREAD_LIMIT, t,
12968 /* Add HSA-specific grid sizes, if available. */
12969 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt),
12970 OMP_CLAUSE__GRIDDIM_))
12972 t = get_target_argument_identifier (GOMP_DEVICE_HSA, true,
12973 GOMP_TARGET_ARG_HSA_KERNEL_ATTRIBUTES);
12974 args.quick_push (t);
12975 args.quick_push (grid_get_kernel_launch_attributes (gsi, tgt_stmt));
12978 /* Produce more, perhaps device specific, arguments here. */
12980 tree argarray = create_tmp_var (build_array_type_nelts (ptr_type_node,
12981 args.length () + 1),
12982 ".omp_target_args");
12983 for (unsigned i = 0; i < args.length (); i++)
12985 tree ref = build4 (ARRAY_REF, ptr_type_node, argarray,
12986 build_int_cst (integer_type_node, i),
12987 NULL_TREE, NULL_TREE);
12988 gsi_insert_before (gsi, gimple_build_assign (ref, args[i]),
12991 tree ref = build4 (ARRAY_REF, ptr_type_node, argarray,
12992 build_int_cst (integer_type_node, args.length ()),
12993 NULL_TREE, NULL_TREE);
12994 gsi_insert_before (gsi, gimple_build_assign (ref, null_pointer_node),
12996 TREE_ADDRESSABLE (argarray) = 1;
12997 return build_fold_addr_expr (argarray);
13000 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
13003 expand_omp_target (struct omp_region *region)
13005 basic_block entry_bb, exit_bb, new_bb;
13006 struct function *child_cfun;
13007 tree child_fn, block, t;
13008 gimple_stmt_iterator gsi;
13009 gomp_target *entry_stmt;
13012 bool offloaded, data_region;
13014 entry_stmt = as_a <gomp_target *> (last_stmt (region->entry));
13015 new_bb = region->entry;
13017 offloaded = is_gimple_omp_offloaded (entry_stmt);
13018 switch (gimple_omp_target_kind (entry_stmt))
13020 case GF_OMP_TARGET_KIND_REGION:
13021 case GF_OMP_TARGET_KIND_UPDATE:
13022 case GF_OMP_TARGET_KIND_ENTER_DATA:
13023 case GF_OMP_TARGET_KIND_EXIT_DATA:
13024 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
13025 case GF_OMP_TARGET_KIND_OACC_KERNELS:
13026 case GF_OMP_TARGET_KIND_OACC_UPDATE:
13027 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
13028 case GF_OMP_TARGET_KIND_OACC_DECLARE:
13029 data_region = false;
13031 case GF_OMP_TARGET_KIND_DATA:
13032 case GF_OMP_TARGET_KIND_OACC_DATA:
13033 case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
13034 data_region = true;
13037 gcc_unreachable ();
13040 child_fn = NULL_TREE;
13044 child_fn = gimple_omp_target_child_fn (entry_stmt);
13045 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
13048 /* Supported by expand_omp_taskreg, but not here. */
13049 if (child_cfun != NULL)
13050 gcc_checking_assert (!child_cfun->cfg);
13051 gcc_checking_assert (!gimple_in_ssa_p (cfun));
13053 entry_bb = region->entry;
13054 exit_bb = region->exit;
13056 if (gimple_omp_target_kind (entry_stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS)
13057 mark_loops_in_oacc_kernels_region (region->entry, region->exit);
13061 unsigned srcidx, dstidx, num;
13063 /* If the offloading region needs data sent from the parent
13064 function, then the very first statement (except possible
13065 tree profile counter updates) of the offloading body
13066 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
13067 &.OMP_DATA_O is passed as an argument to the child function,
13068 we need to replace it with the argument as seen by the child
13071 In most cases, this will end up being the identity assignment
13072 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
13073 a function call that has been inlined, the original PARM_DECL
13074 .OMP_DATA_I may have been converted into a different local
13075 variable. In which case, we need to keep the assignment. */
13076 tree data_arg = gimple_omp_target_data_arg (entry_stmt);
13079 basic_block entry_succ_bb = single_succ (entry_bb);
13080 gimple_stmt_iterator gsi;
13082 gimple *tgtcopy_stmt = NULL;
13083 tree sender = TREE_VEC_ELT (data_arg, 0);
13085 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
13087 gcc_assert (!gsi_end_p (gsi));
13088 stmt = gsi_stmt (gsi);
13089 if (gimple_code (stmt) != GIMPLE_ASSIGN)
13092 if (gimple_num_ops (stmt) == 2)
13094 tree arg = gimple_assign_rhs1 (stmt);
13096 /* We're ignoring the subcode because we're
13097 effectively doing a STRIP_NOPS. */
13099 if (TREE_CODE (arg) == ADDR_EXPR
13100 && TREE_OPERAND (arg, 0) == sender)
13102 tgtcopy_stmt = stmt;
13108 gcc_assert (tgtcopy_stmt != NULL);
13109 arg = DECL_ARGUMENTS (child_fn);
13111 gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg);
13112 gsi_remove (&gsi, true);
13115 /* Declare local variables needed in CHILD_CFUN. */
13116 block = DECL_INITIAL (child_fn);
13117 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
13118 /* The gimplifier could record temporaries in the offloading block
13119 rather than in containing function's local_decls chain,
13120 which would mean cgraph missed finalizing them. Do it now. */
13121 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
13122 if (TREE_CODE (t) == VAR_DECL
13124 && !DECL_EXTERNAL (t))
13125 varpool_node::finalize_decl (t);
13126 DECL_SAVED_TREE (child_fn) = NULL;
13127 /* We'll create a CFG for child_fn, so no gimple body is needed. */
13128 gimple_set_body (child_fn, NULL);
13129 TREE_USED (block) = 1;
13131 /* Reset DECL_CONTEXT on function arguments. */
13132 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
13133 DECL_CONTEXT (t) = child_fn;
13135 /* Split ENTRY_BB at GIMPLE_*,
13136 so that it can be moved to the child function. */
13137 gsi = gsi_last_bb (entry_bb);
13138 stmt = gsi_stmt (gsi);
13140 && gimple_code (stmt) == gimple_code (entry_stmt));
13141 e = split_block (entry_bb, stmt);
13142 gsi_remove (&gsi, true);
13143 entry_bb = e->dest;
13144 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
13146 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
13149 gsi = gsi_last_bb (exit_bb);
13150 gcc_assert (!gsi_end_p (gsi)
13151 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
13152 stmt = gimple_build_return (NULL);
13153 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
13154 gsi_remove (&gsi, true);
13157 /* Move the offloading region into CHILD_CFUN. */
13159 block = gimple_block (entry_stmt);
13161 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
13163 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
13164 /* When the OMP expansion process cannot guarantee an up-to-date
13165 loop tree arrange for the child function to fixup loops. */
13166 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
13167 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
13169 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
13170 num = vec_safe_length (child_cfun->local_decls);
13171 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
13173 t = (*child_cfun->local_decls)[srcidx];
13174 if (DECL_CONTEXT (t) == cfun->decl)
13176 if (srcidx != dstidx)
13177 (*child_cfun->local_decls)[dstidx] = t;
13181 vec_safe_truncate (child_cfun->local_decls, dstidx);
13183 /* Inform the callgraph about the new function. */
13184 child_cfun->curr_properties = cfun->curr_properties;
13185 child_cfun->has_simduid_loops |= cfun->has_simduid_loops;
13186 child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops;
13187 cgraph_node *node = cgraph_node::get_create (child_fn);
13188 node->parallelized_function = 1;
13189 cgraph_node::add_new_function (child_fn, true);
13191 /* Add the new function to the offload table. */
13192 if (ENABLE_OFFLOADING)
13193 vec_safe_push (offload_funcs, child_fn);
13195 bool need_asm = DECL_ASSEMBLER_NAME_SET_P (current_function_decl)
13196 && !DECL_ASSEMBLER_NAME_SET_P (child_fn);
13198 /* Fix the callgraph edges for child_cfun. Those for cfun will be
13199 fixed in a following pass. */
13200 push_cfun (child_cfun);
13202 assign_assembler_name_if_neeeded (child_fn);
13203 cgraph_edge::rebuild_edges ();
13205 /* Some EH regions might become dead, see PR34608. If
13206 pass_cleanup_cfg isn't the first pass to happen with the
13207 new child, these dead EH edges might cause problems.
13208 Clean them up now. */
13209 if (flag_exceptions)
13212 bool changed = false;
13214 FOR_EACH_BB_FN (bb, cfun)
13215 changed |= gimple_purge_dead_eh_edges (bb);
13217 cleanup_tree_cfg ();
13219 if (flag_checking && !loops_state_satisfies_p (LOOPS_NEED_FIXUP))
13220 verify_loop_structure ();
13223 if (dump_file && !gimple_in_ssa_p (cfun))
13225 omp_any_child_fn_dumped = true;
13226 dump_function_header (dump_file, child_fn, dump_flags);
13227 dump_function_to_file (child_fn, dump_file, dump_flags);
13231 /* Emit a library call to launch the offloading region, or do data
13233 tree t1, t2, t3, t4, device, cond, depend, c, clauses;
13234 enum built_in_function start_ix;
13235 location_t clause_loc;
13236 unsigned int flags_i = 0;
13237 bool oacc_kernels_p = false;
13239 switch (gimple_omp_target_kind (entry_stmt))
13241 case GF_OMP_TARGET_KIND_REGION:
13242 start_ix = BUILT_IN_GOMP_TARGET;
13244 case GF_OMP_TARGET_KIND_DATA:
13245 start_ix = BUILT_IN_GOMP_TARGET_DATA;
13247 case GF_OMP_TARGET_KIND_UPDATE:
13248 start_ix = BUILT_IN_GOMP_TARGET_UPDATE;
13250 case GF_OMP_TARGET_KIND_ENTER_DATA:
13251 start_ix = BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA;
13253 case GF_OMP_TARGET_KIND_EXIT_DATA:
13254 start_ix = BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA;
13255 flags_i |= GOMP_TARGET_FLAG_EXIT_DATA;
13257 case GF_OMP_TARGET_KIND_OACC_KERNELS:
13258 oacc_kernels_p = true;
13260 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
13261 start_ix = BUILT_IN_GOACC_PARALLEL;
13263 case GF_OMP_TARGET_KIND_OACC_DATA:
13264 case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
13265 start_ix = BUILT_IN_GOACC_DATA_START;
13267 case GF_OMP_TARGET_KIND_OACC_UPDATE:
13268 start_ix = BUILT_IN_GOACC_UPDATE;
13270 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
13271 start_ix = BUILT_IN_GOACC_ENTER_EXIT_DATA;
13273 case GF_OMP_TARGET_KIND_OACC_DECLARE:
13274 start_ix = BUILT_IN_GOACC_DECLARE;
13277 gcc_unreachable ();
13280 clauses = gimple_omp_target_clauses (entry_stmt);
13282 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
13283 library choose) and there is no conditional. */
13285 device = build_int_cst (integer_type_node, GOMP_DEVICE_ICV);
13287 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
13289 cond = OMP_CLAUSE_IF_EXPR (c);
13291 c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE);
13294 /* Even if we pass it to all library function calls, it is currently only
13295 defined/used for the OpenMP target ones. */
13296 gcc_checking_assert (start_ix == BUILT_IN_GOMP_TARGET
13297 || start_ix == BUILT_IN_GOMP_TARGET_DATA
13298 || start_ix == BUILT_IN_GOMP_TARGET_UPDATE
13299 || start_ix == BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA);
13301 device = OMP_CLAUSE_DEVICE_ID (c);
13302 clause_loc = OMP_CLAUSE_LOCATION (c);
13305 clause_loc = gimple_location (entry_stmt);
13307 c = find_omp_clause (clauses, OMP_CLAUSE_NOWAIT);
13309 flags_i |= GOMP_TARGET_FLAG_NOWAIT;
13311 /* Ensure 'device' is of the correct type. */
13312 device = fold_convert_loc (clause_loc, integer_type_node, device);
13314 /* If we found the clause 'if (cond)', build
13315 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
13318 cond = gimple_boolify (cond);
13320 basic_block cond_bb, then_bb, else_bb;
13324 tmp_var = create_tmp_var (TREE_TYPE (device));
13326 e = split_block_after_labels (new_bb);
13329 gsi = gsi_last_bb (new_bb);
13331 e = split_block (new_bb, gsi_stmt (gsi));
13337 then_bb = create_empty_bb (cond_bb);
13338 else_bb = create_empty_bb (then_bb);
13339 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
13340 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
13342 stmt = gimple_build_cond_empty (cond);
13343 gsi = gsi_last_bb (cond_bb);
13344 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
13346 gsi = gsi_start_bb (then_bb);
13347 stmt = gimple_build_assign (tmp_var, device);
13348 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
13350 gsi = gsi_start_bb (else_bb);
13351 stmt = gimple_build_assign (tmp_var,
13352 build_int_cst (integer_type_node,
13353 GOMP_DEVICE_HOST_FALLBACK));
13354 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
13356 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
13357 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
13358 add_bb_to_loop (then_bb, cond_bb->loop_father);
13359 add_bb_to_loop (else_bb, cond_bb->loop_father);
13360 make_edge (then_bb, new_bb, EDGE_FALLTHRU);
13361 make_edge (else_bb, new_bb, EDGE_FALLTHRU);
13364 gsi = gsi_last_bb (new_bb);
13368 gsi = gsi_last_bb (new_bb);
13369 device = force_gimple_operand_gsi (&gsi, device, true, NULL_TREE,
13370 true, GSI_SAME_STMT);
13373 t = gimple_omp_target_data_arg (entry_stmt);
13376 t1 = size_zero_node;
13377 t2 = build_zero_cst (ptr_type_node);
13383 t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1))));
13384 t1 = size_binop (PLUS_EXPR, t1, size_int (1));
13385 t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0));
13386 t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1));
13387 t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2));
13391 bool tagging = false;
13392 /* The maximum number used by any start_ix, without varargs. */
13393 auto_vec<tree, 11> args;
13394 args.quick_push (device);
13396 args.quick_push (build_fold_addr_expr (child_fn));
13397 args.quick_push (t1);
13398 args.quick_push (t2);
13399 args.quick_push (t3);
13400 args.quick_push (t4);
13403 case BUILT_IN_GOACC_DATA_START:
13404 case BUILT_IN_GOACC_DECLARE:
13405 case BUILT_IN_GOMP_TARGET_DATA:
13407 case BUILT_IN_GOMP_TARGET:
13408 case BUILT_IN_GOMP_TARGET_UPDATE:
13409 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA:
13410 args.quick_push (build_int_cst (unsigned_type_node, flags_i));
13411 c = find_omp_clause (clauses, OMP_CLAUSE_DEPEND);
13413 depend = OMP_CLAUSE_DECL (c);
13415 depend = build_int_cst (ptr_type_node, 0);
13416 args.quick_push (depend);
13417 if (start_ix == BUILT_IN_GOMP_TARGET)
13418 args.quick_push (get_target_arguments (&gsi, entry_stmt));
13420 case BUILT_IN_GOACC_PARALLEL:
13422 set_oacc_fn_attrib (child_fn, clauses, oacc_kernels_p, &args);
13426 case BUILT_IN_GOACC_ENTER_EXIT_DATA:
13427 case BUILT_IN_GOACC_UPDATE:
13429 tree t_async = NULL_TREE;
13431 /* If present, use the value specified by the respective
13432 clause, making sure that is of the correct type. */
13433 c = find_omp_clause (clauses, OMP_CLAUSE_ASYNC);
13435 t_async = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
13437 OMP_CLAUSE_ASYNC_EXPR (c));
13439 /* Default values for t_async. */
13440 t_async = fold_convert_loc (gimple_location (entry_stmt),
13442 build_int_cst (integer_type_node,
13444 if (tagging && t_async)
13446 unsigned HOST_WIDE_INT i_async = GOMP_LAUNCH_OP_MAX;
13448 if (TREE_CODE (t_async) == INTEGER_CST)
13450 /* See if we can pack the async arg in to the tag's
13452 i_async = TREE_INT_CST_LOW (t_async);
13453 if (i_async < GOMP_LAUNCH_OP_MAX)
13454 t_async = NULL_TREE;
13456 i_async = GOMP_LAUNCH_OP_MAX;
13458 args.safe_push (oacc_launch_pack (GOMP_LAUNCH_ASYNC, NULL_TREE,
13462 args.safe_push (t_async);
13464 /* Save the argument index, and ... */
13465 unsigned t_wait_idx = args.length ();
13466 unsigned num_waits = 0;
13467 c = find_omp_clause (clauses, OMP_CLAUSE_WAIT);
13469 /* ... push a placeholder. */
13470 args.safe_push (integer_zero_node);
13472 for (; c; c = OMP_CLAUSE_CHAIN (c))
13473 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WAIT)
13475 args.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c),
13477 OMP_CLAUSE_WAIT_EXPR (c)));
13481 if (!tagging || num_waits)
13485 /* Now that we know the number, update the placeholder. */
13487 len = oacc_launch_pack (GOMP_LAUNCH_WAIT, NULL_TREE, num_waits);
13489 len = build_int_cst (integer_type_node, num_waits);
13490 len = fold_convert_loc (gimple_location (entry_stmt),
13491 unsigned_type_node, len);
13492 args[t_wait_idx] = len;
13497 gcc_unreachable ();
13500 /* Push terminal marker - zero. */
13501 args.safe_push (oacc_launch_pack (0, NULL_TREE, 0));
13503 g = gimple_build_call_vec (builtin_decl_explicit (start_ix), args);
13504 gimple_set_location (g, gimple_location (entry_stmt));
13505 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
13508 g = gsi_stmt (gsi);
13509 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET);
13510 gsi_remove (&gsi, true);
13512 if (data_region && region->exit)
13514 gsi = gsi_last_bb (region->exit);
13515 g = gsi_stmt (gsi);
13516 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN);
13517 gsi_remove (&gsi, true);
13521 /* Expand KFOR loop as a GPGPU kernel, i.e. as a body only with iteration
13522 variable derived from the thread number. */
13525 grid_expand_omp_for_loop (struct omp_region *kfor)
13529 gimple_stmt_iterator gsi;
13531 struct omp_for_data fd;
13533 gomp_for *for_stmt = as_a <gomp_for *> (last_stmt (kfor->entry));
13534 gcc_checking_assert (gimple_omp_for_kind (for_stmt)
13535 == GF_OMP_FOR_KIND_GRID_LOOP);
13536 basic_block body_bb = FALLTHRU_EDGE (kfor->entry)->dest;
13538 gcc_assert (gimple_omp_for_collapse (for_stmt) == 1);
13539 gcc_assert (kfor->cont);
13540 extract_omp_for_data (for_stmt, &fd, NULL);
13542 itype = type = TREE_TYPE (fd.loop.v);
13543 if (POINTER_TYPE_P (type))
13544 itype = signed_type_for (type);
13546 gsi = gsi_start_bb (body_bb);
13549 step = fd.loop.step;
13550 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
13551 true, NULL_TREE, true, GSI_SAME_STMT);
13552 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
13553 true, NULL_TREE, true, GSI_SAME_STMT);
13554 threadid = build_call_expr (builtin_decl_explicit
13555 (BUILT_IN_OMP_GET_THREAD_NUM), 0);
13556 threadid = fold_convert (itype, threadid);
13557 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
13558 true, GSI_SAME_STMT);
13560 tree startvar = fd.loop.v;
13561 t = fold_build2 (MULT_EXPR, itype, threadid, step);
13562 if (POINTER_TYPE_P (type))
13563 t = fold_build_pointer_plus (n1, t);
13565 t = fold_build2 (PLUS_EXPR, type, t, n1);
13566 t = fold_convert (type, t);
13567 t = force_gimple_operand_gsi (&gsi, t,
13569 && TREE_ADDRESSABLE (startvar),
13570 NULL_TREE, true, GSI_SAME_STMT);
13571 gassign *assign_stmt = gimple_build_assign (startvar, t);
13572 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
13574 /* Remove the omp for statement */
13575 gsi = gsi_last_bb (kfor->entry);
13576 gsi_remove (&gsi, true);
13578 /* Remove the GIMPLE_OMP_CONTINUE statement. */
13579 gsi = gsi_last_bb (kfor->cont);
13580 gcc_assert (!gsi_end_p (gsi)
13581 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_CONTINUE);
13582 gsi_remove (&gsi, true);
13584 /* Replace the GIMPLE_OMP_RETURN with a real return. */
13585 gsi = gsi_last_bb (kfor->exit);
13586 gcc_assert (!gsi_end_p (gsi)
13587 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
13588 gsi_remove (&gsi, true);
13590 /* Fixup the much simpler CFG. */
13591 remove_edge (find_edge (kfor->cont, body_bb));
13593 if (kfor->cont != body_bb)
13594 set_immediate_dominator (CDI_DOMINATORS, kfor->cont, body_bb);
13595 set_immediate_dominator (CDI_DOMINATORS, kfor->exit, kfor->cont);
13598 /* Structure passed to grid_remap_kernel_arg_accesses so that it can remap
13601 struct grid_arg_decl_map
13607 /* Invoked through walk_gimple_op, will remap all PARM_DECLs to the ones
13608 pertaining to kernel function. */
13611 grid_remap_kernel_arg_accesses (tree *tp, int *walk_subtrees, void *data)
13613 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
13614 struct grid_arg_decl_map *adm = (struct grid_arg_decl_map *) wi->info;
13617 if (t == adm->old_arg)
13618 *tp = adm->new_arg;
13619 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
13623 static void expand_omp (struct omp_region *region);
13625 /* If TARGET region contains a kernel body for loop, remove its region from the
13626 TARGET and expand it in GPGPU kernel fashion. */
13629 grid_expand_target_grid_body (struct omp_region *target)
13631 if (!hsa_gen_requested_p ())
13634 gomp_target *tgt_stmt = as_a <gomp_target *> (last_stmt (target->entry));
13635 struct omp_region **pp;
13637 for (pp = &target->inner; *pp; pp = &(*pp)->next)
13638 if ((*pp)->type == GIMPLE_OMP_GRID_BODY)
13641 struct omp_region *gpukernel = *pp;
13643 tree orig_child_fndecl = gimple_omp_target_child_fn (tgt_stmt);
13646 /* HSA cannot handle OACC stuff. */
13647 if (gimple_omp_target_kind (tgt_stmt) != GF_OMP_TARGET_KIND_REGION)
13649 gcc_checking_assert (orig_child_fndecl);
13650 gcc_assert (!find_omp_clause (gimple_omp_target_clauses (tgt_stmt),
13651 OMP_CLAUSE__GRIDDIM_));
13652 cgraph_node *n = cgraph_node::get (orig_child_fndecl);
13654 hsa_register_kernel (n);
13658 gcc_assert (find_omp_clause (gimple_omp_target_clauses (tgt_stmt),
13659 OMP_CLAUSE__GRIDDIM_));
13660 tree inside_block = gimple_block (first_stmt (single_succ (gpukernel->entry)));
13661 *pp = gpukernel->next;
13662 for (pp = &gpukernel->inner; *pp; pp = &(*pp)->next)
13663 if ((*pp)->type == GIMPLE_OMP_FOR)
13666 struct omp_region *kfor = *pp;
13668 gcc_assert (gimple_omp_for_kind (last_stmt ((kfor)->entry))
13669 == GF_OMP_FOR_KIND_GRID_LOOP);
13672 expand_omp (kfor->inner);
13673 if (gpukernel->inner)
13674 expand_omp (gpukernel->inner);
13676 tree kern_fndecl = copy_node (orig_child_fndecl);
13677 DECL_NAME (kern_fndecl) = clone_function_name (kern_fndecl, "kernel");
13678 SET_DECL_ASSEMBLER_NAME (kern_fndecl, DECL_NAME (kern_fndecl));
13679 tree tgtblock = gimple_block (tgt_stmt);
13680 tree fniniblock = make_node (BLOCK);
13681 BLOCK_ABSTRACT_ORIGIN (fniniblock) = tgtblock;
13682 BLOCK_SOURCE_LOCATION (fniniblock) = BLOCK_SOURCE_LOCATION (tgtblock);
13683 BLOCK_SOURCE_END_LOCATION (fniniblock) = BLOCK_SOURCE_END_LOCATION (tgtblock);
13684 DECL_INITIAL (kern_fndecl) = fniniblock;
13685 push_struct_function (kern_fndecl);
13686 cfun->function_end_locus = gimple_location (tgt_stmt);
13689 tree old_parm_decl = DECL_ARGUMENTS (kern_fndecl);
13690 gcc_assert (!DECL_CHAIN (old_parm_decl));
13691 tree new_parm_decl = copy_node (DECL_ARGUMENTS (kern_fndecl));
13692 DECL_CONTEXT (new_parm_decl) = kern_fndecl;
13693 DECL_ARGUMENTS (kern_fndecl) = new_parm_decl;
13694 gcc_assert (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (kern_fndecl))));
13695 DECL_RESULT (kern_fndecl) = copy_node (DECL_RESULT (kern_fndecl));
13696 DECL_CONTEXT (DECL_RESULT (kern_fndecl)) = kern_fndecl;
13697 struct function *kern_cfun = DECL_STRUCT_FUNCTION (kern_fndecl);
13698 kern_cfun->curr_properties = cfun->curr_properties;
13700 remove_edge (BRANCH_EDGE (kfor->entry));
13701 grid_expand_omp_for_loop (kfor);
13703 /* Remove the omp for statement */
13704 gimple_stmt_iterator gsi = gsi_last_bb (gpukernel->entry);
13705 gsi_remove (&gsi, true);
13706 /* Replace the GIMPLE_OMP_RETURN at the end of the kernel region with a real
13708 gsi = gsi_last_bb (gpukernel->exit);
13709 gcc_assert (!gsi_end_p (gsi)
13710 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
13711 gimple *ret_stmt = gimple_build_return (NULL);
13712 gsi_insert_after (&gsi, ret_stmt, GSI_SAME_STMT);
13713 gsi_remove (&gsi, true);
13715 /* Statements in the first BB in the target construct have been produced by
13716 target lowering and must be copied inside the GPUKERNEL, with the two
13717 exceptions of the first OMP statement and the OMP_DATA assignment
13719 gsi = gsi_start_bb (single_succ (gpukernel->entry));
13720 tree data_arg = gimple_omp_target_data_arg (tgt_stmt);
13721 tree sender = data_arg ? TREE_VEC_ELT (data_arg, 0) : NULL;
13722 for (gimple_stmt_iterator tsi = gsi_start_bb (single_succ (target->entry));
13723 !gsi_end_p (tsi); gsi_next (&tsi))
13725 gimple *stmt = gsi_stmt (tsi);
13726 if (is_gimple_omp (stmt))
13729 && is_gimple_assign (stmt)
13730 && TREE_CODE (gimple_assign_rhs1 (stmt)) == ADDR_EXPR
13731 && TREE_OPERAND (gimple_assign_rhs1 (stmt), 0) == sender)
13733 gimple *copy = gimple_copy (stmt);
13734 gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
13735 gimple_set_block (copy, fniniblock);
13738 move_sese_region_to_fn (kern_cfun, single_succ (gpukernel->entry),
13739 gpukernel->exit, inside_block);
13741 cgraph_node *kcn = cgraph_node::get_create (kern_fndecl);
13742 kcn->mark_force_output ();
13743 cgraph_node *orig_child = cgraph_node::get (orig_child_fndecl);
13745 hsa_register_kernel (kcn, orig_child);
13747 cgraph_node::add_new_function (kern_fndecl, true);
13748 push_cfun (kern_cfun);
13749 cgraph_edge::rebuild_edges ();
13751 /* Re-map any mention of the PARM_DECL of the original function to the
13752 PARM_DECL of the new one.
13754 TODO: It would be great if lowering produced references into the GPU
13755 kernel decl straight away and we did not have to do this. */
13756 struct grid_arg_decl_map adm;
13757 adm.old_arg = old_parm_decl;
13758 adm.new_arg = new_parm_decl;
13760 FOR_EACH_BB_FN (bb, kern_cfun)
13762 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
13764 gimple *stmt = gsi_stmt (gsi);
13765 struct walk_stmt_info wi;
13766 memset (&wi, 0, sizeof (wi));
13768 walk_gimple_op (stmt, grid_remap_kernel_arg_accesses, &wi);
13776 /* Expand the parallel region tree rooted at REGION. Expansion
13777 proceeds in depth-first order. Innermost regions are expanded
13778 first. This way, parallel regions that require a new function to
13779 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
13780 internal dependencies in their body. */
13783 expand_omp (struct omp_region *region)
13785 omp_any_child_fn_dumped = false;
13788 location_t saved_location;
13789 gimple *inner_stmt = NULL;
13791 /* First, determine whether this is a combined parallel+workshare
13793 if (region->type == GIMPLE_OMP_PARALLEL)
13794 determine_parallel_type (region);
13795 else if (region->type == GIMPLE_OMP_TARGET)
13796 grid_expand_target_grid_body (region);
13798 if (region->type == GIMPLE_OMP_FOR
13799 && gimple_omp_for_combined_p (last_stmt (region->entry)))
13800 inner_stmt = last_stmt (region->inner->entry);
13803 expand_omp (region->inner);
13805 saved_location = input_location;
13806 if (gimple_has_location (last_stmt (region->entry)))
13807 input_location = gimple_location (last_stmt (region->entry));
13809 switch (region->type)
13811 case GIMPLE_OMP_PARALLEL:
13812 case GIMPLE_OMP_TASK:
13813 expand_omp_taskreg (region);
13816 case GIMPLE_OMP_FOR:
13817 expand_omp_for (region, inner_stmt);
13820 case GIMPLE_OMP_SECTIONS:
13821 expand_omp_sections (region);
13824 case GIMPLE_OMP_SECTION:
13825 /* Individual omp sections are handled together with their
13826 parent GIMPLE_OMP_SECTIONS region. */
13829 case GIMPLE_OMP_SINGLE:
13830 expand_omp_single (region);
13833 case GIMPLE_OMP_ORDERED:
13835 gomp_ordered *ord_stmt
13836 = as_a <gomp_ordered *> (last_stmt (region->entry));
13837 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt),
13838 OMP_CLAUSE_DEPEND))
13840 /* We'll expand these when expanding corresponding
13841 worksharing region with ordered(n) clause. */
13842 gcc_assert (region->outer
13843 && region->outer->type == GIMPLE_OMP_FOR);
13844 region->ord_stmt = ord_stmt;
13849 case GIMPLE_OMP_MASTER:
13850 case GIMPLE_OMP_TASKGROUP:
13851 case GIMPLE_OMP_CRITICAL:
13852 case GIMPLE_OMP_TEAMS:
13853 expand_omp_synch (region);
13856 case GIMPLE_OMP_ATOMIC_LOAD:
13857 expand_omp_atomic (region);
13860 case GIMPLE_OMP_TARGET:
13861 expand_omp_target (region);
13865 gcc_unreachable ();
13868 input_location = saved_location;
13869 region = region->next;
13871 if (omp_any_child_fn_dumped)
13874 dump_function_header (dump_file, current_function_decl, dump_flags);
13875 omp_any_child_fn_dumped = false;
13880 /* Helper for build_omp_regions. Scan the dominator tree starting at
13881 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
13882 true, the function ends once a single tree is built (otherwise, whole
13883 forest of OMP constructs may be built). */
13886 build_omp_regions_1 (basic_block bb, struct omp_region *parent,
13889 gimple_stmt_iterator gsi;
13893 gsi = gsi_last_bb (bb);
13894 if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi)))
13896 struct omp_region *region;
13897 enum gimple_code code;
13899 stmt = gsi_stmt (gsi);
13900 code = gimple_code (stmt);
13901 if (code == GIMPLE_OMP_RETURN)
13903 /* STMT is the return point out of region PARENT. Mark it
13904 as the exit point and make PARENT the immediately
13905 enclosing region. */
13906 gcc_assert (parent);
13909 parent = parent->outer;
13911 else if (code == GIMPLE_OMP_ATOMIC_STORE)
13913 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
13914 GIMPLE_OMP_RETURN, but matches with
13915 GIMPLE_OMP_ATOMIC_LOAD. */
13916 gcc_assert (parent);
13917 gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD);
13920 parent = parent->outer;
13922 else if (code == GIMPLE_OMP_CONTINUE)
13924 gcc_assert (parent);
13927 else if (code == GIMPLE_OMP_SECTIONS_SWITCH)
13929 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
13930 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
13934 region = new_omp_region (bb, code, parent);
13936 if (code == GIMPLE_OMP_TARGET)
13938 switch (gimple_omp_target_kind (stmt))
13940 case GF_OMP_TARGET_KIND_REGION:
13941 case GF_OMP_TARGET_KIND_DATA:
13942 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
13943 case GF_OMP_TARGET_KIND_OACC_KERNELS:
13944 case GF_OMP_TARGET_KIND_OACC_DATA:
13945 case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
13947 case GF_OMP_TARGET_KIND_UPDATE:
13948 case GF_OMP_TARGET_KIND_ENTER_DATA:
13949 case GF_OMP_TARGET_KIND_EXIT_DATA:
13950 case GF_OMP_TARGET_KIND_OACC_UPDATE:
13951 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
13952 case GF_OMP_TARGET_KIND_OACC_DECLARE:
13953 /* ..., other than for those stand-alone directives... */
13957 gcc_unreachable ();
13960 else if (code == GIMPLE_OMP_ORDERED
13961 && find_omp_clause (gimple_omp_ordered_clauses
13962 (as_a <gomp_ordered *> (stmt)),
13963 OMP_CLAUSE_DEPEND))
13964 /* #pragma omp ordered depend is also just a stand-alone
13967 /* ..., this directive becomes the parent for a new region. */
13973 if (single_tree && !parent)
13976 for (son = first_dom_son (CDI_DOMINATORS, bb);
13978 son = next_dom_son (CDI_DOMINATORS, son))
13979 build_omp_regions_1 (son, parent, single_tree);
13982 /* Builds the tree of OMP regions rooted at ROOT, storing it to
13983 root_omp_region. */
13986 build_omp_regions_root (basic_block root)
13988 gcc_assert (root_omp_region == NULL);
13989 build_omp_regions_1 (root, NULL, true);
13990 gcc_assert (root_omp_region != NULL);
13993 /* Expands omp construct (and its subconstructs) starting in HEAD. */
13996 omp_expand_local (basic_block head)
13998 build_omp_regions_root (head);
13999 if (dump_file && (dump_flags & TDF_DETAILS))
14001 fprintf (dump_file, "\nOMP region tree\n\n");
14002 dump_omp_region (dump_file, root_omp_region, 0);
14003 fprintf (dump_file, "\n");
14006 remove_exit_barriers (root_omp_region);
14007 expand_omp (root_omp_region);
14009 free_omp_regions ();
14012 /* Scan the CFG and build a tree of OMP regions. Return the root of
14013 the OMP region tree. */
14016 build_omp_regions (void)
14018 gcc_assert (root_omp_region == NULL);
14019 calculate_dominance_info (CDI_DOMINATORS);
14020 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false);
14023 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
14025 static unsigned int
14026 execute_expand_omp (void)
14028 build_omp_regions ();
14030 if (!root_omp_region)
14035 fprintf (dump_file, "\nOMP region tree\n\n");
14036 dump_omp_region (dump_file, root_omp_region, 0);
14037 fprintf (dump_file, "\n");
14040 remove_exit_barriers (root_omp_region);
14042 expand_omp (root_omp_region);
14044 if (flag_checking && !loops_state_satisfies_p (LOOPS_NEED_FIXUP))
14045 verify_loop_structure ();
14046 cleanup_tree_cfg ();
14048 free_omp_regions ();
14053 /* OMP expansion -- the default pass, run before creation of SSA form. */
14057 const pass_data pass_data_expand_omp =
14059 GIMPLE_PASS, /* type */
14060 "ompexp", /* name */
14061 OPTGROUP_NONE, /* optinfo_flags */
14062 TV_NONE, /* tv_id */
14063 PROP_gimple_any, /* properties_required */
14064 PROP_gimple_eomp, /* properties_provided */
14065 0, /* properties_destroyed */
14066 0, /* todo_flags_start */
14067 0, /* todo_flags_finish */
14070 class pass_expand_omp : public gimple_opt_pass
14073 pass_expand_omp (gcc::context *ctxt)
14074 : gimple_opt_pass (pass_data_expand_omp, ctxt)
14077 /* opt_pass methods: */
14078 virtual unsigned int execute (function *)
14080 bool gate = ((flag_cilkplus != 0 || flag_openacc != 0 || flag_openmp != 0
14081 || flag_openmp_simd != 0)
14082 && !seen_error ());
14084 /* This pass always runs, to provide PROP_gimple_eomp.
14085 But often, there is nothing to do. */
14089 return execute_expand_omp ();
14092 }; // class pass_expand_omp
14094 } // anon namespace
14097 make_pass_expand_omp (gcc::context *ctxt)
14099 return new pass_expand_omp (ctxt);
14104 const pass_data pass_data_expand_omp_ssa =
14106 GIMPLE_PASS, /* type */
14107 "ompexpssa", /* name */
14108 OPTGROUP_NONE, /* optinfo_flags */
14109 TV_NONE, /* tv_id */
14110 PROP_cfg | PROP_ssa, /* properties_required */
14111 PROP_gimple_eomp, /* properties_provided */
14112 0, /* properties_destroyed */
14113 0, /* todo_flags_start */
14114 TODO_cleanup_cfg | TODO_rebuild_alias, /* todo_flags_finish */
14117 class pass_expand_omp_ssa : public gimple_opt_pass
14120 pass_expand_omp_ssa (gcc::context *ctxt)
14121 : gimple_opt_pass (pass_data_expand_omp_ssa, ctxt)
14124 /* opt_pass methods: */
14125 virtual bool gate (function *fun)
14127 return !(fun->curr_properties & PROP_gimple_eomp);
14129 virtual unsigned int execute (function *) { return execute_expand_omp (); }
14130 opt_pass * clone () { return new pass_expand_omp_ssa (m_ctxt); }
14132 }; // class pass_expand_omp_ssa
14134 } // anon namespace
14137 make_pass_expand_omp_ssa (gcc::context *ctxt)
14139 return new pass_expand_omp_ssa (ctxt);
14142 /* Routines to lower OMP directives into OMP-GIMPLE. */
14144 /* If ctx is a worksharing context inside of a cancellable parallel
14145 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
14146 and conditional branch to parallel's cancel_label to handle
14147 cancellation in the implicit barrier. */
14150 maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body)
14152 gimple *omp_return = gimple_seq_last_stmt (*body);
14153 gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
14154 if (gimple_omp_return_nowait_p (omp_return))
14157 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL
14158 && ctx->outer->cancellable)
14160 tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
14161 tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
14162 tree lhs = create_tmp_var (c_bool_type);
14163 gimple_omp_return_set_lhs (omp_return, lhs);
14164 tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
14165 gimple *g = gimple_build_cond (NE_EXPR, lhs,
14166 fold_convert (c_bool_type,
14167 boolean_false_node),
14168 ctx->outer->cancel_label, fallthru_label);
14169 gimple_seq_add_stmt (body, g);
14170 gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
14174 /* Lower the OpenMP sections directive in the current statement in GSI_P.
14175 CTX is the enclosing OMP context for the current statement. */
14178 lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14180 tree block, control;
14181 gimple_stmt_iterator tgsi;
14182 gomp_sections *stmt;
14184 gbind *new_stmt, *bind;
14185 gimple_seq ilist, dlist, olist, new_body;
14187 stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p));
14189 push_gimplify_context ();
14193 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
14194 &ilist, &dlist, ctx, NULL);
14196 new_body = gimple_omp_body (stmt);
14197 gimple_omp_set_body (stmt, NULL);
14198 tgsi = gsi_start (new_body);
14199 for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
14204 sec_start = gsi_stmt (tgsi);
14205 sctx = maybe_lookup_ctx (sec_start);
14208 lower_omp (gimple_omp_body_ptr (sec_start), sctx);
14209 gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
14210 GSI_CONTINUE_LINKING);
14211 gimple_omp_set_body (sec_start, NULL);
14213 if (gsi_one_before_end_p (tgsi))
14215 gimple_seq l = NULL;
14216 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
14218 gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
14219 gimple_omp_section_set_last (sec_start);
14222 gsi_insert_after (&tgsi, gimple_build_omp_return (false),
14223 GSI_CONTINUE_LINKING);
14226 block = make_node (BLOCK);
14227 bind = gimple_build_bind (NULL, new_body, block);
14230 lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx);
14232 block = make_node (BLOCK);
14233 new_stmt = gimple_build_bind (NULL, NULL, block);
14234 gsi_replace (gsi_p, new_stmt, true);
14236 pop_gimplify_context (new_stmt);
14237 gimple_bind_append_vars (new_stmt, ctx->block_vars);
14238 BLOCK_VARS (block) = gimple_bind_vars (bind);
14239 if (BLOCK_VARS (block))
14240 TREE_USED (block) = 1;
14243 gimple_seq_add_seq (&new_body, ilist);
14244 gimple_seq_add_stmt (&new_body, stmt);
14245 gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
14246 gimple_seq_add_stmt (&new_body, bind);
14248 control = create_tmp_var (unsigned_type_node, ".section");
14249 t = gimple_build_omp_continue (control, control);
14250 gimple_omp_sections_set_control (stmt, control);
14251 gimple_seq_add_stmt (&new_body, t);
14253 gimple_seq_add_seq (&new_body, olist);
14254 if (ctx->cancellable)
14255 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
14256 gimple_seq_add_seq (&new_body, dlist);
14258 new_body = maybe_catch_exception (new_body);
14260 t = gimple_build_omp_return
14261 (!!find_omp_clause (gimple_omp_sections_clauses (stmt),
14262 OMP_CLAUSE_NOWAIT));
14263 gimple_seq_add_stmt (&new_body, t);
14264 maybe_add_implicit_barrier_cancel (ctx, &new_body);
14266 gimple_bind_set_body (new_stmt, new_body);
14270 /* A subroutine of lower_omp_single. Expand the simple form of
14271 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
14273 if (GOMP_single_start ())
14275 [ GOMP_barrier (); ] -> unless 'nowait' is present.
14277 FIXME. It may be better to delay expanding the logic of this until
14278 pass_expand_omp. The expanded logic may make the job more difficult
14279 to a synchronization analysis pass. */
14282 lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p)
14284 location_t loc = gimple_location (single_stmt);
14285 tree tlabel = create_artificial_label (loc);
14286 tree flabel = create_artificial_label (loc);
14287 gimple *call, *cond;
14290 decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START);
14291 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
14292 call = gimple_build_call (decl, 0);
14293 gimple_call_set_lhs (call, lhs);
14294 gimple_seq_add_stmt (pre_p, call);
14296 cond = gimple_build_cond (EQ_EXPR, lhs,
14297 fold_convert_loc (loc, TREE_TYPE (lhs),
14298 boolean_true_node),
14300 gimple_seq_add_stmt (pre_p, cond);
14301 gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
14302 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
14303 gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
14307 /* A subroutine of lower_omp_single. Expand the simple form of
14308 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
14310 #pragma omp single copyprivate (a, b, c)
14312 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
14315 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
14321 GOMP_single_copy_end (©out);
14332 FIXME. It may be better to delay expanding the logic of this until
14333 pass_expand_omp. The expanded logic may make the job more difficult
14334 to a synchronization analysis pass. */
14337 lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p,
14340 tree ptr_type, t, l0, l1, l2, bfn_decl;
14341 gimple_seq copyin_seq;
14342 location_t loc = gimple_location (single_stmt);
14344 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
14346 ptr_type = build_pointer_type (ctx->record_type);
14347 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
14349 l0 = create_artificial_label (loc);
14350 l1 = create_artificial_label (loc);
14351 l2 = create_artificial_label (loc);
14353 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
14354 t = build_call_expr_loc (loc, bfn_decl, 0);
14355 t = fold_convert_loc (loc, ptr_type, t);
14356 gimplify_assign (ctx->receiver_decl, t, pre_p);
14358 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
14359 build_int_cst (ptr_type, 0));
14360 t = build3 (COND_EXPR, void_type_node, t,
14361 build_and_jump (&l0), build_and_jump (&l1));
14362 gimplify_and_add (t, pre_p);
14364 gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
14366 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
14369 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
14372 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
14373 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
14374 t = build_call_expr_loc (loc, bfn_decl, 1, t);
14375 gimplify_and_add (t, pre_p);
14377 t = build_and_jump (&l2);
14378 gimplify_and_add (t, pre_p);
14380 gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
14382 gimple_seq_add_seq (pre_p, copyin_seq);
14384 gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
14388 /* Expand code for an OpenMP single directive. */
14391 lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14395 gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p));
14397 gimple_seq bind_body, bind_body_tail = NULL, dlist;
14399 push_gimplify_context ();
14401 block = make_node (BLOCK);
14402 bind = gimple_build_bind (NULL, NULL, block);
14403 gsi_replace (gsi_p, bind, true);
14406 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
14407 &bind_body, &dlist, ctx, NULL);
14408 lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
14410 gimple_seq_add_stmt (&bind_body, single_stmt);
14412 if (ctx->record_type)
14413 lower_omp_single_copy (single_stmt, &bind_body, ctx);
14415 lower_omp_single_simple (single_stmt, &bind_body);
14417 gimple_omp_set_body (single_stmt, NULL);
14419 gimple_seq_add_seq (&bind_body, dlist);
14421 bind_body = maybe_catch_exception (bind_body);
14423 t = gimple_build_omp_return
14424 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt),
14425 OMP_CLAUSE_NOWAIT));
14426 gimple_seq_add_stmt (&bind_body_tail, t);
14427 maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail);
14428 if (ctx->record_type)
14430 gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
14431 tree clobber = build_constructor (ctx->record_type, NULL);
14432 TREE_THIS_VOLATILE (clobber) = 1;
14433 gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
14434 clobber), GSI_SAME_STMT);
14436 gimple_seq_add_seq (&bind_body, bind_body_tail);
14437 gimple_bind_set_body (bind, bind_body);
14439 pop_gimplify_context (bind);
14441 gimple_bind_append_vars (bind, ctx->block_vars);
14442 BLOCK_VARS (block) = ctx->block_vars;
14443 if (BLOCK_VARS (block))
14444 TREE_USED (block) = 1;
14448 /* Expand code for an OpenMP master directive. */
14451 lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14453 tree block, lab = NULL, x, bfn_decl;
14454 gimple *stmt = gsi_stmt (*gsi_p);
14456 location_t loc = gimple_location (stmt);
14459 push_gimplify_context ();
14461 block = make_node (BLOCK);
14462 bind = gimple_build_bind (NULL, NULL, block);
14463 gsi_replace (gsi_p, bind, true);
14464 gimple_bind_add_stmt (bind, stmt);
14466 bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
14467 x = build_call_expr_loc (loc, bfn_decl, 0);
14468 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node);
14469 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
14471 gimplify_and_add (x, &tseq);
14472 gimple_bind_add_seq (bind, tseq);
14474 lower_omp (gimple_omp_body_ptr (stmt), ctx);
14475 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
14476 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
14477 gimple_omp_set_body (stmt, NULL);
14479 gimple_bind_add_stmt (bind, gimple_build_label (lab));
14481 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
14483 pop_gimplify_context (bind);
14485 gimple_bind_append_vars (bind, ctx->block_vars);
14486 BLOCK_VARS (block) = ctx->block_vars;
14490 /* Expand code for an OpenMP taskgroup directive. */
14493 lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14495 gimple *stmt = gsi_stmt (*gsi_p);
14498 tree block = make_node (BLOCK);
14500 bind = gimple_build_bind (NULL, NULL, block);
14501 gsi_replace (gsi_p, bind, true);
14502 gimple_bind_add_stmt (bind, stmt);
14504 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
14506 gimple_bind_add_stmt (bind, x);
14508 lower_omp (gimple_omp_body_ptr (stmt), ctx);
14509 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
14510 gimple_omp_set_body (stmt, NULL);
14512 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
14514 gimple_bind_append_vars (bind, ctx->block_vars);
14515 BLOCK_VARS (block) = ctx->block_vars;
14519 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
14522 lower_omp_ordered_clauses (gimple_stmt_iterator *gsi_p, gomp_ordered *ord_stmt,
14525 struct omp_for_data fd;
14526 if (!ctx->outer || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR)
14529 unsigned int len = gimple_omp_for_collapse (ctx->outer->stmt);
14530 struct omp_for_data_loop *loops = XALLOCAVEC (struct omp_for_data_loop, len);
14531 extract_omp_for_data (as_a <gomp_for *> (ctx->outer->stmt), &fd, loops);
14535 tree *list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
14536 tree c = gimple_omp_ordered_clauses (ord_stmt);
14537 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
14538 && OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_SINK)
14540 /* Merge depend clauses from multiple adjacent
14541 #pragma omp ordered depend(sink:...) constructs
14542 into one #pragma omp ordered depend(sink:...), so that
14543 we can optimize them together. */
14544 gimple_stmt_iterator gsi = *gsi_p;
14546 while (!gsi_end_p (gsi))
14548 gimple *stmt = gsi_stmt (gsi);
14549 if (is_gimple_debug (stmt)
14550 || gimple_code (stmt) == GIMPLE_NOP)
14555 if (gimple_code (stmt) != GIMPLE_OMP_ORDERED)
14557 gomp_ordered *ord_stmt2 = as_a <gomp_ordered *> (stmt);
14558 c = gimple_omp_ordered_clauses (ord_stmt2);
14560 || OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND
14561 || OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_SINK)
14564 list_p = &OMP_CLAUSE_CHAIN (*list_p);
14566 gsi_remove (&gsi, true);
14570 /* Canonicalize sink dependence clauses into one folded clause if
14573 The basic algorithm is to create a sink vector whose first
14574 element is the GCD of all the first elements, and whose remaining
14575 elements are the minimum of the subsequent columns.
14577 We ignore dependence vectors whose first element is zero because
14578 such dependencies are known to be executed by the same thread.
14580 We take into account the direction of the loop, so a minimum
14581 becomes a maximum if the loop is iterating forwards. We also
14582 ignore sink clauses where the loop direction is unknown, or where
14583 the offsets are clearly invalid because they are not a multiple
14584 of the loop increment.
14588 #pragma omp for ordered(2)
14589 for (i=0; i < N; ++i)
14590 for (j=0; j < M; ++j)
14592 #pragma omp ordered \
14593 depend(sink:i-8,j-2) \
14594 depend(sink:i,j-1) \ // Completely ignored because i+0.
14595 depend(sink:i-4,j-3) \
14596 depend(sink:i-6,j-4)
14597 #pragma omp ordered depend(source)
14602 depend(sink:-gcd(8,4,6),-min(2,3,4))
14607 /* FIXME: Computing GCD's where the first element is zero is
14608 non-trivial in the presence of collapsed loops. Do this later. */
14609 if (fd.collapse > 1)
14612 wide_int *folded_deps = XALLOCAVEC (wide_int, 2 * len - 1);
14613 memset (folded_deps, 0, sizeof (*folded_deps) * (2 * len - 1));
14614 tree folded_dep = NULL_TREE;
14615 /* TRUE if the first dimension's offset is negative. */
14616 bool neg_offset_p = false;
14618 list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
14620 while ((c = *list_p) != NULL)
14622 bool remove = false;
14624 gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND);
14625 if (OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_SINK)
14626 goto next_ordered_clause;
14629 for (vec = OMP_CLAUSE_DECL (c), i = 0;
14630 vec && TREE_CODE (vec) == TREE_LIST;
14631 vec = TREE_CHAIN (vec), ++i)
14633 gcc_assert (i < len);
14635 /* extract_omp_for_data has canonicalized the condition. */
14636 gcc_assert (fd.loops[i].cond_code == LT_EXPR
14637 || fd.loops[i].cond_code == GT_EXPR);
14638 bool forward = fd.loops[i].cond_code == LT_EXPR;
14639 bool maybe_lexically_later = true;
14641 /* While the committee makes up its mind, bail if we have any
14642 non-constant steps. */
14643 if (TREE_CODE (fd.loops[i].step) != INTEGER_CST)
14644 goto lower_omp_ordered_ret;
14646 tree itype = TREE_TYPE (TREE_VALUE (vec));
14647 if (POINTER_TYPE_P (itype))
14649 wide_int offset = wide_int::from (TREE_PURPOSE (vec),
14650 TYPE_PRECISION (itype),
14651 TYPE_SIGN (itype));
14653 /* Ignore invalid offsets that are not multiples of the step. */
14654 if (!wi::multiple_of_p
14655 (wi::abs (offset), wi::abs ((wide_int) fd.loops[i].step),
14658 warning_at (OMP_CLAUSE_LOCATION (c), 0,
14659 "ignoring sink clause with offset that is not "
14660 "a multiple of the loop step");
14662 goto next_ordered_clause;
14665 /* Calculate the first dimension. The first dimension of
14666 the folded dependency vector is the GCD of the first
14667 elements, while ignoring any first elements whose offset
14671 /* Ignore dependence vectors whose first dimension is 0. */
14675 goto next_ordered_clause;
14679 if (!TYPE_UNSIGNED (itype) && (forward ^ wi::neg_p (offset)))
14681 error_at (OMP_CLAUSE_LOCATION (c),
14682 "first offset must be in opposite direction "
14683 "of loop iterations");
14684 goto lower_omp_ordered_ret;
14688 neg_offset_p = forward;
14689 /* Initialize the first time around. */
14690 if (folded_dep == NULL_TREE)
14693 folded_deps[0] = offset;
14696 folded_deps[0] = wi::gcd (folded_deps[0],
14700 /* Calculate minimum for the remaining dimensions. */
14703 folded_deps[len + i - 1] = offset;
14704 if (folded_dep == c)
14705 folded_deps[i] = offset;
14706 else if (maybe_lexically_later
14707 && !wi::eq_p (folded_deps[i], offset))
14709 if (forward ^ wi::gts_p (folded_deps[i], offset))
14713 for (j = 1; j <= i; j++)
14714 folded_deps[j] = folded_deps[len + j - 1];
14717 maybe_lexically_later = false;
14721 gcc_assert (i == len);
14725 next_ordered_clause:
14727 *list_p = OMP_CLAUSE_CHAIN (c);
14729 list_p = &OMP_CLAUSE_CHAIN (c);
14735 folded_deps[0] = -folded_deps[0];
14737 tree itype = TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep)));
14738 if (POINTER_TYPE_P (itype))
14741 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep))
14742 = wide_int_to_tree (itype, folded_deps[0]);
14743 OMP_CLAUSE_CHAIN (folded_dep) = gimple_omp_ordered_clauses (ord_stmt);
14744 *gimple_omp_ordered_clauses_ptr (ord_stmt) = folded_dep;
14747 lower_omp_ordered_ret:
14749 /* Ordered without clauses is #pragma omp threads, while we want
14750 a nop instead if we remove all clauses. */
14751 if (gimple_omp_ordered_clauses (ord_stmt) == NULL_TREE)
14752 gsi_replace (gsi_p, gimple_build_nop (), true);
14756 /* Expand code for an OpenMP ordered directive. */
14759 lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14762 gimple *stmt = gsi_stmt (*gsi_p);
14763 gomp_ordered *ord_stmt = as_a <gomp_ordered *> (stmt);
14766 bool simd = find_omp_clause (gimple_omp_ordered_clauses (ord_stmt),
14768 bool threads = find_omp_clause (gimple_omp_ordered_clauses (ord_stmt),
14769 OMP_CLAUSE_THREADS);
14771 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt),
14772 OMP_CLAUSE_DEPEND))
14774 /* FIXME: This is needs to be moved to the expansion to verify various
14775 conditions only testable on cfg with dominators computed, and also
14776 all the depend clauses to be merged still might need to be available
14777 for the runtime checks. */
14779 lower_omp_ordered_clauses (gsi_p, ord_stmt, ctx);
14783 push_gimplify_context ();
14785 block = make_node (BLOCK);
14786 bind = gimple_build_bind (NULL, NULL, block);
14787 gsi_replace (gsi_p, bind, true);
14788 gimple_bind_add_stmt (bind, stmt);
14792 x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START, 1,
14793 build_int_cst (NULL_TREE, threads));
14794 cfun->has_simduid_loops = true;
14797 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
14799 gimple_bind_add_stmt (bind, x);
14801 lower_omp (gimple_omp_body_ptr (stmt), ctx);
14802 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
14803 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
14804 gimple_omp_set_body (stmt, NULL);
14807 x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END, 1,
14808 build_int_cst (NULL_TREE, threads));
14810 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END),
14812 gimple_bind_add_stmt (bind, x);
14814 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
14816 pop_gimplify_context (bind);
14818 gimple_bind_append_vars (bind, ctx->block_vars);
14819 BLOCK_VARS (block) = gimple_bind_vars (bind);
14823 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
14824 substitution of a couple of function calls. But in the NAMED case,
14825 requires that languages coordinate a symbol name. It is therefore
14826 best put here in common code. */
14828 static GTY(()) hash_map<tree, tree> *critical_name_mutexes;
14831 lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14834 tree name, lock, unlock;
14835 gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p));
14837 location_t loc = gimple_location (stmt);
14840 name = gimple_omp_critical_name (stmt);
14845 if (!critical_name_mutexes)
14846 critical_name_mutexes = hash_map<tree, tree>::create_ggc (10);
14848 tree *n = critical_name_mutexes->get (name);
14853 decl = create_tmp_var_raw (ptr_type_node);
14855 new_str = ACONCAT ((".gomp_critical_user_",
14856 IDENTIFIER_POINTER (name), NULL));
14857 DECL_NAME (decl) = get_identifier (new_str);
14858 TREE_PUBLIC (decl) = 1;
14859 TREE_STATIC (decl) = 1;
14860 DECL_COMMON (decl) = 1;
14861 DECL_ARTIFICIAL (decl) = 1;
14862 DECL_IGNORED_P (decl) = 1;
14864 varpool_node::finalize_decl (decl);
14866 critical_name_mutexes->put (name, decl);
14871 /* If '#pragma omp critical' is inside offloaded region or
14872 inside function marked as offloadable, the symbol must be
14873 marked as offloadable too. */
14875 if (cgraph_node::get (current_function_decl)->offloadable)
14876 varpool_node::get_create (decl)->offloadable = 1;
14878 for (octx = ctx->outer; octx; octx = octx->outer)
14879 if (is_gimple_omp_offloaded (octx->stmt))
14881 varpool_node::get_create (decl)->offloadable = 1;
14885 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
14886 lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl));
14888 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
14889 unlock = build_call_expr_loc (loc, unlock, 1,
14890 build_fold_addr_expr_loc (loc, decl));
14894 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
14895 lock = build_call_expr_loc (loc, lock, 0);
14897 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
14898 unlock = build_call_expr_loc (loc, unlock, 0);
14901 push_gimplify_context ();
14903 block = make_node (BLOCK);
14904 bind = gimple_build_bind (NULL, NULL, block);
14905 gsi_replace (gsi_p, bind, true);
14906 gimple_bind_add_stmt (bind, stmt);
14908 tbody = gimple_bind_body (bind);
14909 gimplify_and_add (lock, &tbody);
14910 gimple_bind_set_body (bind, tbody);
14912 lower_omp (gimple_omp_body_ptr (stmt), ctx);
14913 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
14914 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
14915 gimple_omp_set_body (stmt, NULL);
14917 tbody = gimple_bind_body (bind);
14918 gimplify_and_add (unlock, &tbody);
14919 gimple_bind_set_body (bind, tbody);
14921 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
14923 pop_gimplify_context (bind);
14924 gimple_bind_append_vars (bind, ctx->block_vars);
14925 BLOCK_VARS (block) = gimple_bind_vars (bind);
14929 /* A subroutine of lower_omp_for. Generate code to emit the predicate
14930 for a lastprivate clause. Given a loop control predicate of (V
14931 cond N2), we gate the clause on (!(V cond N2)). The lowered form
14932 is appended to *DLIST, iterator initialization is appended to
14936 lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
14937 gimple_seq *dlist, struct omp_context *ctx)
14939 tree clauses, cond, vinit;
14940 enum tree_code cond_code;
14943 cond_code = fd->loop.cond_code;
14944 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
14946 /* When possible, use a strict equality expression. This can let VRP
14947 type optimizations deduce the value and remove a copy. */
14948 if (tree_fits_shwi_p (fd->loop.step))
14950 HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
14951 if (step == 1 || step == -1)
14952 cond_code = EQ_EXPR;
14955 tree n2 = fd->loop.n2;
14956 if (fd->collapse > 1
14957 && TREE_CODE (n2) != INTEGER_CST
14958 && gimple_omp_for_combined_into_p (fd->for_stmt))
14960 struct omp_context *taskreg_ctx = NULL;
14961 if (gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
14963 gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt);
14964 if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR
14965 || gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_DISTRIBUTE)
14967 if (gimple_omp_for_combined_into_p (gfor))
14969 gcc_assert (ctx->outer->outer
14970 && is_parallel_ctx (ctx->outer->outer));
14971 taskreg_ctx = ctx->outer->outer;
14975 struct omp_for_data outer_fd;
14976 extract_omp_for_data (gfor, &outer_fd, NULL);
14977 n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2);
14980 else if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_TASKLOOP)
14981 taskreg_ctx = ctx->outer->outer;
14983 else if (is_taskreg_ctx (ctx->outer))
14984 taskreg_ctx = ctx->outer;
14989 = find_omp_clause (gimple_omp_taskreg_clauses (taskreg_ctx->stmt),
14990 OMP_CLAUSE__LOOPTEMP_);
14991 gcc_assert (innerc);
14992 for (i = 0; i < fd->collapse; i++)
14994 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
14995 OMP_CLAUSE__LOOPTEMP_);
14996 gcc_assert (innerc);
14998 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
14999 OMP_CLAUSE__LOOPTEMP_);
15001 n2 = fold_convert (TREE_TYPE (n2),
15002 lookup_decl (OMP_CLAUSE_DECL (innerc),
15006 cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2);
15008 clauses = gimple_omp_for_clauses (fd->for_stmt);
15010 lower_lastprivate_clauses (clauses, cond, &stmts, ctx);
15011 if (!gimple_seq_empty_p (stmts))
15013 gimple_seq_add_seq (&stmts, *dlist);
15016 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
15017 vinit = fd->loop.n1;
15018 if (cond_code == EQ_EXPR
15019 && tree_fits_shwi_p (fd->loop.n2)
15020 && ! integer_zerop (fd->loop.n2))
15021 vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
15023 vinit = unshare_expr (vinit);
15025 /* Initialize the iterator variable, so that threads that don't execute
15026 any iterations don't execute the lastprivate clauses by accident. */
15027 gimplify_assign (fd->loop.v, vinit, body_p);
15032 /* Lower code for an OMP loop directive. */
15035 lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15037 tree *rhs_p, block;
15038 struct omp_for_data fd, *fdp = NULL;
15039 gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
15041 gimple_seq omp_for_body, body, dlist;
15042 gimple_seq oacc_head = NULL, oacc_tail = NULL;
15045 push_gimplify_context ();
15047 lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
15049 block = make_node (BLOCK);
15050 new_stmt = gimple_build_bind (NULL, NULL, block);
15051 /* Replace at gsi right away, so that 'stmt' is no member
15052 of a sequence anymore as we're going to add to a different
15054 gsi_replace (gsi_p, new_stmt, true);
15056 /* Move declaration of temporaries in the loop body before we make
15058 omp_for_body = gimple_omp_body (stmt);
15059 if (!gimple_seq_empty_p (omp_for_body)
15060 && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
15063 = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body));
15064 tree vars = gimple_bind_vars (inner_bind);
15065 gimple_bind_append_vars (new_stmt, vars);
15066 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
15067 keep them on the inner_bind and it's block. */
15068 gimple_bind_set_vars (inner_bind, NULL_TREE);
15069 if (gimple_bind_block (inner_bind))
15070 BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
15073 if (gimple_omp_for_combined_into_p (stmt))
15075 extract_omp_for_data (stmt, &fd, NULL);
15078 /* We need two temporaries with fd.loop.v type (istart/iend)
15079 and then (fd.collapse - 1) temporaries with the same
15080 type for count2 ... countN-1 vars if not constant. */
15082 tree type = fd.iter_type;
15083 if (fd.collapse > 1
15084 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
15085 count += fd.collapse - 1;
15087 = (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
15088 || gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP);
15089 tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
15090 tree clauses = *pc;
15093 = find_omp_clause (gimple_omp_taskreg_clauses (ctx->outer->stmt),
15094 OMP_CLAUSE__LOOPTEMP_);
15095 for (i = 0; i < count; i++)
15100 gcc_assert (outerc);
15101 temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
15102 outerc = find_omp_clause (OMP_CLAUSE_CHAIN (outerc),
15103 OMP_CLAUSE__LOOPTEMP_);
15107 temp = create_tmp_var (type);
15108 insert_decl_map (&ctx->outer->cb, temp, temp);
15110 *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
15111 OMP_CLAUSE_DECL (*pc) = temp;
15112 pc = &OMP_CLAUSE_CHAIN (*pc);
15117 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
15120 lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
15122 gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt));
15124 lower_omp (gimple_omp_body_ptr (stmt), ctx);
15126 /* Lower the header expressions. At this point, we can assume that
15127 the header is of the form:
15129 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
15131 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
15132 using the .omp_data_s mapping, if needed. */
15133 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
15135 rhs_p = gimple_omp_for_initial_ptr (stmt, i);
15136 if (!is_gimple_min_invariant (*rhs_p))
15137 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
15139 rhs_p = gimple_omp_for_final_ptr (stmt, i);
15140 if (!is_gimple_min_invariant (*rhs_p))
15141 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
15143 rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
15144 if (!is_gimple_min_invariant (*rhs_p))
15145 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
15148 /* Once lowered, extract the bounds and clauses. */
15149 extract_omp_for_data (stmt, &fd, NULL);
15151 if (is_gimple_omp_oacc (ctx->stmt)
15152 && !ctx_in_oacc_kernels_region (ctx))
15153 lower_oacc_head_tail (gimple_location (stmt),
15154 gimple_omp_for_clauses (stmt),
15155 &oacc_head, &oacc_tail, ctx);
15157 /* Add OpenACC partitioning and reduction markers just before the loop */
15159 gimple_seq_add_seq (&body, oacc_head);
15161 lower_omp_for_lastprivate (&fd, &body, &dlist, ctx);
15163 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
15164 for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
15165 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
15166 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
15168 OMP_CLAUSE_DECL (c) = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
15169 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c)))
15170 OMP_CLAUSE_LINEAR_STEP (c)
15171 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c),
15175 if (!gimple_omp_for_grid_phony (stmt))
15176 gimple_seq_add_stmt (&body, stmt);
15177 gimple_seq_add_seq (&body, gimple_omp_body (stmt));
15179 if (!gimple_omp_for_grid_phony (stmt))
15180 gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
15183 /* After the loop, add exit clauses. */
15184 lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx);
15186 if (ctx->cancellable)
15187 gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
15189 gimple_seq_add_seq (&body, dlist);
15191 body = maybe_catch_exception (body);
15193 if (!gimple_omp_for_grid_phony (stmt))
15195 /* Region exit marker goes at the end of the loop body. */
15196 gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait));
15197 maybe_add_implicit_barrier_cancel (ctx, &body);
15200 /* Add OpenACC joining and reduction markers just after the loop. */
15202 gimple_seq_add_seq (&body, oacc_tail);
15204 pop_gimplify_context (new_stmt);
15206 gimple_bind_append_vars (new_stmt, ctx->block_vars);
15207 BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
15208 if (BLOCK_VARS (block))
15209 TREE_USED (block) = 1;
15211 gimple_bind_set_body (new_stmt, body);
15212 gimple_omp_set_body (stmt, NULL);
15213 gimple_omp_for_set_pre_body (stmt, NULL);
15216 /* Callback for walk_stmts. Check if the current statement only contains
15217 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
15220 check_combined_parallel (gimple_stmt_iterator *gsi_p,
15221 bool *handled_ops_p,
15222 struct walk_stmt_info *wi)
15224 int *info = (int *) wi->info;
15225 gimple *stmt = gsi_stmt (*gsi_p);
15227 *handled_ops_p = true;
15228 switch (gimple_code (stmt))
15232 case GIMPLE_OMP_FOR:
15233 case GIMPLE_OMP_SECTIONS:
15234 *info = *info == 0 ? 1 : -1;
15243 struct omp_taskcopy_context
15245 /* This field must be at the beginning, as we do "inheritance": Some
15246 callback functions for tree-inline.c (e.g., omp_copy_decl)
15247 receive a copy_body_data pointer that is up-casted to an
15248 omp_context pointer. */
15254 task_copyfn_copy_decl (tree var, copy_body_data *cb)
15256 struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
15258 if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
15259 return create_tmp_var (TREE_TYPE (var));
15265 task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
15267 tree name, new_fields = NULL, type, f;
15269 type = lang_hooks.types.make_type (RECORD_TYPE);
15270 name = DECL_NAME (TYPE_NAME (orig_type));
15271 name = build_decl (gimple_location (tcctx->ctx->stmt),
15272 TYPE_DECL, name, type);
15273 TYPE_NAME (type) = name;
15275 for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
15277 tree new_f = copy_node (f);
15278 DECL_CONTEXT (new_f) = type;
15279 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
15280 TREE_CHAIN (new_f) = new_fields;
15281 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
15282 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
15283 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
15285 new_fields = new_f;
15286 tcctx->cb.decl_map->put (f, new_f);
15288 TYPE_FIELDS (type) = nreverse (new_fields);
15289 layout_type (type);
15293 /* Create task copyfn. */
15296 create_task_copyfn (gomp_task *task_stmt, omp_context *ctx)
15298 struct function *child_cfun;
15299 tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
15300 tree record_type, srecord_type, bind, list;
15301 bool record_needs_remap = false, srecord_needs_remap = false;
15303 struct omp_taskcopy_context tcctx;
15304 location_t loc = gimple_location (task_stmt);
15306 child_fn = gimple_omp_task_copy_fn (task_stmt);
15307 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
15308 gcc_assert (child_cfun->cfg == NULL);
15309 DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
15311 /* Reset DECL_CONTEXT on function arguments. */
15312 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
15313 DECL_CONTEXT (t) = child_fn;
15315 /* Populate the function. */
15316 push_gimplify_context ();
15317 push_cfun (child_cfun);
15319 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
15320 TREE_SIDE_EFFECTS (bind) = 1;
15322 DECL_SAVED_TREE (child_fn) = bind;
15323 DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
15325 /* Remap src and dst argument types if needed. */
15326 record_type = ctx->record_type;
15327 srecord_type = ctx->srecord_type;
15328 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
15329 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
15331 record_needs_remap = true;
15334 for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
15335 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
15337 srecord_needs_remap = true;
15341 if (record_needs_remap || srecord_needs_remap)
15343 memset (&tcctx, '\0', sizeof (tcctx));
15344 tcctx.cb.src_fn = ctx->cb.src_fn;
15345 tcctx.cb.dst_fn = child_fn;
15346 tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn);
15347 gcc_checking_assert (tcctx.cb.src_node);
15348 tcctx.cb.dst_node = tcctx.cb.src_node;
15349 tcctx.cb.src_cfun = ctx->cb.src_cfun;
15350 tcctx.cb.copy_decl = task_copyfn_copy_decl;
15351 tcctx.cb.eh_lp_nr = 0;
15352 tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
15353 tcctx.cb.decl_map = new hash_map<tree, tree>;
15356 if (record_needs_remap)
15357 record_type = task_copyfn_remap_type (&tcctx, record_type);
15358 if (srecord_needs_remap)
15359 srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
15362 tcctx.cb.decl_map = NULL;
15364 arg = DECL_ARGUMENTS (child_fn);
15365 TREE_TYPE (arg) = build_pointer_type (record_type);
15366 sarg = DECL_CHAIN (arg);
15367 TREE_TYPE (sarg) = build_pointer_type (srecord_type);
15369 /* First pass: initialize temporaries used in record_type and srecord_type
15370 sizes and field offsets. */
15371 if (tcctx.cb.decl_map)
15372 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
15373 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
15377 decl = OMP_CLAUSE_DECL (c);
15378 p = tcctx.cb.decl_map->get (decl);
15381 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
15382 sf = (tree) n->value;
15383 sf = *tcctx.cb.decl_map->get (sf);
15384 src = build_simple_mem_ref_loc (loc, sarg);
15385 src = omp_build_component_ref (src, sf);
15386 t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
15387 append_to_statement_list (t, &list);
15390 /* Second pass: copy shared var pointers and copy construct non-VLA
15391 firstprivate vars. */
15392 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
15393 switch (OMP_CLAUSE_CODE (c))
15395 splay_tree_key key;
15396 case OMP_CLAUSE_SHARED:
15397 decl = OMP_CLAUSE_DECL (c);
15398 key = (splay_tree_key) decl;
15399 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
15400 key = (splay_tree_key) &DECL_UID (decl);
15401 n = splay_tree_lookup (ctx->field_map, key);
15404 f = (tree) n->value;
15405 if (tcctx.cb.decl_map)
15406 f = *tcctx.cb.decl_map->get (f);
15407 n = splay_tree_lookup (ctx->sfield_map, key);
15408 sf = (tree) n->value;
15409 if (tcctx.cb.decl_map)
15410 sf = *tcctx.cb.decl_map->get (sf);
15411 src = build_simple_mem_ref_loc (loc, sarg);
15412 src = omp_build_component_ref (src, sf);
15413 dst = build_simple_mem_ref_loc (loc, arg);
15414 dst = omp_build_component_ref (dst, f);
15415 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
15416 append_to_statement_list (t, &list);
15418 case OMP_CLAUSE_FIRSTPRIVATE:
15419 decl = OMP_CLAUSE_DECL (c);
15420 if (is_variable_sized (decl))
15422 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
15425 f = (tree) n->value;
15426 if (tcctx.cb.decl_map)
15427 f = *tcctx.cb.decl_map->get (f);
15428 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
15431 sf = (tree) n->value;
15432 if (tcctx.cb.decl_map)
15433 sf = *tcctx.cb.decl_map->get (sf);
15434 src = build_simple_mem_ref_loc (loc, sarg);
15435 src = omp_build_component_ref (src, sf);
15436 if (use_pointer_for_field (decl, NULL) || is_reference (decl))
15437 src = build_simple_mem_ref_loc (loc, src);
15441 dst = build_simple_mem_ref_loc (loc, arg);
15442 dst = omp_build_component_ref (dst, f);
15443 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
15444 append_to_statement_list (t, &list);
15446 case OMP_CLAUSE_PRIVATE:
15447 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
15449 decl = OMP_CLAUSE_DECL (c);
15450 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
15451 f = (tree) n->value;
15452 if (tcctx.cb.decl_map)
15453 f = *tcctx.cb.decl_map->get (f);
15454 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
15457 sf = (tree) n->value;
15458 if (tcctx.cb.decl_map)
15459 sf = *tcctx.cb.decl_map->get (sf);
15460 src = build_simple_mem_ref_loc (loc, sarg);
15461 src = omp_build_component_ref (src, sf);
15462 if (use_pointer_for_field (decl, NULL))
15463 src = build_simple_mem_ref_loc (loc, src);
15467 dst = build_simple_mem_ref_loc (loc, arg);
15468 dst = omp_build_component_ref (dst, f);
15469 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
15470 append_to_statement_list (t, &list);
15476 /* Last pass: handle VLA firstprivates. */
15477 if (tcctx.cb.decl_map)
15478 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
15479 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
15483 decl = OMP_CLAUSE_DECL (c);
15484 if (!is_variable_sized (decl))
15486 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
15489 f = (tree) n->value;
15490 f = *tcctx.cb.decl_map->get (f);
15491 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
15492 ind = DECL_VALUE_EXPR (decl);
15493 gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
15494 gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
15495 n = splay_tree_lookup (ctx->sfield_map,
15496 (splay_tree_key) TREE_OPERAND (ind, 0));
15497 sf = (tree) n->value;
15498 sf = *tcctx.cb.decl_map->get (sf);
15499 src = build_simple_mem_ref_loc (loc, sarg);
15500 src = omp_build_component_ref (src, sf);
15501 src = build_simple_mem_ref_loc (loc, src);
15502 dst = build_simple_mem_ref_loc (loc, arg);
15503 dst = omp_build_component_ref (dst, f);
15504 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
15505 append_to_statement_list (t, &list);
15506 n = splay_tree_lookup (ctx->field_map,
15507 (splay_tree_key) TREE_OPERAND (ind, 0));
15508 df = (tree) n->value;
15509 df = *tcctx.cb.decl_map->get (df);
15510 ptr = build_simple_mem_ref_loc (loc, arg);
15511 ptr = omp_build_component_ref (ptr, df);
15512 t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
15513 build_fold_addr_expr_loc (loc, dst));
15514 append_to_statement_list (t, &list);
15517 t = build1 (RETURN_EXPR, void_type_node, NULL);
15518 append_to_statement_list (t, &list);
15520 if (tcctx.cb.decl_map)
15521 delete tcctx.cb.decl_map;
15522 pop_gimplify_context (NULL);
15523 BIND_EXPR_BODY (bind) = list;
15528 lower_depend_clauses (tree *pclauses, gimple_seq *iseq, gimple_seq *oseq)
15532 size_t n_in = 0, n_out = 0, idx = 2, i;
15534 clauses = find_omp_clause (*pclauses, OMP_CLAUSE_DEPEND);
15535 gcc_assert (clauses);
15536 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
15537 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
15538 switch (OMP_CLAUSE_DEPEND_KIND (c))
15540 case OMP_CLAUSE_DEPEND_IN:
15543 case OMP_CLAUSE_DEPEND_OUT:
15544 case OMP_CLAUSE_DEPEND_INOUT:
15547 case OMP_CLAUSE_DEPEND_SOURCE:
15548 case OMP_CLAUSE_DEPEND_SINK:
15551 gcc_unreachable ();
15553 tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2);
15554 tree array = create_tmp_var (type);
15555 TREE_ADDRESSABLE (array) = 1;
15556 tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
15558 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out));
15559 gimple_seq_add_stmt (iseq, g);
15560 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
15562 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out));
15563 gimple_seq_add_stmt (iseq, g);
15564 for (i = 0; i < 2; i++)
15566 if ((i ? n_in : n_out) == 0)
15568 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
15569 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
15570 && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i))
15572 tree t = OMP_CLAUSE_DECL (c);
15573 t = fold_convert (ptr_type_node, t);
15574 gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
15575 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
15576 NULL_TREE, NULL_TREE);
15577 g = gimple_build_assign (r, t);
15578 gimple_seq_add_stmt (iseq, g);
15581 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
15582 OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
15583 OMP_CLAUSE_CHAIN (c) = *pclauses;
15585 tree clobber = build_constructor (type, NULL);
15586 TREE_THIS_VOLATILE (clobber) = 1;
15587 g = gimple_build_assign (array, clobber);
15588 gimple_seq_add_stmt (oseq, g);
15591 /* Lower the OpenMP parallel or task directive in the current statement
15592 in GSI_P. CTX holds context information for the directive. */
15595 lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15599 gimple *stmt = gsi_stmt (*gsi_p);
15600 gbind *par_bind, *bind, *dep_bind = NULL;
15601 gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body;
15602 location_t loc = gimple_location (stmt);
15604 clauses = gimple_omp_taskreg_clauses (stmt);
15606 = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
15607 par_body = gimple_bind_body (par_bind);
15608 child_fn = ctx->cb.dst_fn;
15609 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
15610 && !gimple_omp_parallel_combined_p (stmt))
15612 struct walk_stmt_info wi;
15615 memset (&wi, 0, sizeof (wi));
15617 wi.val_only = true;
15618 walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
15620 gimple_omp_parallel_set_combined_p (stmt, true);
15622 gimple_seq dep_ilist = NULL;
15623 gimple_seq dep_olist = NULL;
15624 if (gimple_code (stmt) == GIMPLE_OMP_TASK
15625 && find_omp_clause (clauses, OMP_CLAUSE_DEPEND))
15627 push_gimplify_context ();
15628 dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
15629 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt),
15630 &dep_ilist, &dep_olist);
15633 if (ctx->srecord_type)
15634 create_task_copyfn (as_a <gomp_task *> (stmt), ctx);
15636 push_gimplify_context ();
15641 bool phony_construct = gimple_code (stmt) == GIMPLE_OMP_PARALLEL
15642 && gimple_omp_parallel_grid_phony (as_a <gomp_parallel *> (stmt));
15643 if (phony_construct && ctx->record_type)
15645 gcc_checking_assert (!ctx->receiver_decl);
15646 ctx->receiver_decl = create_tmp_var
15647 (build_reference_type (ctx->record_type), ".omp_rec");
15649 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
15650 lower_omp (&par_body, ctx);
15651 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL)
15652 lower_reduction_clauses (clauses, &par_rlist, ctx);
15654 /* Declare all the variables created by mapping and the variables
15655 declared in the scope of the parallel body. */
15656 record_vars_into (ctx->block_vars, child_fn);
15657 record_vars_into (gimple_bind_vars (par_bind), child_fn);
15659 if (ctx->record_type)
15662 = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
15663 : ctx->record_type, ".omp_data_o");
15664 DECL_NAMELESS (ctx->sender_decl) = 1;
15665 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
15666 gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
15671 lower_send_clauses (clauses, &ilist, &olist, ctx);
15672 lower_send_shared_vars (&ilist, &olist, ctx);
15674 if (ctx->record_type)
15676 tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL);
15677 TREE_THIS_VOLATILE (clobber) = 1;
15678 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
15682 /* Once all the expansions are done, sequence all the different
15683 fragments inside gimple_omp_body. */
15687 if (ctx->record_type)
15689 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
15690 /* fixup_child_record_type might have changed receiver_decl's type. */
15691 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
15692 gimple_seq_add_stmt (&new_body,
15693 gimple_build_assign (ctx->receiver_decl, t));
15696 gimple_seq_add_seq (&new_body, par_ilist);
15697 gimple_seq_add_seq (&new_body, par_body);
15698 gimple_seq_add_seq (&new_body, par_rlist);
15699 if (ctx->cancellable)
15700 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
15701 gimple_seq_add_seq (&new_body, par_olist);
15702 new_body = maybe_catch_exception (new_body);
15703 if (gimple_code (stmt) == GIMPLE_OMP_TASK)
15704 gimple_seq_add_stmt (&new_body,
15705 gimple_build_omp_continue (integer_zero_node,
15706 integer_zero_node));
15707 if (!phony_construct)
15709 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
15710 gimple_omp_set_body (stmt, new_body);
15713 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
15714 gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
15715 gimple_bind_add_seq (bind, ilist);
15716 if (!phony_construct)
15717 gimple_bind_add_stmt (bind, stmt);
15719 gimple_bind_add_seq (bind, new_body);
15720 gimple_bind_add_seq (bind, olist);
15722 pop_gimplify_context (NULL);
15726 gimple_bind_add_seq (dep_bind, dep_ilist);
15727 gimple_bind_add_stmt (dep_bind, bind);
15728 gimple_bind_add_seq (dep_bind, dep_olist);
15729 pop_gimplify_context (dep_bind);
15733 /* Lower the GIMPLE_OMP_TARGET in the current statement
15734 in GSI_P. CTX holds context information for the directive. */
15737 lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15740 tree child_fn, t, c;
15741 gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p));
15742 gbind *tgt_bind, *bind, *dep_bind = NULL;
15743 gimple_seq tgt_body, olist, ilist, fplist, new_body;
15744 location_t loc = gimple_location (stmt);
15745 bool offloaded, data_region;
15746 unsigned int map_cnt = 0;
15748 offloaded = is_gimple_omp_offloaded (stmt);
15749 switch (gimple_omp_target_kind (stmt))
15751 case GF_OMP_TARGET_KIND_REGION:
15752 case GF_OMP_TARGET_KIND_UPDATE:
15753 case GF_OMP_TARGET_KIND_ENTER_DATA:
15754 case GF_OMP_TARGET_KIND_EXIT_DATA:
15755 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
15756 case GF_OMP_TARGET_KIND_OACC_KERNELS:
15757 case GF_OMP_TARGET_KIND_OACC_UPDATE:
15758 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
15759 case GF_OMP_TARGET_KIND_OACC_DECLARE:
15760 data_region = false;
15762 case GF_OMP_TARGET_KIND_DATA:
15763 case GF_OMP_TARGET_KIND_OACC_DATA:
15764 case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
15765 data_region = true;
15768 gcc_unreachable ();
15771 clauses = gimple_omp_target_clauses (stmt);
15773 gimple_seq dep_ilist = NULL;
15774 gimple_seq dep_olist = NULL;
15775 if (find_omp_clause (clauses, OMP_CLAUSE_DEPEND))
15777 push_gimplify_context ();
15778 dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
15779 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt),
15780 &dep_ilist, &dep_olist);
15787 tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt));
15788 tgt_body = gimple_bind_body (tgt_bind);
15790 else if (data_region)
15791 tgt_body = gimple_omp_body (stmt);
15792 child_fn = ctx->cb.dst_fn;
15794 push_gimplify_context ();
15797 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
15798 switch (OMP_CLAUSE_CODE (c))
15804 case OMP_CLAUSE_MAP:
15806 /* First check what we're prepared to handle in the following. */
15807 switch (OMP_CLAUSE_MAP_KIND (c))
15809 case GOMP_MAP_ALLOC:
15811 case GOMP_MAP_FROM:
15812 case GOMP_MAP_TOFROM:
15813 case GOMP_MAP_POINTER:
15814 case GOMP_MAP_TO_PSET:
15815 case GOMP_MAP_DELETE:
15816 case GOMP_MAP_RELEASE:
15817 case GOMP_MAP_ALWAYS_TO:
15818 case GOMP_MAP_ALWAYS_FROM:
15819 case GOMP_MAP_ALWAYS_TOFROM:
15820 case GOMP_MAP_FIRSTPRIVATE_POINTER:
15821 case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
15822 case GOMP_MAP_STRUCT:
15823 case GOMP_MAP_ALWAYS_POINTER:
15825 case GOMP_MAP_FORCE_ALLOC:
15826 case GOMP_MAP_FORCE_TO:
15827 case GOMP_MAP_FORCE_FROM:
15828 case GOMP_MAP_FORCE_TOFROM:
15829 case GOMP_MAP_FORCE_PRESENT:
15830 case GOMP_MAP_FORCE_DEVICEPTR:
15831 case GOMP_MAP_DEVICE_RESIDENT:
15832 case GOMP_MAP_LINK:
15833 gcc_assert (is_gimple_omp_oacc (stmt));
15836 gcc_unreachable ();
15840 case OMP_CLAUSE_TO:
15841 case OMP_CLAUSE_FROM:
15843 var = OMP_CLAUSE_DECL (c);
15846 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
15847 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
15848 && (OMP_CLAUSE_MAP_KIND (c)
15849 != GOMP_MAP_FIRSTPRIVATE_POINTER)))
15854 if (DECL_SIZE (var)
15855 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
15857 tree var2 = DECL_VALUE_EXPR (var);
15858 gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
15859 var2 = TREE_OPERAND (var2, 0);
15860 gcc_assert (DECL_P (var2));
15865 && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
15866 && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
15867 || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE))
15869 if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
15871 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx))
15872 && varpool_node::get_create (var)->offloadable)
15875 tree type = build_pointer_type (TREE_TYPE (var));
15876 tree new_var = lookup_decl (var, ctx);
15877 x = create_tmp_var_raw (type, get_name (new_var));
15878 gimple_add_tmp_var (x);
15879 x = build_simple_mem_ref (x);
15880 SET_DECL_VALUE_EXPR (new_var, x);
15881 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
15886 if (!maybe_lookup_field (var, ctx))
15889 /* Don't remap oacc parallel reduction variables, because the
15890 intermediate result must be local to each gang. */
15891 if (offloaded && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
15892 && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
15894 x = build_receiver_ref (var, true, ctx);
15895 tree new_var = lookup_decl (var, ctx);
15897 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
15898 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
15899 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
15900 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
15901 x = build_simple_mem_ref (x);
15902 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
15904 gcc_assert (is_gimple_omp_oacc (ctx->stmt));
15905 if (is_reference (new_var))
15907 /* Create a local object to hold the instance
15909 tree type = TREE_TYPE (TREE_TYPE (new_var));
15910 const char *id = IDENTIFIER_POINTER (DECL_NAME (new_var));
15911 tree inst = create_tmp_var (type, id);
15912 gimplify_assign (inst, fold_indirect_ref (x), &fplist);
15913 x = build_fold_addr_expr (inst);
15915 gimplify_assign (new_var, x, &fplist);
15917 else if (DECL_P (new_var))
15919 SET_DECL_VALUE_EXPR (new_var, x);
15920 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
15923 gcc_unreachable ();
15928 case OMP_CLAUSE_FIRSTPRIVATE:
15929 if (is_oacc_parallel (ctx))
15930 goto oacc_firstprivate;
15932 var = OMP_CLAUSE_DECL (c);
15933 if (!is_reference (var)
15934 && !is_gimple_reg_type (TREE_TYPE (var)))
15936 tree new_var = lookup_decl (var, ctx);
15937 if (is_variable_sized (var))
15939 tree pvar = DECL_VALUE_EXPR (var);
15940 gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
15941 pvar = TREE_OPERAND (pvar, 0);
15942 gcc_assert (DECL_P (pvar));
15943 tree new_pvar = lookup_decl (pvar, ctx);
15944 x = build_fold_indirect_ref (new_pvar);
15945 TREE_THIS_NOTRAP (x) = 1;
15948 x = build_receiver_ref (var, true, ctx);
15949 SET_DECL_VALUE_EXPR (new_var, x);
15950 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
15954 case OMP_CLAUSE_PRIVATE:
15955 if (is_gimple_omp_oacc (ctx->stmt))
15957 var = OMP_CLAUSE_DECL (c);
15958 if (is_variable_sized (var))
15960 tree new_var = lookup_decl (var, ctx);
15961 tree pvar = DECL_VALUE_EXPR (var);
15962 gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
15963 pvar = TREE_OPERAND (pvar, 0);
15964 gcc_assert (DECL_P (pvar));
15965 tree new_pvar = lookup_decl (pvar, ctx);
15966 x = build_fold_indirect_ref (new_pvar);
15967 TREE_THIS_NOTRAP (x) = 1;
15968 SET_DECL_VALUE_EXPR (new_var, x);
15969 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
15973 case OMP_CLAUSE_USE_DEVICE_PTR:
15974 case OMP_CLAUSE_IS_DEVICE_PTR:
15975 var = OMP_CLAUSE_DECL (c);
15977 if (is_variable_sized (var))
15979 tree new_var = lookup_decl (var, ctx);
15980 tree pvar = DECL_VALUE_EXPR (var);
15981 gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
15982 pvar = TREE_OPERAND (pvar, 0);
15983 gcc_assert (DECL_P (pvar));
15984 tree new_pvar = lookup_decl (pvar, ctx);
15985 x = build_fold_indirect_ref (new_pvar);
15986 TREE_THIS_NOTRAP (x) = 1;
15987 SET_DECL_VALUE_EXPR (new_var, x);
15988 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
15990 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
15992 tree new_var = lookup_decl (var, ctx);
15993 tree type = build_pointer_type (TREE_TYPE (var));
15994 x = create_tmp_var_raw (type, get_name (new_var));
15995 gimple_add_tmp_var (x);
15996 x = build_simple_mem_ref (x);
15997 SET_DECL_VALUE_EXPR (new_var, x);
15998 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
16002 tree new_var = lookup_decl (var, ctx);
16003 x = create_tmp_var_raw (TREE_TYPE (new_var), get_name (new_var));
16004 gimple_add_tmp_var (x);
16005 SET_DECL_VALUE_EXPR (new_var, x);
16006 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
16013 target_nesting_level++;
16014 lower_omp (&tgt_body, ctx);
16015 target_nesting_level--;
16017 else if (data_region)
16018 lower_omp (&tgt_body, ctx);
16022 /* Declare all the variables created by mapping and the variables
16023 declared in the scope of the target body. */
16024 record_vars_into (ctx->block_vars, child_fn);
16025 record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
16030 if (ctx->record_type)
16033 = create_tmp_var (ctx->record_type, ".omp_data_arr");
16034 DECL_NAMELESS (ctx->sender_decl) = 1;
16035 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
16036 t = make_tree_vec (3);
16037 TREE_VEC_ELT (t, 0) = ctx->sender_decl;
16038 TREE_VEC_ELT (t, 1)
16039 = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt),
16040 ".omp_data_sizes");
16041 DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
16042 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
16043 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
16044 tree tkind_type = short_unsigned_type_node;
16045 int talign_shift = 8;
16046 TREE_VEC_ELT (t, 2)
16047 = create_tmp_var (build_array_type_nelts (tkind_type, map_cnt),
16048 ".omp_data_kinds");
16049 DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
16050 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
16051 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
16052 gimple_omp_target_set_data_arg (stmt, t);
16054 vec<constructor_elt, va_gc> *vsize;
16055 vec<constructor_elt, va_gc> *vkind;
16056 vec_alloc (vsize, map_cnt);
16057 vec_alloc (vkind, map_cnt);
16058 unsigned int map_idx = 0;
16060 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
16061 switch (OMP_CLAUSE_CODE (c))
16063 tree ovar, nc, s, purpose, var, x, type;
16064 unsigned int talign;
16069 case OMP_CLAUSE_MAP:
16070 case OMP_CLAUSE_TO:
16071 case OMP_CLAUSE_FROM:
16072 oacc_firstprivate_map:
16074 ovar = OMP_CLAUSE_DECL (c);
16075 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
16076 && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
16077 || (OMP_CLAUSE_MAP_KIND (c)
16078 == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
16080 if (!DECL_P (ovar))
16082 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
16083 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
16085 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c))
16086 == get_base_address (ovar));
16087 nc = OMP_CLAUSE_CHAIN (c);
16088 ovar = OMP_CLAUSE_DECL (nc);
16092 tree x = build_sender_ref (ovar, ctx);
16094 = build_fold_addr_expr_with_type (ovar, ptr_type_node);
16095 gimplify_assign (x, v, &ilist);
16101 if (DECL_SIZE (ovar)
16102 && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST)
16104 tree ovar2 = DECL_VALUE_EXPR (ovar);
16105 gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
16106 ovar2 = TREE_OPERAND (ovar2, 0);
16107 gcc_assert (DECL_P (ovar2));
16110 if (!maybe_lookup_field (ovar, ctx))
16114 talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
16115 if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
16116 talign = DECL_ALIGN_UNIT (ovar);
16119 var = lookup_decl_in_outer_ctx (ovar, ctx);
16120 x = build_sender_ref (ovar, ctx);
16122 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
16123 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
16124 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
16125 && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
16127 gcc_assert (offloaded);
16129 = create_tmp_var (TREE_TYPE (TREE_TYPE (x)));
16130 mark_addressable (avar);
16131 gimplify_assign (avar, build_fold_addr_expr (var), &ilist);
16132 talign = DECL_ALIGN_UNIT (avar);
16133 avar = build_fold_addr_expr (avar);
16134 gimplify_assign (x, avar, &ilist);
16136 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
16138 gcc_assert (is_gimple_omp_oacc (ctx->stmt));
16139 if (!is_reference (var))
16141 if (is_gimple_reg (var)
16142 && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
16143 TREE_NO_WARNING (var) = 1;
16144 var = build_fold_addr_expr (var);
16147 talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
16148 gimplify_assign (x, var, &ilist);
16150 else if (is_gimple_reg (var))
16152 gcc_assert (offloaded);
16153 tree avar = create_tmp_var (TREE_TYPE (var));
16154 mark_addressable (avar);
16155 enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c);
16156 if (GOMP_MAP_COPY_TO_P (map_kind)
16157 || map_kind == GOMP_MAP_POINTER
16158 || map_kind == GOMP_MAP_TO_PSET
16159 || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
16161 /* If we need to initialize a temporary
16162 with VAR because it is not addressable, and
16163 the variable hasn't been initialized yet, then
16164 we'll get a warning for the store to avar.
16165 Don't warn in that case, the mapping might
16167 TREE_NO_WARNING (var) = 1;
16168 gimplify_assign (avar, var, &ilist);
16170 avar = build_fold_addr_expr (avar);
16171 gimplify_assign (x, avar, &ilist);
16172 if ((GOMP_MAP_COPY_FROM_P (map_kind)
16173 || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
16174 && !TYPE_READONLY (TREE_TYPE (var)))
16176 x = unshare_expr (x);
16177 x = build_simple_mem_ref (x);
16178 gimplify_assign (var, x, &olist);
16183 var = build_fold_addr_expr (var);
16184 gimplify_assign (x, var, &ilist);
16188 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
16190 gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
16191 s = TREE_TYPE (ovar);
16192 if (TREE_CODE (s) == REFERENCE_TYPE)
16194 s = TYPE_SIZE_UNIT (s);
16197 s = OMP_CLAUSE_SIZE (c);
16198 if (s == NULL_TREE)
16199 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
16200 s = fold_convert (size_type_node, s);
16201 purpose = size_int (map_idx++);
16202 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
16203 if (TREE_CODE (s) != INTEGER_CST)
16204 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
16206 unsigned HOST_WIDE_INT tkind, tkind_zero;
16207 switch (OMP_CLAUSE_CODE (c))
16209 case OMP_CLAUSE_MAP:
16210 tkind = OMP_CLAUSE_MAP_KIND (c);
16211 tkind_zero = tkind;
16212 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c))
16215 case GOMP_MAP_ALLOC:
16217 case GOMP_MAP_FROM:
16218 case GOMP_MAP_TOFROM:
16219 case GOMP_MAP_ALWAYS_TO:
16220 case GOMP_MAP_ALWAYS_FROM:
16221 case GOMP_MAP_ALWAYS_TOFROM:
16222 case GOMP_MAP_RELEASE:
16223 tkind_zero = GOMP_MAP_ZERO_LEN_ARRAY_SECTION;
16225 case GOMP_MAP_DELETE:
16226 tkind_zero = GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION;
16230 if (tkind_zero != tkind)
16232 if (integer_zerop (s))
16233 tkind = tkind_zero;
16234 else if (integer_nonzerop (s))
16235 tkind_zero = tkind;
16238 case OMP_CLAUSE_FIRSTPRIVATE:
16239 gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
16240 tkind = GOMP_MAP_TO;
16241 tkind_zero = tkind;
16243 case OMP_CLAUSE_TO:
16244 tkind = GOMP_MAP_TO;
16245 tkind_zero = tkind;
16247 case OMP_CLAUSE_FROM:
16248 tkind = GOMP_MAP_FROM;
16249 tkind_zero = tkind;
16252 gcc_unreachable ();
16254 gcc_checking_assert (tkind
16255 < (HOST_WIDE_INT_C (1U) << talign_shift));
16256 gcc_checking_assert (tkind_zero
16257 < (HOST_WIDE_INT_C (1U) << talign_shift));
16258 talign = ceil_log2 (talign);
16259 tkind |= talign << talign_shift;
16260 tkind_zero |= talign << talign_shift;
16261 gcc_checking_assert (tkind
16262 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
16263 gcc_checking_assert (tkind_zero
16264 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
16265 if (tkind == tkind_zero)
16266 x = build_int_cstu (tkind_type, tkind);
16269 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 0;
16270 x = build3 (COND_EXPR, tkind_type,
16271 fold_build2 (EQ_EXPR, boolean_type_node,
16272 unshare_expr (s), size_zero_node),
16273 build_int_cstu (tkind_type, tkind_zero),
16274 build_int_cstu (tkind_type, tkind));
16276 CONSTRUCTOR_APPEND_ELT (vkind, purpose, x);
16281 case OMP_CLAUSE_FIRSTPRIVATE:
16282 if (is_oacc_parallel (ctx))
16283 goto oacc_firstprivate_map;
16284 ovar = OMP_CLAUSE_DECL (c);
16285 if (is_reference (ovar))
16286 talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
16288 talign = DECL_ALIGN_UNIT (ovar);
16289 var = lookup_decl_in_outer_ctx (ovar, ctx);
16290 x = build_sender_ref (ovar, ctx);
16291 tkind = GOMP_MAP_FIRSTPRIVATE;
16292 type = TREE_TYPE (ovar);
16293 if (is_reference (ovar))
16294 type = TREE_TYPE (type);
16295 if ((INTEGRAL_TYPE_P (type)
16296 && TYPE_PRECISION (type) <= POINTER_SIZE)
16297 || TREE_CODE (type) == POINTER_TYPE)
16299 tkind = GOMP_MAP_FIRSTPRIVATE_INT;
16301 if (is_reference (var))
16302 t = build_simple_mem_ref (var);
16303 else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
16304 TREE_NO_WARNING (var) = 1;
16305 if (TREE_CODE (type) != POINTER_TYPE)
16306 t = fold_convert (pointer_sized_int_node, t);
16307 t = fold_convert (TREE_TYPE (x), t);
16308 gimplify_assign (x, t, &ilist);
16310 else if (is_reference (var))
16311 gimplify_assign (x, var, &ilist);
16312 else if (is_gimple_reg (var))
16314 tree avar = create_tmp_var (TREE_TYPE (var));
16315 mark_addressable (avar);
16316 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
16317 TREE_NO_WARNING (var) = 1;
16318 gimplify_assign (avar, var, &ilist);
16319 avar = build_fold_addr_expr (avar);
16320 gimplify_assign (x, avar, &ilist);
16324 var = build_fold_addr_expr (var);
16325 gimplify_assign (x, var, &ilist);
16327 if (tkind == GOMP_MAP_FIRSTPRIVATE_INT)
16329 else if (is_reference (var))
16330 s = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
16332 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
16333 s = fold_convert (size_type_node, s);
16334 purpose = size_int (map_idx++);
16335 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
16336 if (TREE_CODE (s) != INTEGER_CST)
16337 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
16339 gcc_checking_assert (tkind
16340 < (HOST_WIDE_INT_C (1U) << talign_shift));
16341 talign = ceil_log2 (talign);
16342 tkind |= talign << talign_shift;
16343 gcc_checking_assert (tkind
16344 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
16345 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
16346 build_int_cstu (tkind_type, tkind));
16349 case OMP_CLAUSE_USE_DEVICE_PTR:
16350 case OMP_CLAUSE_IS_DEVICE_PTR:
16351 ovar = OMP_CLAUSE_DECL (c);
16352 var = lookup_decl_in_outer_ctx (ovar, ctx);
16353 x = build_sender_ref (ovar, ctx);
16354 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR)
16355 tkind = GOMP_MAP_USE_DEVICE_PTR;
16357 tkind = GOMP_MAP_FIRSTPRIVATE_INT;
16358 type = TREE_TYPE (ovar);
16359 if (TREE_CODE (type) == ARRAY_TYPE)
16360 var = build_fold_addr_expr (var);
16363 if (is_reference (ovar))
16365 type = TREE_TYPE (type);
16366 if (TREE_CODE (type) != ARRAY_TYPE)
16367 var = build_simple_mem_ref (var);
16368 var = fold_convert (TREE_TYPE (x), var);
16371 gimplify_assign (x, var, &ilist);
16373 purpose = size_int (map_idx++);
16374 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
16375 gcc_checking_assert (tkind
16376 < (HOST_WIDE_INT_C (1U) << talign_shift));
16377 gcc_checking_assert (tkind
16378 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
16379 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
16380 build_int_cstu (tkind_type, tkind));
16384 gcc_assert (map_idx == map_cnt);
16386 DECL_INITIAL (TREE_VEC_ELT (t, 1))
16387 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
16388 DECL_INITIAL (TREE_VEC_ELT (t, 2))
16389 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
16390 for (int i = 1; i <= 2; i++)
16391 if (!TREE_STATIC (TREE_VEC_ELT (t, i)))
16393 gimple_seq initlist = NULL;
16394 force_gimple_operand (build1 (DECL_EXPR, void_type_node,
16395 TREE_VEC_ELT (t, i)),
16396 &initlist, true, NULL_TREE);
16397 gimple_seq_add_seq (&ilist, initlist);
16399 tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, i)),
16401 TREE_THIS_VOLATILE (clobber) = 1;
16402 gimple_seq_add_stmt (&olist,
16403 gimple_build_assign (TREE_VEC_ELT (t, i),
16407 tree clobber = build_constructor (ctx->record_type, NULL);
16408 TREE_THIS_VOLATILE (clobber) = 1;
16409 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
16413 /* Once all the expansions are done, sequence all the different
16414 fragments inside gimple_omp_body. */
16419 && ctx->record_type)
16421 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
16422 /* fixup_child_record_type might have changed receiver_decl's type. */
16423 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
16424 gimple_seq_add_stmt (&new_body,
16425 gimple_build_assign (ctx->receiver_decl, t));
16427 gimple_seq_add_seq (&new_body, fplist);
16429 if (offloaded || data_region)
16431 tree prev = NULL_TREE;
16432 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
16433 switch (OMP_CLAUSE_CODE (c))
16438 case OMP_CLAUSE_FIRSTPRIVATE:
16439 if (is_gimple_omp_oacc (ctx->stmt))
16441 var = OMP_CLAUSE_DECL (c);
16442 if (is_reference (var)
16443 || is_gimple_reg_type (TREE_TYPE (var)))
16445 tree new_var = lookup_decl (var, ctx);
16447 type = TREE_TYPE (var);
16448 if (is_reference (var))
16449 type = TREE_TYPE (type);
16450 if ((INTEGRAL_TYPE_P (type)
16451 && TYPE_PRECISION (type) <= POINTER_SIZE)
16452 || TREE_CODE (type) == POINTER_TYPE)
16454 x = build_receiver_ref (var, false, ctx);
16455 if (TREE_CODE (type) != POINTER_TYPE)
16456 x = fold_convert (pointer_sized_int_node, x);
16457 x = fold_convert (type, x);
16458 gimplify_expr (&x, &new_body, NULL, is_gimple_val,
16460 if (is_reference (var))
16462 tree v = create_tmp_var_raw (type, get_name (var));
16463 gimple_add_tmp_var (v);
16464 TREE_ADDRESSABLE (v) = 1;
16465 gimple_seq_add_stmt (&new_body,
16466 gimple_build_assign (v, x));
16467 x = build_fold_addr_expr (v);
16469 gimple_seq_add_stmt (&new_body,
16470 gimple_build_assign (new_var, x));
16474 x = build_receiver_ref (var, !is_reference (var), ctx);
16475 gimplify_expr (&x, &new_body, NULL, is_gimple_val,
16477 gimple_seq_add_stmt (&new_body,
16478 gimple_build_assign (new_var, x));
16481 else if (is_variable_sized (var))
16483 tree pvar = DECL_VALUE_EXPR (var);
16484 gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
16485 pvar = TREE_OPERAND (pvar, 0);
16486 gcc_assert (DECL_P (pvar));
16487 tree new_var = lookup_decl (pvar, ctx);
16488 x = build_receiver_ref (var, false, ctx);
16489 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
16490 gimple_seq_add_stmt (&new_body,
16491 gimple_build_assign (new_var, x));
16494 case OMP_CLAUSE_PRIVATE:
16495 if (is_gimple_omp_oacc (ctx->stmt))
16497 var = OMP_CLAUSE_DECL (c);
16498 if (is_reference (var))
16500 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
16501 tree new_var = lookup_decl (var, ctx);
16502 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
16503 if (TREE_CONSTANT (x))
16505 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
16507 gimple_add_tmp_var (x);
16508 TREE_ADDRESSABLE (x) = 1;
16509 x = build_fold_addr_expr_loc (clause_loc, x);
16514 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
16515 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
16516 gimple_seq_add_stmt (&new_body,
16517 gimple_build_assign (new_var, x));
16520 case OMP_CLAUSE_USE_DEVICE_PTR:
16521 case OMP_CLAUSE_IS_DEVICE_PTR:
16522 var = OMP_CLAUSE_DECL (c);
16523 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR)
16524 x = build_sender_ref (var, ctx);
16526 x = build_receiver_ref (var, false, ctx);
16527 if (is_variable_sized (var))
16529 tree pvar = DECL_VALUE_EXPR (var);
16530 gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
16531 pvar = TREE_OPERAND (pvar, 0);
16532 gcc_assert (DECL_P (pvar));
16533 tree new_var = lookup_decl (pvar, ctx);
16534 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
16535 gimple_seq_add_stmt (&new_body,
16536 gimple_build_assign (new_var, x));
16538 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
16540 tree new_var = lookup_decl (var, ctx);
16541 new_var = DECL_VALUE_EXPR (new_var);
16542 gcc_assert (TREE_CODE (new_var) == MEM_REF);
16543 new_var = TREE_OPERAND (new_var, 0);
16544 gcc_assert (DECL_P (new_var));
16545 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
16546 gimple_seq_add_stmt (&new_body,
16547 gimple_build_assign (new_var, x));
16551 tree type = TREE_TYPE (var);
16552 tree new_var = lookup_decl (var, ctx);
16553 if (is_reference (var))
16555 type = TREE_TYPE (type);
16556 if (TREE_CODE (type) != ARRAY_TYPE)
16558 tree v = create_tmp_var_raw (type, get_name (var));
16559 gimple_add_tmp_var (v);
16560 TREE_ADDRESSABLE (v) = 1;
16561 x = fold_convert (type, x);
16562 gimplify_expr (&x, &new_body, NULL, is_gimple_val,
16564 gimple_seq_add_stmt (&new_body,
16565 gimple_build_assign (v, x));
16566 x = build_fold_addr_expr (v);
16569 new_var = DECL_VALUE_EXPR (new_var);
16570 x = fold_convert (TREE_TYPE (new_var), x);
16571 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
16572 gimple_seq_add_stmt (&new_body,
16573 gimple_build_assign (new_var, x));
16577 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
16578 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
16579 are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
16580 or references to VLAs. */
16581 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
16582 switch (OMP_CLAUSE_CODE (c))
16587 case OMP_CLAUSE_MAP:
16588 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
16589 || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
16591 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
16592 HOST_WIDE_INT offset = 0;
16594 var = OMP_CLAUSE_DECL (c);
16596 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
16597 && is_global_var (maybe_lookup_decl_in_outer_ctx (var,
16599 && varpool_node::get_create (var)->offloadable)
16601 if (TREE_CODE (var) == INDIRECT_REF
16602 && TREE_CODE (TREE_OPERAND (var, 0)) == COMPONENT_REF)
16603 var = TREE_OPERAND (var, 0);
16604 if (TREE_CODE (var) == COMPONENT_REF)
16606 var = get_addr_base_and_unit_offset (var, &offset);
16607 gcc_assert (var != NULL_TREE && DECL_P (var));
16609 else if (DECL_SIZE (var)
16610 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
16612 tree var2 = DECL_VALUE_EXPR (var);
16613 gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
16614 var2 = TREE_OPERAND (var2, 0);
16615 gcc_assert (DECL_P (var2));
16618 tree new_var = lookup_decl (var, ctx), x;
16619 tree type = TREE_TYPE (new_var);
16621 if (TREE_CODE (OMP_CLAUSE_DECL (c)) == INDIRECT_REF
16622 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0))
16625 type = TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0));
16627 new_var = build2 (MEM_REF, type,
16628 build_fold_addr_expr (new_var),
16629 build_int_cst (build_pointer_type (type),
16632 else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPONENT_REF)
16634 type = TREE_TYPE (OMP_CLAUSE_DECL (c));
16635 is_ref = TREE_CODE (type) == REFERENCE_TYPE;
16636 new_var = build2 (MEM_REF, type,
16637 build_fold_addr_expr (new_var),
16638 build_int_cst (build_pointer_type (type),
16642 is_ref = is_reference (var);
16643 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
16645 bool ref_to_array = false;
16648 type = TREE_TYPE (type);
16649 if (TREE_CODE (type) == ARRAY_TYPE)
16651 type = build_pointer_type (type);
16652 ref_to_array = true;
16655 else if (TREE_CODE (type) == ARRAY_TYPE)
16657 tree decl2 = DECL_VALUE_EXPR (new_var);
16658 gcc_assert (TREE_CODE (decl2) == MEM_REF);
16659 decl2 = TREE_OPERAND (decl2, 0);
16660 gcc_assert (DECL_P (decl2));
16662 type = TREE_TYPE (new_var);
16664 x = build_receiver_ref (OMP_CLAUSE_DECL (prev), false, ctx);
16665 x = fold_convert_loc (clause_loc, type, x);
16666 if (!integer_zerop (OMP_CLAUSE_SIZE (c)))
16668 tree bias = OMP_CLAUSE_SIZE (c);
16670 bias = lookup_decl (bias, ctx);
16671 bias = fold_convert_loc (clause_loc, sizetype, bias);
16672 bias = fold_build1_loc (clause_loc, NEGATE_EXPR, sizetype,
16674 x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
16675 TREE_TYPE (x), x, bias);
16678 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
16679 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
16680 if (is_ref && !ref_to_array)
16682 tree t = create_tmp_var_raw (type, get_name (var));
16683 gimple_add_tmp_var (t);
16684 TREE_ADDRESSABLE (t) = 1;
16685 gimple_seq_add_stmt (&new_body,
16686 gimple_build_assign (t, x));
16687 x = build_fold_addr_expr_loc (clause_loc, t);
16689 gimple_seq_add_stmt (&new_body,
16690 gimple_build_assign (new_var, x));
16693 else if (OMP_CLAUSE_CHAIN (c)
16694 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c))
16696 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
16697 == GOMP_MAP_FIRSTPRIVATE_POINTER
16698 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
16699 == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
16702 case OMP_CLAUSE_PRIVATE:
16703 var = OMP_CLAUSE_DECL (c);
16704 if (is_variable_sized (var))
16706 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
16707 tree new_var = lookup_decl (var, ctx);
16708 tree pvar = DECL_VALUE_EXPR (var);
16709 gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
16710 pvar = TREE_OPERAND (pvar, 0);
16711 gcc_assert (DECL_P (pvar));
16712 tree new_pvar = lookup_decl (pvar, ctx);
16713 tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
16714 tree al = size_int (DECL_ALIGN (var));
16715 tree x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
16716 x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
16717 x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar), x);
16718 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
16719 gimple_seq_add_stmt (&new_body,
16720 gimple_build_assign (new_pvar, x));
16722 else if (is_reference (var) && !is_gimple_omp_oacc (ctx->stmt))
16724 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
16725 tree new_var = lookup_decl (var, ctx);
16726 tree x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
16727 if (TREE_CONSTANT (x))
16732 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
16733 tree rtype = TREE_TYPE (TREE_TYPE (new_var));
16734 tree al = size_int (TYPE_ALIGN (rtype));
16735 x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
16738 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
16739 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
16740 gimple_seq_add_stmt (&new_body,
16741 gimple_build_assign (new_var, x));
16746 gimple_seq fork_seq = NULL;
16747 gimple_seq join_seq = NULL;
16749 if (is_oacc_parallel (ctx))
16751 /* If there are reductions on the offloaded region itself, treat
16752 them as a dummy GANG loop. */
16753 tree level = build_int_cst (integer_type_node, GOMP_DIM_GANG);
16755 lower_oacc_reductions (gimple_location (ctx->stmt), clauses, level,
16756 false, NULL, NULL, &fork_seq, &join_seq, ctx);
16759 gimple_seq_add_seq (&new_body, fork_seq);
16760 gimple_seq_add_seq (&new_body, tgt_body);
16761 gimple_seq_add_seq (&new_body, join_seq);
16764 new_body = maybe_catch_exception (new_body);
16766 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
16767 gimple_omp_set_body (stmt, new_body);
16770 bind = gimple_build_bind (NULL, NULL,
16771 tgt_bind ? gimple_bind_block (tgt_bind)
16773 gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
16774 gimple_bind_add_seq (bind, ilist);
16775 gimple_bind_add_stmt (bind, stmt);
16776 gimple_bind_add_seq (bind, olist);
16778 pop_gimplify_context (NULL);
16782 gimple_bind_add_seq (dep_bind, dep_ilist);
16783 gimple_bind_add_stmt (dep_bind, bind);
16784 gimple_bind_add_seq (dep_bind, dep_olist);
16785 pop_gimplify_context (dep_bind);
16789 /* Expand code for an OpenMP teams directive. */
16792 lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
16794 gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p));
16795 push_gimplify_context ();
16797 tree block = make_node (BLOCK);
16798 gbind *bind = gimple_build_bind (NULL, NULL, block);
16799 gsi_replace (gsi_p, bind, true);
16800 gimple_seq bind_body = NULL;
16801 gimple_seq dlist = NULL;
16802 gimple_seq olist = NULL;
16804 tree num_teams = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
16805 OMP_CLAUSE_NUM_TEAMS);
16806 if (num_teams == NULL_TREE)
16807 num_teams = build_int_cst (unsigned_type_node, 0);
16810 num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams);
16811 num_teams = fold_convert (unsigned_type_node, num_teams);
16812 gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
16814 tree thread_limit = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
16815 OMP_CLAUSE_THREAD_LIMIT);
16816 if (thread_limit == NULL_TREE)
16817 thread_limit = build_int_cst (unsigned_type_node, 0);
16820 thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
16821 thread_limit = fold_convert (unsigned_type_node, thread_limit);
16822 gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
16826 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
16827 &bind_body, &dlist, ctx, NULL);
16828 lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
16829 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx);
16830 if (!gimple_omp_teams_grid_phony (teams_stmt))
16832 gimple_seq_add_stmt (&bind_body, teams_stmt);
16833 location_t loc = gimple_location (teams_stmt);
16834 tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS);
16835 gimple *call = gimple_build_call (decl, 2, num_teams, thread_limit);
16836 gimple_set_location (call, loc);
16837 gimple_seq_add_stmt (&bind_body, call);
16840 gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
16841 gimple_omp_set_body (teams_stmt, NULL);
16842 gimple_seq_add_seq (&bind_body, olist);
16843 gimple_seq_add_seq (&bind_body, dlist);
16844 if (!gimple_omp_teams_grid_phony (teams_stmt))
16845 gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
16846 gimple_bind_set_body (bind, bind_body);
16848 pop_gimplify_context (bind);
16850 gimple_bind_append_vars (bind, ctx->block_vars);
16851 BLOCK_VARS (block) = ctx->block_vars;
16852 if (BLOCK_VARS (block))
16853 TREE_USED (block) = 1;
16856 /* Expand code within an artificial GIMPLE_OMP_GRID_BODY OMP construct. */
16859 lower_omp_grid_body (gimple_stmt_iterator *gsi_p, omp_context *ctx)
16861 gimple *stmt = gsi_stmt (*gsi_p);
16862 lower_omp (gimple_omp_body_ptr (stmt), ctx);
16863 gimple_seq_add_stmt (gimple_omp_body_ptr (stmt),
16864 gimple_build_omp_return (false));
16868 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
16869 regimplified. If DATA is non-NULL, lower_omp_1 is outside
16870 of OMP context, but with task_shared_vars set. */
16873 lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
16878 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
16879 if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t))
16882 if (task_shared_vars
16884 && bitmap_bit_p (task_shared_vars, DECL_UID (t)))
16887 /* If a global variable has been privatized, TREE_CONSTANT on
16888 ADDR_EXPR might be wrong. */
16889 if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
16890 recompute_tree_invariant_for_addr_expr (t);
16892 *walk_subtrees = !IS_TYPE_OR_DECL_P (t);
16896 /* Data to be communicated between lower_omp_regimplify_operands and
16897 lower_omp_regimplify_operands_p. */
16899 struct lower_omp_regimplify_operands_data
16905 /* Helper function for lower_omp_regimplify_operands. Find
16906 omp_member_access_dummy_var vars and adjust temporarily their
16907 DECL_VALUE_EXPRs if needed. */
16910 lower_omp_regimplify_operands_p (tree *tp, int *walk_subtrees,
16913 tree t = omp_member_access_dummy_var (*tp);
16916 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
16917 lower_omp_regimplify_operands_data *ldata
16918 = (lower_omp_regimplify_operands_data *) wi->info;
16919 tree o = maybe_lookup_decl (t, ldata->ctx);
16922 ldata->decls->safe_push (DECL_VALUE_EXPR (*tp));
16923 ldata->decls->safe_push (*tp);
16924 tree v = unshare_and_remap (DECL_VALUE_EXPR (*tp), t, o);
16925 SET_DECL_VALUE_EXPR (*tp, v);
16928 *walk_subtrees = !IS_TYPE_OR_DECL_P (*tp);
16932 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
16933 of omp_member_access_dummy_var vars during regimplification. */
16936 lower_omp_regimplify_operands (omp_context *ctx, gimple *stmt,
16937 gimple_stmt_iterator *gsi_p)
16939 auto_vec<tree, 10> decls;
16942 struct walk_stmt_info wi;
16943 memset (&wi, '\0', sizeof (wi));
16944 struct lower_omp_regimplify_operands_data data;
16946 data.decls = &decls;
16948 walk_gimple_op (stmt, lower_omp_regimplify_operands_p, &wi);
16950 gimple_regimplify_operands (stmt, gsi_p);
16951 while (!decls.is_empty ())
16953 tree t = decls.pop ();
16954 tree v = decls.pop ();
16955 SET_DECL_VALUE_EXPR (t, v);
16960 lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
16962 gimple *stmt = gsi_stmt (*gsi_p);
16963 struct walk_stmt_info wi;
16966 if (gimple_has_location (stmt))
16967 input_location = gimple_location (stmt);
16969 if (task_shared_vars)
16970 memset (&wi, '\0', sizeof (wi));
16972 /* If we have issued syntax errors, avoid doing any heavy lifting.
16973 Just replace the OMP directives with a NOP to avoid
16974 confusing RTL expansion. */
16975 if (seen_error () && is_gimple_omp (stmt))
16977 gsi_replace (gsi_p, gimple_build_nop (), true);
16981 switch (gimple_code (stmt))
16985 gcond *cond_stmt = as_a <gcond *> (stmt);
16986 if ((ctx || task_shared_vars)
16987 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
16988 lower_omp_regimplify_p,
16989 ctx ? NULL : &wi, NULL)
16990 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
16991 lower_omp_regimplify_p,
16992 ctx ? NULL : &wi, NULL)))
16993 lower_omp_regimplify_operands (ctx, cond_stmt, gsi_p);
16997 lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx);
16999 case GIMPLE_EH_FILTER:
17000 lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
17003 lower_omp (gimple_try_eval_ptr (stmt), ctx);
17004 lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
17006 case GIMPLE_TRANSACTION:
17007 lower_omp (gimple_transaction_body_ptr (
17008 as_a <gtransaction *> (stmt)),
17012 lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx);
17014 case GIMPLE_OMP_PARALLEL:
17015 case GIMPLE_OMP_TASK:
17016 ctx = maybe_lookup_ctx (stmt);
17018 if (ctx->cancellable)
17019 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
17020 lower_omp_taskreg (gsi_p, ctx);
17022 case GIMPLE_OMP_FOR:
17023 ctx = maybe_lookup_ctx (stmt);
17025 if (ctx->cancellable)
17026 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
17027 lower_omp_for (gsi_p, ctx);
17029 case GIMPLE_OMP_SECTIONS:
17030 ctx = maybe_lookup_ctx (stmt);
17032 if (ctx->cancellable)
17033 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
17034 lower_omp_sections (gsi_p, ctx);
17036 case GIMPLE_OMP_SINGLE:
17037 ctx = maybe_lookup_ctx (stmt);
17039 lower_omp_single (gsi_p, ctx);
17041 case GIMPLE_OMP_MASTER:
17042 ctx = maybe_lookup_ctx (stmt);
17044 lower_omp_master (gsi_p, ctx);
17046 case GIMPLE_OMP_TASKGROUP:
17047 ctx = maybe_lookup_ctx (stmt);
17049 lower_omp_taskgroup (gsi_p, ctx);
17051 case GIMPLE_OMP_ORDERED:
17052 ctx = maybe_lookup_ctx (stmt);
17054 lower_omp_ordered (gsi_p, ctx);
17056 case GIMPLE_OMP_CRITICAL:
17057 ctx = maybe_lookup_ctx (stmt);
17059 lower_omp_critical (gsi_p, ctx);
17061 case GIMPLE_OMP_ATOMIC_LOAD:
17062 if ((ctx || task_shared_vars)
17063 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
17064 as_a <gomp_atomic_load *> (stmt)),
17065 lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
17066 lower_omp_regimplify_operands (ctx, stmt, gsi_p);
17068 case GIMPLE_OMP_TARGET:
17069 ctx = maybe_lookup_ctx (stmt);
17071 lower_omp_target (gsi_p, ctx);
17073 case GIMPLE_OMP_TEAMS:
17074 ctx = maybe_lookup_ctx (stmt);
17076 lower_omp_teams (gsi_p, ctx);
17078 case GIMPLE_OMP_GRID_BODY:
17079 ctx = maybe_lookup_ctx (stmt);
17081 lower_omp_grid_body (gsi_p, ctx);
17085 call_stmt = as_a <gcall *> (stmt);
17086 fndecl = gimple_call_fndecl (call_stmt);
17088 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
17089 switch (DECL_FUNCTION_CODE (fndecl))
17091 case BUILT_IN_GOMP_BARRIER:
17095 case BUILT_IN_GOMP_CANCEL:
17096 case BUILT_IN_GOMP_CANCELLATION_POINT:
17099 if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
17100 cctx = cctx->outer;
17101 gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE);
17102 if (!cctx->cancellable)
17104 if (DECL_FUNCTION_CODE (fndecl)
17105 == BUILT_IN_GOMP_CANCELLATION_POINT)
17107 stmt = gimple_build_nop ();
17108 gsi_replace (gsi_p, stmt, false);
17112 if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
17114 fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
17115 gimple_call_set_fndecl (call_stmt, fndecl);
17116 gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl));
17119 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)));
17120 gimple_call_set_lhs (call_stmt, lhs);
17121 tree fallthru_label;
17122 fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
17124 g = gimple_build_label (fallthru_label);
17125 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
17126 g = gimple_build_cond (NE_EXPR, lhs,
17127 fold_convert (TREE_TYPE (lhs),
17128 boolean_false_node),
17129 cctx->cancel_label, fallthru_label);
17130 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
17137 if ((ctx || task_shared_vars)
17138 && walk_gimple_op (stmt, lower_omp_regimplify_p,
17141 /* Just remove clobbers, this should happen only if we have
17142 "privatized" local addressable variables in SIMD regions,
17143 the clobber isn't needed in that case and gimplifying address
17144 of the ARRAY_REF into a pointer and creating MEM_REF based
17145 clobber would create worse code than we get with the clobber
17147 if (gimple_clobber_p (stmt))
17149 gsi_replace (gsi_p, gimple_build_nop (), true);
17152 lower_omp_regimplify_operands (ctx, stmt, gsi_p);
17159 lower_omp (gimple_seq *body, omp_context *ctx)
17161 location_t saved_location = input_location;
17162 gimple_stmt_iterator gsi;
17163 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
17164 lower_omp_1 (&gsi, ctx);
17165 /* During gimplification, we haven't folded statments inside offloading
17166 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
17167 if (target_nesting_level || taskreg_nesting_level)
17168 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
17170 input_location = saved_location;
17173 /* Returen true if STMT is an assignment of a register-type into a local
17177 grid_reg_assignment_to_local_var_p (gimple *stmt)
17179 gassign *assign = dyn_cast <gassign *> (stmt);
17182 tree lhs = gimple_assign_lhs (assign);
17183 if (TREE_CODE (lhs) != VAR_DECL
17184 || !is_gimple_reg_type (TREE_TYPE (lhs))
17185 || is_global_var (lhs))
17190 /* Return true if all statements in SEQ are assignments to local register-type
17194 grid_seq_only_contains_local_assignments (gimple_seq seq)
17199 gimple_stmt_iterator gsi;
17200 for (gsi = gsi_start (seq); !gsi_end_p (gsi); gsi_next (&gsi))
17201 if (!grid_reg_assignment_to_local_var_p (gsi_stmt (gsi)))
17206 /* Scan statements in SEQ and call itself recursively on any bind. If during
17207 whole search only assignments to register-type local variables and one
17208 single OMP statement is encountered, return true, otherwise return false.
17209 RET is where we store any OMP statement encountered. TARGET_LOC and NAME
17210 are used for dumping a note about a failure. */
17213 grid_find_single_omp_among_assignments_1 (gimple_seq seq, location_t target_loc,
17214 const char *name, gimple **ret)
17216 gimple_stmt_iterator gsi;
17217 for (gsi = gsi_start (seq); !gsi_end_p (gsi); gsi_next (&gsi))
17219 gimple *stmt = gsi_stmt (gsi);
17221 if (grid_reg_assignment_to_local_var_p (stmt))
17223 if (gbind *bind = dyn_cast <gbind *> (stmt))
17225 if (!grid_find_single_omp_among_assignments_1 (gimple_bind_body (bind),
17226 target_loc, name, ret))
17229 else if (is_gimple_omp (stmt))
17233 if (dump_enabled_p ())
17234 dump_printf_loc (MSG_NOTE, target_loc,
17235 "Will not turn target construct into a simple "
17236 "GPGPU kernel because %s construct contains "
17237 "multiple OpenMP constructs\n", name);
17244 if (dump_enabled_p ())
17245 dump_printf_loc (MSG_NOTE, target_loc,
17246 "Will not turn target construct into a simple "
17247 "GPGPU kernel because %s construct contains "
17248 "a complex statement\n", name);
17255 /* Scan statements in SEQ and make sure that it and any binds in it contain
17256 only assignments to local register-type variables and one OMP construct. If
17257 so, return that construct, otherwise return NULL. If dumping is enabled and
17258 function fails, use TARGET_LOC and NAME to dump a note with the reason for
17262 grid_find_single_omp_among_assignments (gimple_seq seq, location_t target_loc,
17267 if (dump_enabled_p ())
17268 dump_printf_loc (MSG_NOTE, target_loc,
17269 "Will not turn target construct into a simple "
17270 "GPGPU kernel because %s construct has empty "
17276 gimple *ret = NULL;
17277 if (grid_find_single_omp_among_assignments_1 (seq, target_loc, name, &ret))
17279 if (!ret && dump_enabled_p ())
17280 dump_printf_loc (MSG_NOTE, target_loc,
17281 "Will not turn target construct into a simple "
17282 "GPGPU kernel because %s construct does not contain"
17283 "any other OpenMP construct\n", name);
17290 /* Walker function looking for statements there is no point gridifying (and for
17291 noreturn function calls which we cannot do). Return non-NULL if such a
17292 function is found. */
17295 grid_find_ungridifiable_statement (gimple_stmt_iterator *gsi,
17296 bool *handled_ops_p,
17297 struct walk_stmt_info *wi)
17299 *handled_ops_p = false;
17300 gimple *stmt = gsi_stmt (*gsi);
17301 switch (gimple_code (stmt))
17304 if (gimple_call_noreturn_p (as_a <gcall *> (stmt)))
17306 *handled_ops_p = true;
17308 return error_mark_node;
17312 /* We may reduce the following list if we find a way to implement the
17313 clauses, but now there is no point trying further. */
17314 case GIMPLE_OMP_CRITICAL:
17315 case GIMPLE_OMP_TASKGROUP:
17316 case GIMPLE_OMP_TASK:
17317 case GIMPLE_OMP_SECTION:
17318 case GIMPLE_OMP_SECTIONS:
17319 case GIMPLE_OMP_SECTIONS_SWITCH:
17320 case GIMPLE_OMP_TARGET:
17321 case GIMPLE_OMP_ORDERED:
17322 *handled_ops_p = true;
17324 return error_mark_node;
17326 case GIMPLE_OMP_FOR:
17327 if ((gimple_omp_for_kind (stmt) & GF_OMP_FOR_SIMD)
17328 && gimple_omp_for_combined_into_p (stmt))
17330 *handled_ops_p = true;
17332 return error_mark_node;
17343 /* If TARGET follows a pattern that can be turned into a gridified GPGPU
17344 kernel, return true, otherwise return false. In the case of success, also
17345 fill in GROUP_SIZE_P with the requested group size or NULL if there is
17349 grid_target_follows_gridifiable_pattern (gomp_target *target, tree *group_size_p)
17351 if (gimple_omp_target_kind (target) != GF_OMP_TARGET_KIND_REGION)
17354 location_t tloc = gimple_location (target);
17356 = grid_find_single_omp_among_assignments (gimple_omp_body (target),
17360 gomp_teams *teams = dyn_cast <gomp_teams *> (stmt);
17361 tree group_size = NULL;
17364 dump_printf_loc (MSG_NOTE, tloc,
17365 "Will not turn target construct into a simple "
17366 "GPGPU kernel because it does not have a sole teams "
17367 "construct in it.\n");
17371 tree clauses = gimple_omp_teams_clauses (teams);
17374 switch (OMP_CLAUSE_CODE (clauses))
17376 case OMP_CLAUSE_NUM_TEAMS:
17377 if (dump_enabled_p ())
17378 dump_printf_loc (MSG_NOTE, tloc,
17379 "Will not turn target construct into a "
17380 "gridified GPGPU kernel because we cannot "
17381 "handle num_teams clause of teams "
17385 case OMP_CLAUSE_REDUCTION:
17386 if (dump_enabled_p ())
17387 dump_printf_loc (MSG_NOTE, tloc,
17388 "Will not turn target construct into a "
17389 "gridified GPGPU kernel because a reduction "
17390 "clause is present\n ");
17393 case OMP_CLAUSE_LASTPRIVATE:
17394 if (dump_enabled_p ())
17395 dump_printf_loc (MSG_NOTE, tloc,
17396 "Will not turn target construct into a "
17397 "gridified GPGPU kernel because a lastprivate "
17398 "clause is present\n ");
17401 case OMP_CLAUSE_THREAD_LIMIT:
17402 group_size = OMP_CLAUSE_OPERAND (clauses, 0);
17408 clauses = OMP_CLAUSE_CHAIN (clauses);
17411 stmt = grid_find_single_omp_among_assignments (gimple_omp_body (teams), tloc,
17415 gomp_for *dist = dyn_cast <gomp_for *> (stmt);
17418 dump_printf_loc (MSG_NOTE, tloc,
17419 "Will not turn target construct into a simple "
17420 "GPGPU kernel because the teams construct does not have "
17421 "a sole distribute construct in it.\n");
17425 gcc_assert (gimple_omp_for_kind (dist) == GF_OMP_FOR_KIND_DISTRIBUTE);
17426 if (!gimple_omp_for_combined_p (dist))
17428 if (dump_enabled_p ())
17429 dump_printf_loc (MSG_NOTE, tloc,
17430 "Will not turn target construct into a gridified GPGPU "
17431 "kernel because we cannot handle a standalone "
17432 "distribute construct\n ");
17435 if (dist->collapse > 1)
17437 if (dump_enabled_p ())
17438 dump_printf_loc (MSG_NOTE, tloc,
17439 "Will not turn target construct into a gridified GPGPU "
17440 "kernel because the distribute construct contains "
17441 "collapse clause\n");
17444 struct omp_for_data fd;
17445 extract_omp_for_data (dist, &fd, NULL);
17448 if (group_size && !operand_equal_p (group_size, fd.chunk_size, 0))
17450 if (dump_enabled_p ())
17451 dump_printf_loc (MSG_NOTE, tloc,
17452 "Will not turn target construct into a "
17453 "gridified GPGPU kernel because the teams "
17454 "thread limit is different from distribute "
17455 "schedule chunk\n");
17458 group_size = fd.chunk_size;
17460 stmt = grid_find_single_omp_among_assignments (gimple_omp_body (dist), tloc,
17462 gomp_parallel *par;
17463 if (!stmt || !(par = dyn_cast <gomp_parallel *> (stmt)))
17466 clauses = gimple_omp_parallel_clauses (par);
17469 switch (OMP_CLAUSE_CODE (clauses))
17471 case OMP_CLAUSE_NUM_THREADS:
17472 if (dump_enabled_p ())
17473 dump_printf_loc (MSG_NOTE, tloc,
17474 "Will not turn target construct into a gridified"
17475 "GPGPU kernel because there is a num_threads "
17476 "clause of the parallel construct\n");
17479 case OMP_CLAUSE_REDUCTION:
17480 if (dump_enabled_p ())
17481 dump_printf_loc (MSG_NOTE, tloc,
17482 "Will not turn target construct into a "
17483 "gridified GPGPU kernel because a reduction "
17484 "clause is present\n ");
17487 case OMP_CLAUSE_LASTPRIVATE:
17488 if (dump_enabled_p ())
17489 dump_printf_loc (MSG_NOTE, tloc,
17490 "Will not turn target construct into a "
17491 "gridified GPGPU kernel because a lastprivate "
17492 "clause is present\n ");
17498 clauses = OMP_CLAUSE_CHAIN (clauses);
17501 stmt = grid_find_single_omp_among_assignments (gimple_omp_body (par), tloc,
17504 if (!stmt || !(gfor = dyn_cast <gomp_for *> (stmt)))
17507 if (gimple_omp_for_kind (gfor) != GF_OMP_FOR_KIND_FOR)
17509 if (dump_enabled_p ())
17510 dump_printf_loc (MSG_NOTE, tloc,
17511 "Will not turn target construct into a gridified GPGPU "
17512 "kernel because the inner loop is not a simple for "
17516 if (gfor->collapse > 1)
17518 if (dump_enabled_p ())
17519 dump_printf_loc (MSG_NOTE, tloc,
17520 "Will not turn target construct into a gridified GPGPU "
17521 "kernel because the inner loop contains collapse "
17526 if (!grid_seq_only_contains_local_assignments (gimple_omp_for_pre_body (gfor)))
17528 if (dump_enabled_p ())
17529 dump_printf_loc (MSG_NOTE, tloc,
17530 "Will not turn target construct into a gridified GPGPU "
17531 "kernel because the inner loop pre_body contains"
17532 "a complex instruction\n");
17536 clauses = gimple_omp_for_clauses (gfor);
17539 switch (OMP_CLAUSE_CODE (clauses))
17541 case OMP_CLAUSE_SCHEDULE:
17542 if (OMP_CLAUSE_SCHEDULE_KIND (clauses) != OMP_CLAUSE_SCHEDULE_AUTO)
17544 if (dump_enabled_p ())
17545 dump_printf_loc (MSG_NOTE, tloc,
17546 "Will not turn target construct into a "
17547 "gridified GPGPU kernel because the inner "
17548 "loop has a non-automatic scheduling clause\n");
17553 case OMP_CLAUSE_REDUCTION:
17554 if (dump_enabled_p ())
17555 dump_printf_loc (MSG_NOTE, tloc,
17556 "Will not turn target construct into a "
17557 "gridified GPGPU kernel because a reduction "
17558 "clause is present\n ");
17561 case OMP_CLAUSE_LASTPRIVATE:
17562 if (dump_enabled_p ())
17563 dump_printf_loc (MSG_NOTE, tloc,
17564 "Will not turn target construct into a "
17565 "gridified GPGPU kernel because a lastprivate "
17566 "clause is present\n ");
17572 clauses = OMP_CLAUSE_CHAIN (clauses);
17575 struct walk_stmt_info wi;
17576 memset (&wi, 0, sizeof (wi));
17577 if (walk_gimple_seq (gimple_omp_body (gfor),
17578 grid_find_ungridifiable_statement,
17581 gimple *bad = (gimple *) wi.info;
17582 if (dump_enabled_p ())
17584 if (is_gimple_call (bad))
17585 dump_printf_loc (MSG_NOTE, tloc,
17586 "Will not turn target construct into a gridified "
17587 " GPGPU kernel because the inner loop contains "
17588 "call to a noreturn function\n");
17589 if (gimple_code (bad) == GIMPLE_OMP_FOR)
17590 dump_printf_loc (MSG_NOTE, tloc,
17591 "Will not turn target construct into a gridified "
17592 " GPGPU kernel because the inner loop contains "
17593 "a simd construct\n");
17595 dump_printf_loc (MSG_NOTE, tloc,
17596 "Will not turn target construct into a gridified "
17597 "GPGPU kernel because the inner loop contains "
17598 "statement %s which cannot be transformed\n",
17599 gimple_code_name[(int) gimple_code (bad)]);
17604 *group_size_p = group_size;
17608 /* Operand walker, used to remap pre-body declarations according to a hash map
17609 provided in DATA. */
17612 grid_remap_prebody_decls (tree *tp, int *walk_subtrees, void *data)
17616 if (DECL_P (t) || TYPE_P (t))
17617 *walk_subtrees = 0;
17619 *walk_subtrees = 1;
17621 if (TREE_CODE (t) == VAR_DECL)
17623 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
17624 hash_map<tree, tree> *declmap = (hash_map<tree, tree> *) wi->info;
17625 tree *repl = declmap->get (t);
17632 /* Copy leading register-type assignments to local variables in SRC to just
17633 before DST, Creating temporaries, adjusting mapping of operands in WI and
17634 remapping operands as necessary. Add any new temporaries to TGT_BIND.
17635 Return the first statement that does not conform to
17636 grid_reg_assignment_to_local_var_p or NULL. */
17639 grid_copy_leading_local_assignments (gimple_seq src, gimple_stmt_iterator *dst,
17640 gbind *tgt_bind, struct walk_stmt_info *wi)
17642 hash_map<tree, tree> *declmap = (hash_map<tree, tree> *) wi->info;
17643 gimple_stmt_iterator gsi;
17644 for (gsi = gsi_start (src); !gsi_end_p (gsi); gsi_next (&gsi))
17646 gimple *stmt = gsi_stmt (gsi);
17647 if (gbind *bind = dyn_cast <gbind *> (stmt))
17649 gimple *r = grid_copy_leading_local_assignments
17650 (gimple_bind_body (bind), dst, tgt_bind, wi);
17656 if (!grid_reg_assignment_to_local_var_p (stmt))
17658 tree lhs = gimple_assign_lhs (as_a <gassign *> (stmt));
17659 tree repl = copy_var_decl (lhs, create_tmp_var_name (NULL),
17661 DECL_CONTEXT (repl) = current_function_decl;
17662 gimple_bind_append_vars (tgt_bind, repl);
17664 declmap->put (lhs, repl);
17665 gassign *copy = as_a <gassign *> (gimple_copy (stmt));
17666 walk_gimple_op (copy, grid_remap_prebody_decls, wi);
17667 gsi_insert_before (dst, copy, GSI_SAME_STMT);
17672 /* Given freshly copied top level kernel SEQ, identify the individual OMP
17673 components, mark them as part of kernel and return the inner loop, and copy
17674 assignment leading to them just before DST, remapping them using WI and
17675 adding new temporaries to TGT_BIND. */
17678 grid_process_kernel_body_copy (gimple_seq seq, gimple_stmt_iterator *dst,
17679 gbind *tgt_bind, struct walk_stmt_info *wi)
17681 gimple *stmt = grid_copy_leading_local_assignments (seq, dst, tgt_bind, wi);
17682 gomp_teams *teams = dyn_cast <gomp_teams *> (stmt);
17683 gcc_assert (teams);
17684 gimple_omp_teams_set_grid_phony (teams, true);
17685 stmt = grid_copy_leading_local_assignments (gimple_omp_body (teams), dst,
17687 gcc_checking_assert (stmt);
17688 gomp_for *dist = dyn_cast <gomp_for *> (stmt);
17690 gimple_seq prebody = gimple_omp_for_pre_body (dist);
17692 grid_copy_leading_local_assignments (prebody, dst, tgt_bind, wi);
17693 gimple_omp_for_set_grid_phony (dist, true);
17694 stmt = grid_copy_leading_local_assignments (gimple_omp_body (dist), dst,
17696 gcc_checking_assert (stmt);
17698 gomp_parallel *parallel = as_a <gomp_parallel *> (stmt);
17699 gimple_omp_parallel_set_grid_phony (parallel, true);
17700 stmt = grid_copy_leading_local_assignments (gimple_omp_body (parallel), dst,
17702 gomp_for *inner_loop = as_a <gomp_for *> (stmt);
17703 gimple_omp_for_set_kind (inner_loop, GF_OMP_FOR_KIND_GRID_LOOP);
17704 prebody = gimple_omp_for_pre_body (inner_loop);
17706 grid_copy_leading_local_assignments (prebody, dst, tgt_bind, wi);
17711 /* If TARGET points to a GOMP_TARGET which follows a gridifiable pattern,
17712 create a GPU kernel for it. GSI must point to the same statement, TGT_BIND
17713 is the bind into which temporaries inserted before TARGET should be
17717 grid_attempt_target_gridification (gomp_target *target,
17718 gimple_stmt_iterator *gsi,
17722 if (!target || !grid_target_follows_gridifiable_pattern (target, &group_size))
17725 location_t loc = gimple_location (target);
17726 if (dump_enabled_p ())
17727 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, loc,
17728 "Target construct will be turned into a gridified GPGPU "
17731 /* Copy target body to a GPUKERNEL construct: */
17732 gimple_seq kernel_seq = copy_gimple_seq_and_replace_locals
17733 (gimple_omp_body (target));
17735 hash_map<tree, tree> *declmap = new hash_map<tree, tree>;
17736 struct walk_stmt_info wi;
17737 memset (&wi, 0, sizeof (struct walk_stmt_info));
17740 /* Copy assignments in between OMP statements before target, mark OMP
17741 statements within copy appropriatly. */
17742 gomp_for *inner_loop = grid_process_kernel_body_copy (kernel_seq, gsi,
17745 gbind *old_bind = as_a <gbind *> (gimple_seq_first (gimple_omp_body (target)));
17746 gbind *new_bind = as_a <gbind *> (gimple_seq_first (kernel_seq));
17747 tree new_block = gimple_bind_block (new_bind);
17748 tree enc_block = BLOCK_SUPERCONTEXT (gimple_bind_block (old_bind));
17749 BLOCK_CHAIN (new_block) = BLOCK_SUBBLOCKS (enc_block);
17750 BLOCK_SUBBLOCKS (enc_block) = new_block;
17751 BLOCK_SUPERCONTEXT (new_block) = enc_block;
17752 gimple *gpukernel = gimple_build_omp_grid_body (kernel_seq);
17753 gimple_seq_add_stmt
17754 (gimple_bind_body_ptr (as_a <gbind *> (gimple_omp_body (target))),
17757 walk_tree (&group_size, grid_remap_prebody_decls, &wi, NULL);
17758 push_gimplify_context ();
17759 size_t collapse = gimple_omp_for_collapse (inner_loop);
17760 for (size_t i = 0; i < collapse; i++)
17762 tree itype, type = TREE_TYPE (gimple_omp_for_index (inner_loop, i));
17763 if (POINTER_TYPE_P (type))
17764 itype = signed_type_for (type);
17768 enum tree_code cond_code = gimple_omp_for_cond (inner_loop, i);
17769 tree n1 = unshare_expr (gimple_omp_for_initial (inner_loop, i));
17770 walk_tree (&n1, grid_remap_prebody_decls, &wi, NULL);
17771 tree n2 = unshare_expr (gimple_omp_for_final (inner_loop, i));
17772 walk_tree (&n2, grid_remap_prebody_decls, &wi, NULL);
17773 adjust_for_condition (loc, &cond_code, &n2);
17775 step = get_omp_for_step_from_incr (loc,
17776 gimple_omp_for_incr (inner_loop, i));
17777 gimple_seq tmpseq = NULL;
17778 n1 = fold_convert (itype, n1);
17779 n2 = fold_convert (itype, n2);
17780 tree t = build_int_cst (itype, (cond_code == LT_EXPR ? -1 : 1));
17781 t = fold_build2 (PLUS_EXPR, itype, step, t);
17782 t = fold_build2 (PLUS_EXPR, itype, t, n2);
17783 t = fold_build2 (MINUS_EXPR, itype, t, n1);
17784 if (TYPE_UNSIGNED (itype) && cond_code == GT_EXPR)
17785 t = fold_build2 (TRUNC_DIV_EXPR, itype,
17786 fold_build1 (NEGATE_EXPR, itype, t),
17787 fold_build1 (NEGATE_EXPR, itype, step));
17789 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
17790 tree gs = fold_convert (uint32_type_node, t);
17791 gimplify_expr (&gs, &tmpseq, NULL, is_gimple_val, fb_rvalue);
17792 if (!gimple_seq_empty_p (tmpseq))
17793 gsi_insert_seq_before (gsi, tmpseq, GSI_SAME_STMT);
17796 if (i == 0 && group_size)
17798 ws = fold_convert (uint32_type_node, group_size);
17800 gimplify_expr (&ws, &tmpseq, NULL, is_gimple_val, fb_rvalue);
17801 if (!gimple_seq_empty_p (tmpseq))
17802 gsi_insert_seq_before (gsi, tmpseq, GSI_SAME_STMT);
17805 ws = build_zero_cst (uint32_type_node);
17807 tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__GRIDDIM_);
17808 OMP_CLAUSE__GRIDDIM__DIMENSION (c) = i;
17809 OMP_CLAUSE__GRIDDIM__SIZE (c) = gs;
17810 OMP_CLAUSE__GRIDDIM__GROUP (c) = ws;
17811 OMP_CLAUSE_CHAIN (c) = gimple_omp_target_clauses (target);
17812 gimple_omp_target_set_clauses (target, c);
17814 pop_gimplify_context (tgt_bind);
17819 /* Walker function doing all the work for create_target_kernels. */
17822 grid_gridify_all_targets_stmt (gimple_stmt_iterator *gsi,
17823 bool *handled_ops_p,
17824 struct walk_stmt_info *incoming)
17826 *handled_ops_p = false;
17828 gimple *stmt = gsi_stmt (*gsi);
17829 gomp_target *target = dyn_cast <gomp_target *> (stmt);
17832 gbind *tgt_bind = (gbind *) incoming->info;
17833 gcc_checking_assert (tgt_bind);
17834 grid_attempt_target_gridification (target, gsi, tgt_bind);
17837 gbind *bind = dyn_cast <gbind *> (stmt);
17840 *handled_ops_p = true;
17841 struct walk_stmt_info wi;
17842 memset (&wi, 0, sizeof (wi));
17844 walk_gimple_seq_mod (gimple_bind_body_ptr (bind),
17845 grid_gridify_all_targets_stmt, NULL, &wi);
17850 /* Attempt to gridify all target constructs in BODY_P. All such targets will
17851 have their bodies duplicated, with the new copy being put into a
17852 gimple_omp_grid_body statement. All kernel-related construct within the
17853 grid_body will be marked with phony flags or kernel kinds. Moreover, some
17854 re-structuring is often needed, such as copying pre-bodies before the target
17855 construct so that kernel grid sizes can be computed. */
17858 grid_gridify_all_targets (gimple_seq *body_p)
17860 struct walk_stmt_info wi;
17861 memset (&wi, 0, sizeof (wi));
17862 walk_gimple_seq_mod (body_p, grid_gridify_all_targets_stmt, NULL, &wi);
17866 /* Main entry point. */
17868 static unsigned int
17869 execute_lower_omp (void)
17875 /* This pass always runs, to provide PROP_gimple_lomp.
17876 But often, there is nothing to do. */
17877 if (flag_cilkplus == 0 && flag_openacc == 0 && flag_openmp == 0
17878 && flag_openmp_simd == 0)
17881 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
17882 delete_omp_context);
17884 body = gimple_body (current_function_decl);
17886 if (hsa_gen_requested_p ())
17887 grid_gridify_all_targets (&body);
17889 scan_omp (&body, NULL);
17890 gcc_assert (taskreg_nesting_level == 0);
17891 FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
17892 finish_taskreg_scan (ctx);
17893 taskreg_contexts.release ();
17895 if (all_contexts->root)
17897 if (task_shared_vars)
17898 push_gimplify_context ();
17899 lower_omp (&body, NULL);
17900 if (task_shared_vars)
17901 pop_gimplify_context (NULL);
17906 splay_tree_delete (all_contexts);
17907 all_contexts = NULL;
17909 BITMAP_FREE (task_shared_vars);
17915 const pass_data pass_data_lower_omp =
17917 GIMPLE_PASS, /* type */
17918 "omplower", /* name */
17919 OPTGROUP_NONE, /* optinfo_flags */
17920 TV_NONE, /* tv_id */
17921 PROP_gimple_any, /* properties_required */
17922 PROP_gimple_lomp, /* properties_provided */
17923 0, /* properties_destroyed */
17924 0, /* todo_flags_start */
17925 0, /* todo_flags_finish */
17928 class pass_lower_omp : public gimple_opt_pass
17931 pass_lower_omp (gcc::context *ctxt)
17932 : gimple_opt_pass (pass_data_lower_omp, ctxt)
17935 /* opt_pass methods: */
17936 virtual unsigned int execute (function *) { return execute_lower_omp (); }
17938 }; // class pass_lower_omp
17940 } // anon namespace
17943 make_pass_lower_omp (gcc::context *ctxt)
17945 return new pass_lower_omp (ctxt);
17948 /* The following is a utility to diagnose structured block violations.
17949 It is not part of the "omplower" pass, as that's invoked too late. It
17950 should be invoked by the respective front ends after gimplification. */
17952 static splay_tree all_labels;
17954 /* Check for mismatched contexts and generate an error if needed. Return
17955 true if an error is detected. */
17958 diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
17959 gimple *branch_ctx, gimple *label_ctx)
17961 gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx));
17962 gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx));
17964 if (label_ctx == branch_ctx)
17967 const char* kind = NULL;
17972 && gimple_code (branch_ctx) == GIMPLE_OMP_FOR
17973 && gimple_omp_for_kind (branch_ctx) == GF_OMP_FOR_KIND_CILKSIMD)
17975 && gimple_code (label_ctx) == GIMPLE_OMP_FOR
17976 && gimple_omp_for_kind (label_ctx) == GF_OMP_FOR_KIND_CILKSIMD))
17977 kind = "Cilk Plus";
17981 if ((branch_ctx && is_gimple_omp_oacc (branch_ctx))
17982 || (label_ctx && is_gimple_omp_oacc (label_ctx)))
17984 gcc_checking_assert (kind == NULL);
17990 gcc_checking_assert (flag_openmp);
17995 Previously we kept track of the label's entire context in diagnose_sb_[12]
17996 so we could traverse it and issue a correct "exit" or "enter" error
17997 message upon a structured block violation.
17999 We built the context by building a list with tree_cons'ing, but there is
18000 no easy counterpart in gimple tuples. It seems like far too much work
18001 for issuing exit/enter error messages. If someone really misses the
18002 distinct error message... patches welcome.
18006 /* Try to avoid confusing the user by producing and error message
18007 with correct "exit" or "enter" verbiage. We prefer "exit"
18008 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
18009 if (branch_ctx == NULL)
18015 if (TREE_VALUE (label_ctx) == branch_ctx)
18020 label_ctx = TREE_CHAIN (label_ctx);
18025 error ("invalid exit from %s structured block", kind);
18027 error ("invalid entry to %s structured block", kind);
18030 /* If it's obvious we have an invalid entry, be specific about the error. */
18031 if (branch_ctx == NULL)
18032 error ("invalid entry to %s structured block", kind);
18035 /* Otherwise, be vague and lazy, but efficient. */
18036 error ("invalid branch to/from %s structured block", kind);
18039 gsi_replace (gsi_p, gimple_build_nop (), false);
18043 /* Pass 1: Create a minimal tree of structured blocks, and record
18044 where each label is found. */
18047 diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
18048 struct walk_stmt_info *wi)
18050 gimple *context = (gimple *) wi->info;
18051 gimple *inner_context;
18052 gimple *stmt = gsi_stmt (*gsi_p);
18054 *handled_ops_p = true;
18056 switch (gimple_code (stmt))
18060 case GIMPLE_OMP_PARALLEL:
18061 case GIMPLE_OMP_TASK:
18062 case GIMPLE_OMP_SECTIONS:
18063 case GIMPLE_OMP_SINGLE:
18064 case GIMPLE_OMP_SECTION:
18065 case GIMPLE_OMP_MASTER:
18066 case GIMPLE_OMP_ORDERED:
18067 case GIMPLE_OMP_CRITICAL:
18068 case GIMPLE_OMP_TARGET:
18069 case GIMPLE_OMP_TEAMS:
18070 case GIMPLE_OMP_TASKGROUP:
18071 /* The minimal context here is just the current OMP construct. */
18072 inner_context = stmt;
18073 wi->info = inner_context;
18074 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
18075 wi->info = context;
18078 case GIMPLE_OMP_FOR:
18079 inner_context = stmt;
18080 wi->info = inner_context;
18081 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18083 walk_gimple_seq (gimple_omp_for_pre_body (stmt),
18084 diagnose_sb_1, NULL, wi);
18085 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
18086 wi->info = context;
18090 splay_tree_insert (all_labels,
18091 (splay_tree_key) gimple_label_label (
18092 as_a <glabel *> (stmt)),
18093 (splay_tree_value) context);
18103 /* Pass 2: Check each branch and see if its context differs from that of
18104 the destination label's context. */
18107 diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
18108 struct walk_stmt_info *wi)
18110 gimple *context = (gimple *) wi->info;
18112 gimple *stmt = gsi_stmt (*gsi_p);
18114 *handled_ops_p = true;
18116 switch (gimple_code (stmt))
18120 case GIMPLE_OMP_PARALLEL:
18121 case GIMPLE_OMP_TASK:
18122 case GIMPLE_OMP_SECTIONS:
18123 case GIMPLE_OMP_SINGLE:
18124 case GIMPLE_OMP_SECTION:
18125 case GIMPLE_OMP_MASTER:
18126 case GIMPLE_OMP_ORDERED:
18127 case GIMPLE_OMP_CRITICAL:
18128 case GIMPLE_OMP_TARGET:
18129 case GIMPLE_OMP_TEAMS:
18130 case GIMPLE_OMP_TASKGROUP:
18132 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
18133 wi->info = context;
18136 case GIMPLE_OMP_FOR:
18138 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18140 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
18141 diagnose_sb_2, NULL, wi);
18142 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
18143 wi->info = context;
18148 gcond *cond_stmt = as_a <gcond *> (stmt);
18149 tree lab = gimple_cond_true_label (cond_stmt);
18152 n = splay_tree_lookup (all_labels,
18153 (splay_tree_key) lab);
18154 diagnose_sb_0 (gsi_p, context,
18155 n ? (gimple *) n->value : NULL);
18157 lab = gimple_cond_false_label (cond_stmt);
18160 n = splay_tree_lookup (all_labels,
18161 (splay_tree_key) lab);
18162 diagnose_sb_0 (gsi_p, context,
18163 n ? (gimple *) n->value : NULL);
18170 tree lab = gimple_goto_dest (stmt);
18171 if (TREE_CODE (lab) != LABEL_DECL)
18174 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
18175 diagnose_sb_0 (gsi_p, context, n ? (gimple *) n->value : NULL);
18179 case GIMPLE_SWITCH:
18181 gswitch *switch_stmt = as_a <gswitch *> (stmt);
18183 for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i)
18185 tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i));
18186 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
18187 if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
18193 case GIMPLE_RETURN:
18194 diagnose_sb_0 (gsi_p, context, NULL);
18204 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
18207 make_gimple_omp_edges (basic_block bb, struct omp_region **region,
18210 gimple *last = last_stmt (bb);
18211 enum gimple_code code = gimple_code (last);
18212 struct omp_region *cur_region = *region;
18213 bool fallthru = false;
18217 case GIMPLE_OMP_PARALLEL:
18218 case GIMPLE_OMP_TASK:
18219 case GIMPLE_OMP_FOR:
18220 case GIMPLE_OMP_SINGLE:
18221 case GIMPLE_OMP_TEAMS:
18222 case GIMPLE_OMP_MASTER:
18223 case GIMPLE_OMP_TASKGROUP:
18224 case GIMPLE_OMP_CRITICAL:
18225 case GIMPLE_OMP_SECTION:
18226 case GIMPLE_OMP_GRID_BODY:
18227 cur_region = new_omp_region (bb, code, cur_region);
18231 case GIMPLE_OMP_ORDERED:
18232 cur_region = new_omp_region (bb, code, cur_region);
18234 if (find_omp_clause (gimple_omp_ordered_clauses
18235 (as_a <gomp_ordered *> (last)),
18236 OMP_CLAUSE_DEPEND))
18237 cur_region = cur_region->outer;
18240 case GIMPLE_OMP_TARGET:
18241 cur_region = new_omp_region (bb, code, cur_region);
18243 switch (gimple_omp_target_kind (last))
18245 case GF_OMP_TARGET_KIND_REGION:
18246 case GF_OMP_TARGET_KIND_DATA:
18247 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
18248 case GF_OMP_TARGET_KIND_OACC_KERNELS:
18249 case GF_OMP_TARGET_KIND_OACC_DATA:
18250 case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
18252 case GF_OMP_TARGET_KIND_UPDATE:
18253 case GF_OMP_TARGET_KIND_ENTER_DATA:
18254 case GF_OMP_TARGET_KIND_EXIT_DATA:
18255 case GF_OMP_TARGET_KIND_OACC_UPDATE:
18256 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
18257 case GF_OMP_TARGET_KIND_OACC_DECLARE:
18258 cur_region = cur_region->outer;
18261 gcc_unreachable ();
18265 case GIMPLE_OMP_SECTIONS:
18266 cur_region = new_omp_region (bb, code, cur_region);
18270 case GIMPLE_OMP_SECTIONS_SWITCH:
18274 case GIMPLE_OMP_ATOMIC_LOAD:
18275 case GIMPLE_OMP_ATOMIC_STORE:
18279 case GIMPLE_OMP_RETURN:
18280 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
18281 somewhere other than the next block. This will be
18283 cur_region->exit = bb;
18284 if (cur_region->type == GIMPLE_OMP_TASK)
18285 /* Add an edge corresponding to not scheduling the task
18287 make_edge (cur_region->entry, bb, EDGE_ABNORMAL);
18288 fallthru = cur_region->type != GIMPLE_OMP_SECTION;
18289 cur_region = cur_region->outer;
18292 case GIMPLE_OMP_CONTINUE:
18293 cur_region->cont = bb;
18294 switch (cur_region->type)
18296 case GIMPLE_OMP_FOR:
18297 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
18298 succs edges as abnormal to prevent splitting
18300 single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL;
18301 /* Make the loopback edge. */
18302 make_edge (bb, single_succ (cur_region->entry),
18305 /* Create an edge from GIMPLE_OMP_FOR to exit, which
18306 corresponds to the case that the body of the loop
18307 is not executed at all. */
18308 make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL);
18309 make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL);
18313 case GIMPLE_OMP_SECTIONS:
18314 /* Wire up the edges into and out of the nested sections. */
18316 basic_block switch_bb = single_succ (cur_region->entry);
18318 struct omp_region *i;
18319 for (i = cur_region->inner; i ; i = i->next)
18321 gcc_assert (i->type == GIMPLE_OMP_SECTION);
18322 make_edge (switch_bb, i->entry, 0);
18323 make_edge (i->exit, bb, EDGE_FALLTHRU);
18326 /* Make the loopback edge to the block with
18327 GIMPLE_OMP_SECTIONS_SWITCH. */
18328 make_edge (bb, switch_bb, 0);
18330 /* Make the edge from the switch to exit. */
18331 make_edge (switch_bb, bb->next_bb, 0);
18336 case GIMPLE_OMP_TASK:
18341 gcc_unreachable ();
18346 gcc_unreachable ();
18349 if (*region != cur_region)
18351 *region = cur_region;
18353 *region_idx = cur_region->entry->index;
18361 static unsigned int
18362 diagnose_omp_structured_block_errors (void)
18364 struct walk_stmt_info wi;
18365 gimple_seq body = gimple_body (current_function_decl);
18367 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
18369 memset (&wi, 0, sizeof (wi));
18370 walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
18372 memset (&wi, 0, sizeof (wi));
18373 wi.want_locations = true;
18374 walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
18376 gimple_set_body (current_function_decl, body);
18378 splay_tree_delete (all_labels);
18386 const pass_data pass_data_diagnose_omp_blocks =
18388 GIMPLE_PASS, /* type */
18389 "*diagnose_omp_blocks", /* name */
18390 OPTGROUP_NONE, /* optinfo_flags */
18391 TV_NONE, /* tv_id */
18392 PROP_gimple_any, /* properties_required */
18393 0, /* properties_provided */
18394 0, /* properties_destroyed */
18395 0, /* todo_flags_start */
18396 0, /* todo_flags_finish */
18399 class pass_diagnose_omp_blocks : public gimple_opt_pass
18402 pass_diagnose_omp_blocks (gcc::context *ctxt)
18403 : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
18406 /* opt_pass methods: */
18407 virtual bool gate (function *)
18409 return flag_cilkplus || flag_openacc || flag_openmp;
18411 virtual unsigned int execute (function *)
18413 return diagnose_omp_structured_block_errors ();
18416 }; // class pass_diagnose_omp_blocks
18418 } // anon namespace
18421 make_pass_diagnose_omp_blocks (gcc::context *ctxt)
18423 return new pass_diagnose_omp_blocks (ctxt);
18426 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
18427 adds their addresses and sizes to constructor-vector V_CTOR. */
18429 add_decls_addresses_to_decl_constructor (vec<tree, va_gc> *v_decls,
18430 vec<constructor_elt, va_gc> *v_ctor)
18432 unsigned len = vec_safe_length (v_decls);
18433 for (unsigned i = 0; i < len; i++)
18435 tree it = (*v_decls)[i];
18436 bool is_var = TREE_CODE (it) == VAR_DECL;
18439 #ifdef ACCEL_COMPILER
18440 && DECL_HAS_VALUE_EXPR_P (it)
18442 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it));
18444 tree size = NULL_TREE;
18446 size = fold_convert (const_ptr_type_node, DECL_SIZE_UNIT (it));
18450 addr = build_fold_addr_expr (it);
18453 #ifdef ACCEL_COMPILER
18454 /* For "omp declare target link" vars add address of the pointer to
18455 the target table, instead of address of the var. */
18456 tree value_expr = DECL_VALUE_EXPR (it);
18457 tree link_ptr_decl = TREE_OPERAND (value_expr, 0);
18458 varpool_node::finalize_decl (link_ptr_decl);
18459 addr = build_fold_addr_expr (link_ptr_decl);
18461 addr = build_fold_addr_expr (it);
18464 /* Most significant bit of the size marks "omp declare target link"
18465 vars in host and target tables. */
18466 unsigned HOST_WIDE_INT isize = tree_to_uhwi (size);
18467 isize |= 1ULL << (int_size_in_bytes (const_ptr_type_node)
18468 * BITS_PER_UNIT - 1);
18469 size = wide_int_to_tree (const_ptr_type_node, isize);
18472 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, addr);
18474 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, size);
18478 /* Create new symbols containing (address, size) pairs for global variables,
18479 marked with "omp declare target" attribute, as well as addresses for the
18480 functions, which are outlined offloading regions. */
18482 omp_finish_file (void)
18484 unsigned num_funcs = vec_safe_length (offload_funcs);
18485 unsigned num_vars = vec_safe_length (offload_vars);
18487 if (num_funcs == 0 && num_vars == 0)
18490 if (targetm_common.have_named_sections)
18492 vec<constructor_elt, va_gc> *v_f, *v_v;
18493 vec_alloc (v_f, num_funcs);
18494 vec_alloc (v_v, num_vars * 2);
18496 add_decls_addresses_to_decl_constructor (offload_funcs, v_f);
18497 add_decls_addresses_to_decl_constructor (offload_vars, v_v);
18499 tree vars_decl_type = build_array_type_nelts (pointer_sized_int_node,
18501 tree funcs_decl_type = build_array_type_nelts (pointer_sized_int_node,
18503 SET_TYPE_ALIGN (vars_decl_type, TYPE_ALIGN (pointer_sized_int_node));
18504 SET_TYPE_ALIGN (funcs_decl_type, TYPE_ALIGN (pointer_sized_int_node));
18505 tree ctor_v = build_constructor (vars_decl_type, v_v);
18506 tree ctor_f = build_constructor (funcs_decl_type, v_f);
18507 TREE_CONSTANT (ctor_v) = TREE_CONSTANT (ctor_f) = 1;
18508 TREE_STATIC (ctor_v) = TREE_STATIC (ctor_f) = 1;
18509 tree funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
18510 get_identifier (".offload_func_table"),
18512 tree vars_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
18513 get_identifier (".offload_var_table"),
18515 TREE_STATIC (funcs_decl) = TREE_STATIC (vars_decl) = 1;
18516 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
18517 otherwise a joint table in a binary will contain padding between
18518 tables from multiple object files. */
18519 DECL_USER_ALIGN (funcs_decl) = DECL_USER_ALIGN (vars_decl) = 1;
18520 SET_DECL_ALIGN (funcs_decl, TYPE_ALIGN (funcs_decl_type));
18521 SET_DECL_ALIGN (vars_decl, TYPE_ALIGN (vars_decl_type));
18522 DECL_INITIAL (funcs_decl) = ctor_f;
18523 DECL_INITIAL (vars_decl) = ctor_v;
18524 set_decl_section_name (funcs_decl, OFFLOAD_FUNC_TABLE_SECTION_NAME);
18525 set_decl_section_name (vars_decl, OFFLOAD_VAR_TABLE_SECTION_NAME);
18527 varpool_node::finalize_decl (vars_decl);
18528 varpool_node::finalize_decl (funcs_decl);
18532 for (unsigned i = 0; i < num_funcs; i++)
18534 tree it = (*offload_funcs)[i];
18535 targetm.record_offload_symbol (it);
18537 for (unsigned i = 0; i < num_vars; i++)
18539 tree it = (*offload_vars)[i];
18540 targetm.record_offload_symbol (it);
18545 /* Find the number of threads (POS = false), or thread number (POS =
18546 true) for an OpenACC region partitioned as MASK. Setup code
18547 required for the calculation is added to SEQ. */
18550 oacc_thread_numbers (bool pos, int mask, gimple_seq *seq)
18552 tree res = pos ? NULL_TREE : build_int_cst (unsigned_type_node, 1);
18555 /* Start at gang level, and examine relevant dimension indices. */
18556 for (ix = GOMP_DIM_GANG; ix != GOMP_DIM_MAX; ix++)
18557 if (GOMP_DIM_MASK (ix) & mask)
18559 tree arg = build_int_cst (unsigned_type_node, ix);
18563 /* We had an outer index, so scale that by the size of
18565 tree n = create_tmp_var (integer_type_node);
18567 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE, 1, arg);
18569 gimple_call_set_lhs (call, n);
18570 gimple_seq_add_stmt (seq, call);
18571 res = fold_build2 (MULT_EXPR, integer_type_node, res, n);
18575 /* Determine index in this dimension. */
18576 tree id = create_tmp_var (integer_type_node);
18577 gimple *call = gimple_build_call_internal
18578 (IFN_GOACC_DIM_POS, 1, arg);
18580 gimple_call_set_lhs (call, id);
18581 gimple_seq_add_stmt (seq, call);
18583 res = fold_build2 (PLUS_EXPR, integer_type_node, res, id);
18589 if (res == NULL_TREE)
18590 res = integer_zero_node;
18595 /* Transform IFN_GOACC_LOOP calls to actual code. See
18596 expand_oacc_for for where these are generated. At the vector
18597 level, we stride loops, such that each member of a warp will
18598 operate on adjacent iterations. At the worker and gang level,
18599 each gang/warp executes a set of contiguous iterations. Chunking
18600 can override this such that each iteration engine executes a
18601 contiguous chunk, and then moves on to stride to the next chunk. */
18604 oacc_xform_loop (gcall *call)
18606 gimple_stmt_iterator gsi = gsi_for_stmt (call);
18607 enum ifn_goacc_loop_kind code
18608 = (enum ifn_goacc_loop_kind) TREE_INT_CST_LOW (gimple_call_arg (call, 0));
18609 tree dir = gimple_call_arg (call, 1);
18610 tree range = gimple_call_arg (call, 2);
18611 tree step = gimple_call_arg (call, 3);
18612 tree chunk_size = NULL_TREE;
18613 unsigned mask = (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call, 5));
18614 tree lhs = gimple_call_lhs (call);
18615 tree type = TREE_TYPE (lhs);
18616 tree diff_type = TREE_TYPE (range);
18617 tree r = NULL_TREE;
18618 gimple_seq seq = NULL;
18619 bool chunking = false, striding = true;
18620 unsigned outer_mask = mask & (~mask + 1); // Outermost partitioning
18621 unsigned inner_mask = mask & ~outer_mask; // Inner partitioning (if any)
18623 #ifdef ACCEL_COMPILER
18624 chunk_size = gimple_call_arg (call, 4);
18625 if (integer_minus_onep (chunk_size) /* Force static allocation. */
18626 || integer_zerop (chunk_size)) /* Default (also static). */
18628 /* If we're at the gang level, we want each to execute a
18629 contiguous run of iterations. Otherwise we want each element
18631 striding = !(outer_mask & GOMP_DIM_MASK (GOMP_DIM_GANG));
18636 /* Chunk of size 1 is striding. */
18637 striding = integer_onep (chunk_size);
18638 chunking = !striding;
18642 /* striding=true, chunking=true
18644 striding=true, chunking=false
18646 striding=false,chunking=true
18647 -> chunks=ceil (range/(chunksize*threads*step))
18648 striding=false,chunking=false
18649 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
18650 push_gimplify_context (true);
18654 default: gcc_unreachable ();
18656 case IFN_GOACC_LOOP_CHUNKS:
18658 r = build_int_cst (type, 1);
18662 = (range - dir) / (chunks * step * num_threads) + dir */
18663 tree per = oacc_thread_numbers (false, mask, &seq);
18664 per = fold_convert (type, per);
18665 chunk_size = fold_convert (type, chunk_size);
18666 per = fold_build2 (MULT_EXPR, type, per, chunk_size);
18667 per = fold_build2 (MULT_EXPR, type, per, step);
18668 r = build2 (MINUS_EXPR, type, range, dir);
18669 r = build2 (PLUS_EXPR, type, r, per);
18670 r = build2 (TRUNC_DIV_EXPR, type, r, per);
18674 case IFN_GOACC_LOOP_STEP:
18676 /* If striding, step by the entire compute volume, otherwise
18677 step by the inner volume. */
18678 unsigned volume = striding ? mask : inner_mask;
18680 r = oacc_thread_numbers (false, volume, &seq);
18681 r = build2 (MULT_EXPR, type, fold_convert (type, r), step);
18685 case IFN_GOACC_LOOP_OFFSET:
18688 r = oacc_thread_numbers (true, mask, &seq);
18689 r = fold_convert (diff_type, r);
18693 tree inner_size = oacc_thread_numbers (false, inner_mask, &seq);
18694 tree outer_size = oacc_thread_numbers (false, outer_mask, &seq);
18695 tree volume = fold_build2 (MULT_EXPR, TREE_TYPE (inner_size),
18696 inner_size, outer_size);
18698 volume = fold_convert (diff_type, volume);
18700 chunk_size = fold_convert (diff_type, chunk_size);
18703 tree per = fold_build2 (MULT_EXPR, diff_type, volume, step);
18705 chunk_size = build2 (MINUS_EXPR, diff_type, range, dir);
18706 chunk_size = build2 (PLUS_EXPR, diff_type, chunk_size, per);
18707 chunk_size = build2 (TRUNC_DIV_EXPR, diff_type, chunk_size, per);
18710 tree span = build2 (MULT_EXPR, diff_type, chunk_size,
18711 fold_convert (diff_type, inner_size));
18712 r = oacc_thread_numbers (true, outer_mask, &seq);
18713 r = fold_convert (diff_type, r);
18714 r = build2 (MULT_EXPR, diff_type, r, span);
18716 tree inner = oacc_thread_numbers (true, inner_mask, &seq);
18717 inner = fold_convert (diff_type, inner);
18718 r = fold_build2 (PLUS_EXPR, diff_type, r, inner);
18722 tree chunk = fold_convert (diff_type, gimple_call_arg (call, 6));
18724 = fold_build2 (MULT_EXPR, diff_type, volume, chunk_size);
18725 per = build2 (MULT_EXPR, diff_type, per, chunk);
18727 r = build2 (PLUS_EXPR, diff_type, r, per);
18730 r = fold_build2 (MULT_EXPR, diff_type, r, step);
18731 if (type != diff_type)
18732 r = fold_convert (type, r);
18735 case IFN_GOACC_LOOP_BOUND:
18740 tree inner_size = oacc_thread_numbers (false, inner_mask, &seq);
18741 tree outer_size = oacc_thread_numbers (false, outer_mask, &seq);
18742 tree volume = fold_build2 (MULT_EXPR, TREE_TYPE (inner_size),
18743 inner_size, outer_size);
18745 volume = fold_convert (diff_type, volume);
18747 chunk_size = fold_convert (diff_type, chunk_size);
18750 tree per = fold_build2 (MULT_EXPR, diff_type, volume, step);
18752 chunk_size = build2 (MINUS_EXPR, diff_type, range, dir);
18753 chunk_size = build2 (PLUS_EXPR, diff_type, chunk_size, per);
18754 chunk_size = build2 (TRUNC_DIV_EXPR, diff_type, chunk_size, per);
18757 tree span = build2 (MULT_EXPR, diff_type, chunk_size,
18758 fold_convert (diff_type, inner_size));
18760 r = fold_build2 (MULT_EXPR, diff_type, span, step);
18762 tree offset = gimple_call_arg (call, 6);
18763 r = build2 (PLUS_EXPR, diff_type, r,
18764 fold_convert (diff_type, offset));
18765 r = build2 (integer_onep (dir) ? MIN_EXPR : MAX_EXPR,
18766 diff_type, r, range);
18768 if (diff_type != type)
18769 r = fold_convert (type, r);
18773 gimplify_assign (lhs, r, &seq);
18775 pop_gimplify_context (NULL);
18777 gsi_replace_with_seq (&gsi, seq, true);
18780 /* Default partitioned and minimum partitioned dimensions. */
18782 static int oacc_default_dims[GOMP_DIM_MAX];
18783 static int oacc_min_dims[GOMP_DIM_MAX];
18785 /* Parse the default dimension parameter. This is a set of
18786 :-separated optional compute dimensions. Each specified dimension
18787 is a positive integer. When device type support is added, it is
18788 planned to be a comma separated list of such compute dimensions,
18789 with all but the first prefixed by the colon-terminated device
18793 oacc_parse_default_dims (const char *dims)
18797 for (ix = GOMP_DIM_MAX; ix--;)
18799 oacc_default_dims[ix] = -1;
18800 oacc_min_dims[ix] = 1;
18803 #ifndef ACCEL_COMPILER
18804 /* Cannot be overridden on the host. */
18809 const char *pos = dims;
18811 for (ix = 0; *pos && ix != GOMP_DIM_MAX; ix++)
18826 val = strtol (pos, CONST_CAST (char **, &eptr), 10);
18827 if (errno || val <= 0 || (int) val != val)
18830 oacc_default_dims[ix] = (int) val;
18836 error_at (UNKNOWN_LOCATION,
18837 "-fopenacc-dim operand is malformed at '%s'", pos);
18841 /* Allow the backend to validate the dimensions. */
18842 targetm.goacc.validate_dims (NULL_TREE, oacc_default_dims, -1);
18843 targetm.goacc.validate_dims (NULL_TREE, oacc_min_dims, -2);
18846 /* Validate and update the dimensions for offloaded FN. ATTRS is the
18847 raw attribute. DIMS is an array of dimensions, which is filled in.
18848 LEVEL is the partitioning level of a routine, or -1 for an offload
18849 region itself. USED is the mask of partitioned execution in the
18853 oacc_validate_dims (tree fn, tree attrs, int *dims, int level, unsigned used)
18855 tree purpose[GOMP_DIM_MAX];
18857 tree pos = TREE_VALUE (attrs);
18858 bool is_kernel = oacc_fn_attrib_kernels_p (attrs);
18860 /* Make sure the attribute creator attached the dimension
18864 for (ix = 0; ix != GOMP_DIM_MAX; ix++)
18866 purpose[ix] = TREE_PURPOSE (pos);
18867 tree val = TREE_VALUE (pos);
18868 dims[ix] = val ? TREE_INT_CST_LOW (val) : -1;
18869 pos = TREE_CHAIN (pos);
18872 bool changed = targetm.goacc.validate_dims (fn, dims, level);
18874 /* Default anything left to 1 or a partitioned default. */
18875 for (ix = 0; ix != GOMP_DIM_MAX; ix++)
18878 /* The OpenACC spec says 'If the [num_gangs] clause is not
18879 specified, an implementation-defined default will be used;
18880 the default may depend on the code within the construct.'
18881 (2.5.6). Thus an implementation is free to choose
18882 non-unity default for a parallel region that doesn't have
18883 any gang-partitioned loops. However, it appears that there
18884 is a sufficient body of user code that expects non-gang
18885 partitioned regions to not execute in gang-redundant mode.
18886 So we (a) don't warn about the non-portability and (b) pick
18887 the minimum permissible dimension size when there is no
18888 partitioned execution. Otherwise we pick the global
18889 default for the dimension, which the user can control. The
18890 same wording and logic applies to num_workers and
18891 vector_length, however the worker- or vector- single
18892 execution doesn't have the same impact as gang-redundant
18893 execution. (If the minimum gang-level partioning is not 1,
18894 the target is probably too confusing.) */
18895 dims[ix] = (used & GOMP_DIM_MASK (ix)
18896 ? oacc_default_dims[ix] : oacc_min_dims[ix]);
18902 /* Replace the attribute with new values. */
18904 for (ix = GOMP_DIM_MAX; ix--;)
18906 pos = tree_cons (purpose[ix],
18907 build_int_cst (integer_type_node, dims[ix]),
18910 TREE_PUBLIC (pos) = 1;
18912 replace_oacc_fn_attrib (fn, pos);
18916 /* Create an empty OpenACC loop structure at LOC. */
18919 new_oacc_loop_raw (oacc_loop *parent, location_t loc)
18921 oacc_loop *loop = XCNEW (oacc_loop);
18923 loop->parent = parent;
18924 loop->child = loop->sibling = NULL;
18928 loop->sibling = parent->child;
18929 parent->child = loop;
18933 loop->marker = NULL;
18934 memset (loop->heads, 0, sizeof (loop->heads));
18935 memset (loop->tails, 0, sizeof (loop->tails));
18936 loop->routine = NULL_TREE;
18938 loop->mask = loop->flags = 0;
18940 loop->chunk_size = 0;
18941 loop->head_end = NULL;
18946 /* Create an outermost, dummy OpenACC loop for offloaded function
18950 new_oacc_loop_outer (tree decl)
18952 return new_oacc_loop_raw (NULL, DECL_SOURCE_LOCATION (decl));
18955 /* Start a new OpenACC loop structure beginning at head marker HEAD.
18956 Link into PARENT loop. Return the new loop. */
18959 new_oacc_loop (oacc_loop *parent, gcall *marker)
18961 oacc_loop *loop = new_oacc_loop_raw (parent, gimple_location (marker));
18963 loop->marker = marker;
18965 /* TODO: This is where device_type flattening would occur for the loop
18968 loop->flags = TREE_INT_CST_LOW (gimple_call_arg (marker, 3));
18970 tree chunk_size = integer_zero_node;
18971 if (loop->flags & OLF_GANG_STATIC)
18972 chunk_size = gimple_call_arg (marker, 4);
18973 loop->chunk_size = chunk_size;
18978 /* Create a dummy loop encompassing a call to a openACC routine.
18979 Extract the routine's partitioning requirements. */
18982 new_oacc_loop_routine (oacc_loop *parent, gcall *call, tree decl, tree attrs)
18984 oacc_loop *loop = new_oacc_loop_raw (parent, gimple_location (call));
18985 int level = oacc_fn_attrib_level (attrs);
18987 gcc_assert (level >= 0);
18989 loop->marker = call;
18990 loop->routine = decl;
18991 loop->mask = ((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1)
18992 ^ (GOMP_DIM_MASK (level) - 1));
18995 /* Finish off the current OpenACC loop ending at tail marker TAIL.
18996 Return the parent loop. */
18999 finish_oacc_loop (oacc_loop *loop)
19001 /* If the loop has been collapsed, don't partition it. */
19003 loop->mask = loop->flags = 0;
19004 return loop->parent;
19007 /* Free all OpenACC loop structures within LOOP (inclusive). */
19010 free_oacc_loop (oacc_loop *loop)
19013 free_oacc_loop (loop->sibling);
19015 free_oacc_loop (loop->child);
19020 /* Dump out the OpenACC loop head or tail beginning at FROM. */
19023 dump_oacc_loop_part (FILE *file, gcall *from, int depth,
19024 const char *title, int level)
19026 enum ifn_unique_kind kind
19027 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (from, 0));
19029 fprintf (file, "%*s%s-%d:\n", depth * 2, "", title, level);
19030 for (gimple_stmt_iterator gsi = gsi_for_stmt (from);;)
19032 gimple *stmt = gsi_stmt (gsi);
19034 if (is_gimple_call (stmt)
19035 && gimple_call_internal_p (stmt)
19036 && gimple_call_internal_fn (stmt) == IFN_UNIQUE)
19038 enum ifn_unique_kind k
19039 = ((enum ifn_unique_kind) TREE_INT_CST_LOW
19040 (gimple_call_arg (stmt, 0)));
19042 if (k == kind && stmt != from)
19045 print_gimple_stmt (file, stmt, depth * 2 + 2, 0);
19048 while (gsi_end_p (gsi))
19049 gsi = gsi_start_bb (single_succ (gsi_bb (gsi)));
19053 /* Dump OpenACC loops LOOP, its siblings and its children. */
19056 dump_oacc_loop (FILE *file, oacc_loop *loop, int depth)
19060 fprintf (file, "%*sLoop %x(%x) %s:%u\n", depth * 2, "",
19061 loop->flags, loop->mask,
19062 LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc));
19065 print_gimple_stmt (file, loop->marker, depth * 2, 0);
19068 fprintf (file, "%*sRoutine %s:%u:%s\n",
19069 depth * 2, "", DECL_SOURCE_FILE (loop->routine),
19070 DECL_SOURCE_LINE (loop->routine),
19071 IDENTIFIER_POINTER (DECL_NAME (loop->routine)));
19073 for (ix = GOMP_DIM_GANG; ix != GOMP_DIM_MAX; ix++)
19074 if (loop->heads[ix])
19075 dump_oacc_loop_part (file, loop->heads[ix], depth, "Head", ix);
19076 for (ix = GOMP_DIM_MAX; ix--;)
19077 if (loop->tails[ix])
19078 dump_oacc_loop_part (file, loop->tails[ix], depth, "Tail", ix);
19081 dump_oacc_loop (file, loop->child, depth + 1);
19083 dump_oacc_loop (file, loop->sibling, depth);
19086 void debug_oacc_loop (oacc_loop *);
19088 /* Dump loops to stderr. */
19090 DEBUG_FUNCTION void
19091 debug_oacc_loop (oacc_loop *loop)
19093 dump_oacc_loop (stderr, loop, 0);
19096 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
19097 structures as we go. By construction these loops are properly
19101 oacc_loop_discover_walk (oacc_loop *loop, basic_block bb)
19106 if (bb->flags & BB_VISITED)
19110 bb->flags |= BB_VISITED;
19112 /* Scan for loop markers. */
19113 for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
19116 gimple *stmt = gsi_stmt (gsi);
19118 if (!is_gimple_call (stmt))
19121 gcall *call = as_a <gcall *> (stmt);
19123 /* If this is a routine, make a dummy loop for it. */
19124 if (tree decl = gimple_call_fndecl (call))
19125 if (tree attrs = get_oacc_fn_attrib (decl))
19127 gcc_assert (!marker);
19128 new_oacc_loop_routine (loop, call, decl, attrs);
19131 if (!gimple_call_internal_p (call))
19134 switch (gimple_call_internal_fn (call))
19139 case IFN_GOACC_LOOP:
19140 /* Count the goacc loop abstraction fns, to determine if the
19141 loop was collapsed already. */
19146 enum ifn_unique_kind kind
19147 = (enum ifn_unique_kind) (TREE_INT_CST_LOW
19148 (gimple_call_arg (call, 0)));
19149 if (kind == IFN_UNIQUE_OACC_HEAD_MARK
19150 || kind == IFN_UNIQUE_OACC_TAIL_MARK)
19152 if (gimple_call_num_args (call) == 2)
19154 gcc_assert (marker && !remaining);
19156 if (kind == IFN_UNIQUE_OACC_TAIL_MARK)
19157 loop = finish_oacc_loop (loop);
19159 loop->head_end = call;
19163 int count = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
19167 if (kind == IFN_UNIQUE_OACC_HEAD_MARK)
19168 loop = new_oacc_loop (loop, call);
19171 gcc_assert (count == remaining);
19175 if (kind == IFN_UNIQUE_OACC_HEAD_MARK)
19176 loop->heads[marker] = call;
19178 loop->tails[remaining] = call;
19185 if (remaining || marker)
19187 bb = single_succ (bb);
19188 gcc_assert (single_pred_p (bb) && !(bb->flags & BB_VISITED));
19192 /* Walk successor blocks. */
19196 FOR_EACH_EDGE (e, ei, bb->succs)
19197 oacc_loop_discover_walk (loop, e->dest);
19200 /* LOOP is the first sibling. Reverse the order in place and return
19201 the new first sibling. Recurse to child loops. */
19204 oacc_loop_sibling_nreverse (oacc_loop *loop)
19206 oacc_loop *last = NULL;
19210 loop->child = oacc_loop_sibling_nreverse (loop->child);
19212 oacc_loop *next = loop->sibling;
19213 loop->sibling = last;
19222 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
19223 the current function. */
19226 oacc_loop_discovery ()
19228 /* Clear basic block flags, in particular BB_VISITED which we're going to use
19229 in the following. */
19232 oacc_loop *top = new_oacc_loop_outer (current_function_decl);
19233 oacc_loop_discover_walk (top, ENTRY_BLOCK_PTR_FOR_FN (cfun));
19235 /* The siblings were constructed in reverse order, reverse them so
19236 that diagnostics come out in an unsurprising order. */
19237 top = oacc_loop_sibling_nreverse (top);
19239 /* Clear basic block flags again. */
19245 /* Transform the abstract internal function markers starting at FROM
19246 to be for partitioning level LEVEL. Stop when we meet another HEAD
19250 oacc_loop_xform_head_tail (gcall *from, int level)
19252 enum ifn_unique_kind kind
19253 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (from, 0));
19254 tree replacement = build_int_cst (unsigned_type_node, level);
19256 for (gimple_stmt_iterator gsi = gsi_for_stmt (from);;)
19258 gimple *stmt = gsi_stmt (gsi);
19260 if (is_gimple_call (stmt)
19261 && gimple_call_internal_p (stmt)
19262 && gimple_call_internal_fn (stmt) == IFN_UNIQUE)
19264 enum ifn_unique_kind k
19265 = ((enum ifn_unique_kind)
19266 TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
19268 if (k == IFN_UNIQUE_OACC_FORK || k == IFN_UNIQUE_OACC_JOIN)
19269 *gimple_call_arg_ptr (stmt, 2) = replacement;
19270 else if (k == kind && stmt != from)
19273 else if (is_gimple_call (stmt)
19274 && gimple_call_internal_p (stmt)
19275 && gimple_call_internal_fn (stmt) == IFN_GOACC_REDUCTION)
19276 *gimple_call_arg_ptr (stmt, 3) = replacement;
19279 while (gsi_end_p (gsi))
19280 gsi = gsi_start_bb (single_succ (gsi_bb (gsi)));
19284 /* Transform the IFN_GOACC_LOOP internal functions by providing the
19285 determined partitioning mask and chunking argument. END_MARKER
19286 points at the end IFN_HEAD_TAIL call intgroducing the loop. IFNS
19287 is the number of IFN_GOACC_LOOP calls for the loop. MASK_ARG is
19288 the replacement partitioning mask and CHUNK_ARG is the replacement
19292 oacc_loop_xform_loop (gcall *end_marker, unsigned ifns,
19293 tree mask_arg, tree chunk_arg)
19295 gimple_stmt_iterator gsi = gsi_for_stmt (end_marker);
19297 gcc_checking_assert (ifns);
19300 for (; !gsi_end_p (gsi); gsi_next (&gsi))
19302 gimple *stmt = gsi_stmt (gsi);
19304 if (!is_gimple_call (stmt))
19307 gcall *call = as_a <gcall *> (stmt);
19309 if (!gimple_call_internal_p (call))
19312 if (gimple_call_internal_fn (call) != IFN_GOACC_LOOP)
19315 *gimple_call_arg_ptr (call, 5) = mask_arg;
19316 *gimple_call_arg_ptr (call, 4) = chunk_arg;
19322 /* The LOOP_BOUND ifn could be in the single successor
19324 basic_block bb = single_succ (gsi_bb (gsi));
19325 gsi = gsi_start_bb (bb);
19329 /* Process the discovered OpenACC loops, setting the correct
19330 partitioning level etc. */
19333 oacc_loop_process (oacc_loop *loop)
19336 oacc_loop_process (loop->child);
19338 if (loop->mask && !loop->routine)
19341 unsigned mask = loop->mask;
19342 unsigned dim = GOMP_DIM_GANG;
19343 tree mask_arg = build_int_cst (unsigned_type_node, mask);
19344 tree chunk_arg = loop->chunk_size;
19346 oacc_loop_xform_loop (loop->head_end, loop->ifns, mask_arg, chunk_arg);
19348 for (ix = 0; ix != GOMP_DIM_MAX && loop->heads[ix]; ix++)
19352 while (!(GOMP_DIM_MASK (dim) & mask))
19355 oacc_loop_xform_head_tail (loop->heads[ix], dim);
19356 oacc_loop_xform_head_tail (loop->tails[ix], dim);
19358 mask ^= GOMP_DIM_MASK (dim);
19363 oacc_loop_process (loop->sibling);
19366 /* Walk the OpenACC loop heirarchy checking and assigning the
19367 programmer-specified partitionings. OUTER_MASK is the partitioning
19368 this loop is contained within. Return mask of partitioning
19369 encountered. If any auto loops are discovered, set GOMP_DIM_MAX
19373 oacc_loop_fixed_partitions (oacc_loop *loop, unsigned outer_mask)
19375 unsigned this_mask = loop->mask;
19376 unsigned mask_all = 0;
19379 #ifdef ACCEL_COMPILER
19380 /* When device_type is supported, we want the device compiler to be
19381 noisy, if the loop parameters are device_type-specific. */
19385 if (!loop->routine)
19387 bool auto_par = (loop->flags & OLF_AUTO) != 0;
19388 bool seq_par = (loop->flags & OLF_SEQ) != 0;
19390 this_mask = ((loop->flags >> OLF_DIM_BASE)
19391 & (GOMP_DIM_MASK (GOMP_DIM_MAX) - 1));
19393 if ((this_mask != 0) + auto_par + seq_par > 1)
19396 error_at (loop->loc,
19398 ? "%<seq%> overrides other OpenACC loop specifiers"
19399 : "%<auto%> conflicts with other OpenACC loop specifiers");
19401 loop->flags &= ~OLF_AUTO;
19405 ~((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1) << OLF_DIM_BASE);
19409 if (auto_par && (loop->flags & OLF_INDEPENDENT))
19410 mask_all |= GOMP_DIM_MASK (GOMP_DIM_MAX);
19413 if (this_mask & outer_mask)
19415 const oacc_loop *outer;
19416 for (outer = loop->parent; outer; outer = outer->parent)
19417 if (outer->mask & this_mask)
19424 error_at (loop->loc,
19425 "%s uses same OpenACC parallelism as containing loop",
19426 loop->routine ? "routine call" : "inner loop");
19427 inform (outer->loc, "containing loop here");
19430 error_at (loop->loc,
19431 "%s uses OpenACC parallelism disallowed by containing routine",
19432 loop->routine ? "routine call" : "loop");
19435 inform (DECL_SOURCE_LOCATION (loop->routine),
19436 "routine %qD declared here", loop->routine);
19438 this_mask &= ~outer_mask;
19442 unsigned outermost = this_mask & -this_mask;
19444 if (outermost && outermost <= outer_mask)
19448 error_at (loop->loc,
19449 "incorrectly nested OpenACC loop parallelism");
19451 const oacc_loop *outer;
19452 for (outer = loop->parent;
19453 outer->flags && outer->flags < outermost;
19454 outer = outer->parent)
19456 inform (outer->loc, "containing loop here");
19459 this_mask &= ~outermost;
19463 loop->mask = this_mask;
19464 mask_all |= this_mask;
19467 mask_all |= oacc_loop_fixed_partitions (loop->child,
19468 outer_mask | this_mask);
19471 mask_all |= oacc_loop_fixed_partitions (loop->sibling, outer_mask);
19476 /* Walk the OpenACC loop heirarchy to assign auto-partitioned loops.
19477 OUTER_MASK is the partitioning this loop is contained within.
19478 Return the cumulative partitioning used by this loop, siblings and
19482 oacc_loop_auto_partitions (oacc_loop *loop, unsigned outer_mask)
19484 unsigned inner_mask = 0;
19487 #ifdef ACCEL_COMPILER
19488 /* When device_type is supported, we want the device compiler to be
19489 noisy, if the loop parameters are device_type-specific. */
19494 inner_mask |= oacc_loop_auto_partitions (loop->child,
19495 outer_mask | loop->mask);
19497 if ((loop->flags & OLF_AUTO) && (loop->flags & OLF_INDEPENDENT))
19499 unsigned this_mask = 0;
19501 /* Determine the outermost partitioning used within this loop. */
19502 this_mask = inner_mask | GOMP_DIM_MASK (GOMP_DIM_MAX);
19503 this_mask = (this_mask & -this_mask);
19505 /* Pick the partitioning just inside that one. */
19508 /* And avoid picking one use by an outer loop. */
19509 this_mask &= ~outer_mask;
19511 if (!this_mask && noisy)
19512 warning_at (loop->loc, 0,
19513 "insufficient partitioning available to parallelize loop");
19516 fprintf (dump_file, "Auto loop %s:%d assigned %d\n",
19517 LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc),
19520 loop->mask = this_mask;
19522 inner_mask |= loop->mask;
19525 inner_mask |= oacc_loop_auto_partitions (loop->sibling, outer_mask);
19530 /* Walk the OpenACC loop heirarchy to check and assign partitioning
19531 axes. Return mask of partitioning. */
19534 oacc_loop_partition (oacc_loop *loop, unsigned outer_mask)
19536 unsigned mask_all = oacc_loop_fixed_partitions (loop, outer_mask);
19538 if (mask_all & GOMP_DIM_MASK (GOMP_DIM_MAX))
19540 mask_all ^= GOMP_DIM_MASK (GOMP_DIM_MAX);
19541 mask_all |= oacc_loop_auto_partitions (loop, outer_mask);
19546 /* Default fork/join early expander. Delete the function calls if
19547 there is no RTL expander. */
19550 default_goacc_fork_join (gcall *ARG_UNUSED (call),
19551 const int *ARG_UNUSED (dims), bool is_fork)
19554 return targetm.have_oacc_fork ();
19556 return targetm.have_oacc_join ();
19559 /* Default goacc.reduction early expander.
19561 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
19562 If RES_PTR is not integer-zerop:
19563 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
19564 TEARDOWN - emit '*RES_PTR = VAR'
19566 emit 'LHS = VAR' */
19569 default_goacc_reduction (gcall *call)
19571 unsigned code = (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call, 0));
19572 gimple_stmt_iterator gsi = gsi_for_stmt (call);
19573 tree lhs = gimple_call_lhs (call);
19574 tree var = gimple_call_arg (call, 2);
19575 gimple_seq seq = NULL;
19577 if (code == IFN_GOACC_REDUCTION_SETUP
19578 || code == IFN_GOACC_REDUCTION_TEARDOWN)
19580 /* Setup and Teardown need to copy from/to the receiver object,
19581 if there is one. */
19582 tree ref_to_res = gimple_call_arg (call, 1);
19584 if (!integer_zerop (ref_to_res))
19586 tree dst = build_simple_mem_ref (ref_to_res);
19589 if (code == IFN_GOACC_REDUCTION_SETUP)
19595 gimple_seq_add_stmt (&seq, gimple_build_assign (dst, src));
19599 /* Copy VAR to LHS, if there is an LHS. */
19601 gimple_seq_add_stmt (&seq, gimple_build_assign (lhs, var));
19603 gsi_replace_with_seq (&gsi, seq, true);
19606 /* Main entry point for oacc transformations which run on the device
19607 compiler after LTO, so we know what the target device is at this
19608 point (including the host fallback). */
19610 static unsigned int
19611 execute_oacc_device_lower ()
19613 tree attrs = get_oacc_fn_attrib (current_function_decl);
19616 /* Not an offloaded function. */
19619 /* Parse the default dim argument exactly once. */
19620 if ((const void *)flag_openacc_dims != &flag_openacc_dims)
19622 oacc_parse_default_dims (flag_openacc_dims);
19623 flag_openacc_dims = (char *)&flag_openacc_dims;
19626 /* Discover, partition and process the loops. */
19627 oacc_loop *loops = oacc_loop_discovery ();
19628 int fn_level = oacc_fn_attrib_level (attrs);
19631 fprintf (dump_file, oacc_fn_attrib_kernels_p (attrs)
19632 ? "Function is kernels offload\n"
19633 : fn_level < 0 ? "Function is parallel offload\n"
19634 : "Function is routine level %d\n", fn_level);
19636 unsigned outer_mask = fn_level >= 0 ? GOMP_DIM_MASK (fn_level) - 1 : 0;
19637 unsigned used_mask = oacc_loop_partition (loops, outer_mask);
19638 int dims[GOMP_DIM_MAX];
19640 oacc_validate_dims (current_function_decl, attrs, dims, fn_level, used_mask);
19644 const char *comma = "Compute dimensions [";
19645 for (int ix = 0; ix != GOMP_DIM_MAX; ix++, comma = ", ")
19646 fprintf (dump_file, "%s%d", comma, dims[ix]);
19647 fprintf (dump_file, "]\n");
19650 oacc_loop_process (loops);
19653 fprintf (dump_file, "OpenACC loops\n");
19654 dump_oacc_loop (dump_file, loops, 0);
19655 fprintf (dump_file, "\n");
19658 /* Offloaded targets may introduce new basic blocks, which require
19659 dominance information to update SSA. */
19660 calculate_dominance_info (CDI_DOMINATORS);
19662 /* Now lower internal loop functions to target-specific code
19665 FOR_ALL_BB_FN (bb, cfun)
19666 for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
19668 gimple *stmt = gsi_stmt (gsi);
19669 if (!is_gimple_call (stmt))
19675 gcall *call = as_a <gcall *> (stmt);
19676 if (!gimple_call_internal_p (call))
19682 /* Rewind to allow rescan. */
19684 bool rescan = false, remove = false;
19685 enum internal_fn ifn_code = gimple_call_internal_fn (call);
19691 case IFN_GOACC_LOOP:
19692 oacc_xform_loop (call);
19696 case IFN_GOACC_REDUCTION:
19697 /* Mark the function for SSA renaming. */
19698 mark_virtual_operands_for_renaming (cfun);
19700 /* If the level is -1, this ended up being an unused
19701 axis. Handle as a default. */
19702 if (integer_minus_onep (gimple_call_arg (call, 3)))
19703 default_goacc_reduction (call);
19705 targetm.goacc.reduction (call);
19711 enum ifn_unique_kind kind
19712 = ((enum ifn_unique_kind)
19713 TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
19718 gcc_unreachable ();
19720 case IFN_UNIQUE_OACC_FORK:
19721 case IFN_UNIQUE_OACC_JOIN:
19722 if (integer_minus_onep (gimple_call_arg (call, 2)))
19724 else if (!targetm.goacc.fork_join
19725 (call, dims, kind == IFN_UNIQUE_OACC_FORK))
19729 case IFN_UNIQUE_OACC_HEAD_MARK:
19730 case IFN_UNIQUE_OACC_TAIL_MARK:
19738 if (gsi_end_p (gsi))
19739 /* We rewound past the beginning of the BB. */
19740 gsi = gsi_start_bb (bb);
19742 /* Undo the rewind. */
19747 if (gimple_vdef (call))
19748 replace_uses_by (gimple_vdef (call), gimple_vuse (call));
19749 if (gimple_call_lhs (call))
19751 /* Propagate the data dependency var. */
19752 gimple *ass = gimple_build_assign (gimple_call_lhs (call),
19753 gimple_call_arg (call, 1));
19754 gsi_replace (&gsi, ass, false);
19757 gsi_remove (&gsi, true);
19760 /* If not rescanning, advance over the call. */
19764 free_oacc_loop (loops);
19769 /* Default launch dimension validator. Force everything to 1. A
19770 backend that wants to provide larger dimensions must override this
19774 default_goacc_validate_dims (tree ARG_UNUSED (decl), int *dims,
19775 int ARG_UNUSED (fn_level))
19777 bool changed = false;
19779 for (unsigned ix = 0; ix != GOMP_DIM_MAX; ix++)
19791 /* Default dimension bound is unknown on accelerator and 1 on host. */
19794 default_goacc_dim_limit (int ARG_UNUSED (axis))
19796 #ifdef ACCEL_COMPILER
19805 const pass_data pass_data_oacc_device_lower =
19807 GIMPLE_PASS, /* type */
19808 "oaccdevlow", /* name */
19809 OPTGROUP_NONE, /* optinfo_flags */
19810 TV_NONE, /* tv_id */
19811 PROP_cfg, /* properties_required */
19812 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
19813 0, /* properties_destroyed */
19814 0, /* todo_flags_start */
19815 TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */
19818 class pass_oacc_device_lower : public gimple_opt_pass
19821 pass_oacc_device_lower (gcc::context *ctxt)
19822 : gimple_opt_pass (pass_data_oacc_device_lower, ctxt)
19825 /* opt_pass methods: */
19826 virtual unsigned int execute (function *)
19828 bool gate = flag_openacc != 0;
19833 return execute_oacc_device_lower ();
19836 }; // class pass_oacc_device_lower
19838 } // anon namespace
19841 make_pass_oacc_device_lower (gcc::context *ctxt)
19843 return new pass_oacc_device_lower (ctxt);
19846 /* "omp declare target link" handling pass. */
19850 const pass_data pass_data_omp_target_link =
19852 GIMPLE_PASS, /* type */
19853 "omptargetlink", /* name */
19854 OPTGROUP_NONE, /* optinfo_flags */
19855 TV_NONE, /* tv_id */
19856 PROP_ssa, /* properties_required */
19857 0, /* properties_provided */
19858 0, /* properties_destroyed */
19859 0, /* todo_flags_start */
19860 TODO_update_ssa, /* todo_flags_finish */
19863 class pass_omp_target_link : public gimple_opt_pass
19866 pass_omp_target_link (gcc::context *ctxt)
19867 : gimple_opt_pass (pass_data_omp_target_link, ctxt)
19870 /* opt_pass methods: */
19871 virtual bool gate (function *fun)
19873 #ifdef ACCEL_COMPILER
19874 tree attrs = DECL_ATTRIBUTES (fun->decl);
19875 return lookup_attribute ("omp declare target", attrs)
19876 || lookup_attribute ("omp target entrypoint", attrs);
19883 virtual unsigned execute (function *);
19886 /* Callback for walk_gimple_stmt used to scan for link var operands. */
19889 find_link_var_op (tree *tp, int *walk_subtrees, void *)
19893 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t)
19894 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t)))
19896 *walk_subtrees = 0;
19904 pass_omp_target_link::execute (function *fun)
19907 FOR_EACH_BB_FN (bb, fun)
19909 gimple_stmt_iterator gsi;
19910 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
19911 if (walk_gimple_stmt (&gsi, NULL, find_link_var_op, NULL))
19912 gimple_regimplify_operands (gsi_stmt (gsi), &gsi);
19918 } // anon namespace
19921 make_pass_omp_target_link (gcc::context *ctxt)
19923 return new pass_omp_target_link (ctxt);
19926 #include "gt-omp-low.h"