1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
32 #include "pointer-set.h"
33 #include "basic-block.h"
34 #include "tree-ssa-alias.h"
35 #include "internal-fn.h"
36 #include "gimple-fold.h"
37 #include "gimple-expr.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
48 #include "gimple-ssa.h"
51 #include "tree-phinodes.h"
52 #include "ssa-iterators.h"
53 #include "tree-ssanames.h"
54 #include "tree-into-ssa.h"
61 #include "tree-pass.h"
63 #include "splay-tree.h"
68 #include "gimple-low.h"
69 #include "tree-cfgcleanup.h"
70 #include "pretty-print.h"
72 #include "tree-nested.h"
76 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
77 phases. The first phase scans the function looking for OMP statements
78 and then for variables that must be replaced to satisfy data sharing
79 clauses. The second phase expands code for the constructs, as well as
80 re-gimplifying things when variables have been replaced with complex
83 Final code generation is done by pass_expand_omp. The flowgraph is
84 scanned for parallel regions which are then moved to a new
85 function, to be invoked by the thread library. */
87 /* Parallel region information. Every parallel and workshare
88 directive is enclosed between two markers, the OMP_* directive
89 and a corresponding OMP_RETURN statement. */
93 /* The enclosing region. */
94 struct omp_region *outer;
96 /* First child region. */
97 struct omp_region *inner;
99 /* Next peer region. */
100 struct omp_region *next;
102 /* Block containing the omp directive as its last stmt. */
105 /* Block containing the OMP_RETURN as its last stmt. */
108 /* Block containing the OMP_CONTINUE as its last stmt. */
111 /* If this is a combined parallel+workshare region, this is a list
112 of additional arguments needed by the combined parallel+workshare
114 vec<tree, va_gc> *ws_args;
116 /* The code for the omp directive of this region. */
117 enum gimple_code type;
119 /* Schedule kind, only used for OMP_FOR type regions. */
120 enum omp_clause_schedule_kind sched_kind;
122 /* True if this is a combined parallel+workshare region. */
123 bool is_combined_parallel;
126 /* Context structure. Used to store information about each parallel
127 directive in the code. */
129 typedef struct omp_context
131 /* This field must be at the beginning, as we do "inheritance": Some
132 callback functions for tree-inline.c (e.g., omp_copy_decl)
133 receive a copy_body_data pointer that is up-casted to an
134 omp_context pointer. */
137 /* The tree of contexts corresponding to the encountered constructs. */
138 struct omp_context *outer;
141 /* Map variables to fields in a structure that allows communication
142 between sending and receiving threads. */
143 splay_tree field_map;
148 /* These are used just by task contexts, if task firstprivate fn is
149 needed. srecord_type is used to communicate from the thread
150 that encountered the task construct to task firstprivate fn,
151 record_type is allocated by GOMP_task, initialized by task firstprivate
152 fn and passed to the task body fn. */
153 splay_tree sfield_map;
156 /* A chain of variables to add to the top-level block surrounding the
157 construct. In the case of a parallel, this is in the child function. */
160 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
161 barriers should jump to during omplower pass. */
164 /* What to do with variables with implicitly determined sharing
166 enum omp_clause_default_kind default_kind;
168 /* Nesting depth of this context. Used to beautify error messages re
169 invalid gotos. The outermost ctx is depth 1, with depth 0 being
170 reserved for the main body of the function. */
173 /* True if this parallel directive is nested within another. */
176 /* True if this construct can be cancelled. */
181 struct omp_for_data_loop
183 tree v, n1, n2, step;
184 enum tree_code cond_code;
187 /* A structure describing the main elements of a parallel loop. */
191 struct omp_for_data_loop loop;
196 bool have_nowait, have_ordered;
197 enum omp_clause_schedule_kind sched_kind;
198 struct omp_for_data_loop *loops;
202 static splay_tree all_contexts;
203 static int taskreg_nesting_level;
204 static int target_nesting_level;
205 static struct omp_region *root_omp_region;
206 static bitmap task_shared_vars;
208 static void scan_omp (gimple_seq *, omp_context *);
209 static tree scan_omp_1_op (tree *, int *, void *);
211 #define WALK_SUBSTMTS \
215 case GIMPLE_EH_FILTER: \
216 case GIMPLE_TRANSACTION: \
217 /* The sub-statements for these should be walked. */ \
218 *handled_ops_p = false; \
221 /* Convenience function for calling scan_omp_1_op on tree operands. */
224 scan_omp_op (tree *tp, omp_context *ctx)
226 struct walk_stmt_info wi;
228 memset (&wi, 0, sizeof (wi));
230 wi.want_locations = true;
232 return walk_tree (tp, scan_omp_1_op, &wi, NULL);
235 static void lower_omp (gimple_seq *, omp_context *);
236 static tree lookup_decl_in_outer_ctx (tree, omp_context *);
237 static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
239 /* Find an OpenMP clause of type KIND within CLAUSES. */
242 find_omp_clause (tree clauses, enum omp_clause_code kind)
244 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
245 if (OMP_CLAUSE_CODE (clauses) == kind)
251 /* Return true if CTX is for an omp parallel. */
254 is_parallel_ctx (omp_context *ctx)
256 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
260 /* Return true if CTX is for an omp task. */
263 is_task_ctx (omp_context *ctx)
265 return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
269 /* Return true if CTX is for an omp parallel or omp task. */
272 is_taskreg_ctx (omp_context *ctx)
274 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL
275 || gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
279 /* Return true if REGION is a combined parallel+workshare region. */
282 is_combined_parallel (struct omp_region *region)
284 return region->is_combined_parallel;
288 /* Extract the header elements of parallel loop FOR_STMT and store
292 extract_omp_for_data (gimple for_stmt, struct omp_for_data *fd,
293 struct omp_for_data_loop *loops)
295 tree t, var, *collapse_iter, *collapse_count;
296 tree count = NULL_TREE, iter_type = long_integer_type_node;
297 struct omp_for_data_loop *loop;
299 struct omp_for_data_loop dummy_loop;
300 location_t loc = gimple_location (for_stmt);
301 bool simd = gimple_omp_for_kind (for_stmt) & GF_OMP_FOR_SIMD;
302 bool distribute = gimple_omp_for_kind (for_stmt)
303 == GF_OMP_FOR_KIND_DISTRIBUTE;
305 fd->for_stmt = for_stmt;
307 fd->collapse = gimple_omp_for_collapse (for_stmt);
308 if (fd->collapse > 1)
311 fd->loops = &fd->loop;
313 fd->have_nowait = distribute || simd;
314 fd->have_ordered = false;
315 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
316 fd->chunk_size = NULL_TREE;
317 collapse_iter = NULL;
318 collapse_count = NULL;
320 for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t))
321 switch (OMP_CLAUSE_CODE (t))
323 case OMP_CLAUSE_NOWAIT:
324 fd->have_nowait = true;
326 case OMP_CLAUSE_ORDERED:
327 fd->have_ordered = true;
329 case OMP_CLAUSE_SCHEDULE:
330 gcc_assert (!distribute);
331 fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t);
332 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t);
334 case OMP_CLAUSE_DIST_SCHEDULE:
335 gcc_assert (distribute);
336 fd->chunk_size = OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t);
338 case OMP_CLAUSE_COLLAPSE:
339 if (fd->collapse > 1)
341 collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t);
342 collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t);
348 /* FIXME: for now map schedule(auto) to schedule(static).
349 There should be analysis to determine whether all iterations
350 are approximately the same amount of work (then schedule(static)
351 is best) or if it varies (then schedule(dynamic,N) is better). */
352 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO)
354 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
355 gcc_assert (fd->chunk_size == NULL);
357 gcc_assert (fd->collapse == 1 || collapse_iter != NULL);
358 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
359 gcc_assert (fd->chunk_size == NULL);
360 else if (fd->chunk_size == NULL)
362 /* We only need to compute a default chunk size for ordered
363 static loops and dynamic loops. */
364 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
366 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
367 ? integer_zero_node : integer_one_node;
370 for (i = 0; i < fd->collapse; i++)
372 if (fd->collapse == 1)
374 else if (loops != NULL)
379 loop->v = gimple_omp_for_index (for_stmt, i);
380 gcc_assert (SSA_VAR_P (loop->v));
381 gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
382 || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE);
383 var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v;
384 loop->n1 = gimple_omp_for_initial (for_stmt, i);
386 loop->cond_code = gimple_omp_for_cond (for_stmt, i);
387 loop->n2 = gimple_omp_for_final (for_stmt, i);
388 switch (loop->cond_code)
394 gcc_assert (gimple_omp_for_kind (for_stmt)
395 == GF_OMP_FOR_KIND_CILKSIMD);
398 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
399 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, 1);
401 loop->n2 = fold_build2_loc (loc,
402 PLUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
403 build_int_cst (TREE_TYPE (loop->n2), 1));
404 loop->cond_code = LT_EXPR;
407 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
408 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, -1);
410 loop->n2 = fold_build2_loc (loc,
411 MINUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
412 build_int_cst (TREE_TYPE (loop->n2), 1));
413 loop->cond_code = GT_EXPR;
419 t = gimple_omp_for_incr (for_stmt, i);
420 gcc_assert (TREE_OPERAND (t, 0) == var);
421 switch (TREE_CODE (t))
424 loop->step = TREE_OPERAND (t, 1);
426 case POINTER_PLUS_EXPR:
427 loop->step = fold_convert (ssizetype, TREE_OPERAND (t, 1));
430 loop->step = TREE_OPERAND (t, 1);
431 loop->step = fold_build1_loc (loc,
432 NEGATE_EXPR, TREE_TYPE (loop->step),
440 || (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
441 && !fd->have_ordered))
443 if (fd->collapse == 1)
444 iter_type = TREE_TYPE (loop->v);
446 || TYPE_PRECISION (iter_type)
447 < TYPE_PRECISION (TREE_TYPE (loop->v)))
449 = build_nonstandard_integer_type
450 (TYPE_PRECISION (TREE_TYPE (loop->v)), 1);
452 else if (iter_type != long_long_unsigned_type_node)
454 if (POINTER_TYPE_P (TREE_TYPE (loop->v)))
455 iter_type = long_long_unsigned_type_node;
456 else if (TYPE_UNSIGNED (TREE_TYPE (loop->v))
457 && TYPE_PRECISION (TREE_TYPE (loop->v))
458 >= TYPE_PRECISION (iter_type))
462 if (loop->cond_code == LT_EXPR)
463 n = fold_build2_loc (loc,
464 PLUS_EXPR, TREE_TYPE (loop->v),
465 loop->n2, loop->step);
468 if (TREE_CODE (n) != INTEGER_CST
469 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n))
470 iter_type = long_long_unsigned_type_node;
472 else if (TYPE_PRECISION (TREE_TYPE (loop->v))
473 > TYPE_PRECISION (iter_type))
477 if (loop->cond_code == LT_EXPR)
480 n2 = fold_build2_loc (loc,
481 PLUS_EXPR, TREE_TYPE (loop->v),
482 loop->n2, loop->step);
486 n1 = fold_build2_loc (loc,
487 MINUS_EXPR, TREE_TYPE (loop->v),
488 loop->n2, loop->step);
491 if (TREE_CODE (n1) != INTEGER_CST
492 || TREE_CODE (n2) != INTEGER_CST
493 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1)
494 || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type)))
495 iter_type = long_long_unsigned_type_node;
499 if (collapse_count && *collapse_count == NULL)
501 t = fold_binary (loop->cond_code, boolean_type_node,
502 fold_convert (TREE_TYPE (loop->v), loop->n1),
503 fold_convert (TREE_TYPE (loop->v), loop->n2));
504 if (t && integer_zerop (t))
505 count = build_zero_cst (long_long_unsigned_type_node);
506 else if ((i == 0 || count != NULL_TREE)
507 && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
508 && TREE_CONSTANT (loop->n1)
509 && TREE_CONSTANT (loop->n2)
510 && TREE_CODE (loop->step) == INTEGER_CST)
512 tree itype = TREE_TYPE (loop->v);
514 if (POINTER_TYPE_P (itype))
515 itype = signed_type_for (itype);
516 t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1));
517 t = fold_build2_loc (loc,
519 fold_convert_loc (loc, itype, loop->step), t);
520 t = fold_build2_loc (loc, PLUS_EXPR, itype, t,
521 fold_convert_loc (loc, itype, loop->n2));
522 t = fold_build2_loc (loc, MINUS_EXPR, itype, t,
523 fold_convert_loc (loc, itype, loop->n1));
524 if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR)
525 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype,
526 fold_build1_loc (loc, NEGATE_EXPR, itype, t),
527 fold_build1_loc (loc, NEGATE_EXPR, itype,
528 fold_convert_loc (loc, itype,
531 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t,
532 fold_convert_loc (loc, itype, loop->step));
533 t = fold_convert_loc (loc, long_long_unsigned_type_node, t);
534 if (count != NULL_TREE)
535 count = fold_build2_loc (loc,
536 MULT_EXPR, long_long_unsigned_type_node,
540 if (TREE_CODE (count) != INTEGER_CST)
543 else if (count && !integer_zerop (count))
550 && (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
551 || fd->have_ordered))
553 if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node)))
554 iter_type = long_long_unsigned_type_node;
556 iter_type = long_integer_type_node;
558 else if (collapse_iter && *collapse_iter != NULL)
559 iter_type = TREE_TYPE (*collapse_iter);
560 fd->iter_type = iter_type;
561 if (collapse_iter && *collapse_iter == NULL)
562 *collapse_iter = create_tmp_var (iter_type, ".iter");
563 if (collapse_count && *collapse_count == NULL)
566 *collapse_count = fold_convert_loc (loc, iter_type, count);
568 *collapse_count = create_tmp_var (iter_type, ".count");
571 if (fd->collapse > 1)
573 fd->loop.v = *collapse_iter;
574 fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0);
575 fd->loop.n2 = *collapse_count;
576 fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1);
577 fd->loop.cond_code = LT_EXPR;
582 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
583 is the immediate dominator of PAR_ENTRY_BB, return true if there
584 are no data dependencies that would prevent expanding the parallel
585 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
587 When expanding a combined parallel+workshare region, the call to
588 the child function may need additional arguments in the case of
589 GIMPLE_OMP_FOR regions. In some cases, these arguments are
590 computed out of variables passed in from the parent to the child
591 via 'struct .omp_data_s'. For instance:
593 #pragma omp parallel for schedule (guided, i * 4)
598 # BLOCK 2 (PAR_ENTRY_BB)
600 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
602 # BLOCK 3 (WS_ENTRY_BB)
603 .omp_data_i = &.omp_data_o;
604 D.1667 = .omp_data_i->i;
606 #pragma omp for schedule (guided, D.1598)
608 When we outline the parallel region, the call to the child function
609 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
610 that value is computed *after* the call site. So, in principle we
611 cannot do the transformation.
613 To see whether the code in WS_ENTRY_BB blocks the combined
614 parallel+workshare call, we collect all the variables used in the
615 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
616 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
619 FIXME. If we had the SSA form built at this point, we could merely
620 hoist the code in block 3 into block 2 and be done with it. But at
621 this point we don't have dataflow information and though we could
622 hack something up here, it is really not worth the aggravation. */
625 workshare_safe_to_combine_p (basic_block ws_entry_bb)
627 struct omp_for_data fd;
628 gimple ws_stmt = last_stmt (ws_entry_bb);
630 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
633 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
635 extract_omp_for_data (ws_stmt, &fd, NULL);
637 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
639 if (fd.iter_type != long_integer_type_node)
642 /* FIXME. We give up too easily here. If any of these arguments
643 are not constants, they will likely involve variables that have
644 been mapped into fields of .omp_data_s for sharing with the child
645 function. With appropriate data flow, it would be possible to
647 if (!is_gimple_min_invariant (fd.loop.n1)
648 || !is_gimple_min_invariant (fd.loop.n2)
649 || !is_gimple_min_invariant (fd.loop.step)
650 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
657 /* Collect additional arguments needed to emit a combined
658 parallel+workshare call. WS_STMT is the workshare directive being
661 static vec<tree, va_gc> *
662 get_ws_args_for (gimple par_stmt, gimple ws_stmt)
665 location_t loc = gimple_location (ws_stmt);
666 vec<tree, va_gc> *ws_args;
668 if (gimple_code (ws_stmt) == GIMPLE_OMP_FOR)
670 struct omp_for_data fd;
673 extract_omp_for_data (ws_stmt, &fd, NULL);
677 if (gimple_omp_for_combined_into_p (ws_stmt))
680 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt),
681 OMP_CLAUSE__LOOPTEMP_);
683 n1 = OMP_CLAUSE_DECL (innerc);
684 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
685 OMP_CLAUSE__LOOPTEMP_);
687 n2 = OMP_CLAUSE_DECL (innerc);
690 vec_alloc (ws_args, 3 + (fd.chunk_size != 0));
692 t = fold_convert_loc (loc, long_integer_type_node, n1);
693 ws_args->quick_push (t);
695 t = fold_convert_loc (loc, long_integer_type_node, n2);
696 ws_args->quick_push (t);
698 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step);
699 ws_args->quick_push (t);
703 t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size);
704 ws_args->quick_push (t);
709 else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
711 /* Number of sections is equal to the number of edges from the
712 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
713 the exit of the sections region. */
714 basic_block bb = single_succ (gimple_bb (ws_stmt));
715 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
716 vec_alloc (ws_args, 1);
717 ws_args->quick_push (t);
725 /* Discover whether REGION is a combined parallel+workshare region. */
728 determine_parallel_type (struct omp_region *region)
730 basic_block par_entry_bb, par_exit_bb;
731 basic_block ws_entry_bb, ws_exit_bb;
733 if (region == NULL || region->inner == NULL
734 || region->exit == NULL || region->inner->exit == NULL
735 || region->inner->cont == NULL)
738 /* We only support parallel+for and parallel+sections. */
739 if (region->type != GIMPLE_OMP_PARALLEL
740 || (region->inner->type != GIMPLE_OMP_FOR
741 && region->inner->type != GIMPLE_OMP_SECTIONS))
744 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
745 WS_EXIT_BB -> PAR_EXIT_BB. */
746 par_entry_bb = region->entry;
747 par_exit_bb = region->exit;
748 ws_entry_bb = region->inner->entry;
749 ws_exit_bb = region->inner->exit;
751 if (single_succ (par_entry_bb) == ws_entry_bb
752 && single_succ (ws_exit_bb) == par_exit_bb
753 && workshare_safe_to_combine_p (ws_entry_bb)
754 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb))
755 || (last_and_only_stmt (ws_entry_bb)
756 && last_and_only_stmt (par_exit_bb))))
758 gimple par_stmt = last_stmt (par_entry_bb);
759 gimple ws_stmt = last_stmt (ws_entry_bb);
761 if (region->inner->type == GIMPLE_OMP_FOR)
763 /* If this is a combined parallel loop, we need to determine
764 whether or not to use the combined library calls. There
765 are two cases where we do not apply the transformation:
766 static loops and any kind of ordered loop. In the first
767 case, we already open code the loop so there is no need
768 to do anything else. In the latter case, the combined
769 parallel loop call would still need extra synchronization
770 to implement ordered semantics, so there would not be any
771 gain in using the combined call. */
772 tree clauses = gimple_omp_for_clauses (ws_stmt);
773 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE);
775 || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC
776 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED))
778 region->is_combined_parallel = false;
779 region->inner->is_combined_parallel = false;
784 region->is_combined_parallel = true;
785 region->inner->is_combined_parallel = true;
786 region->ws_args = get_ws_args_for (par_stmt, ws_stmt);
791 /* Return true if EXPR is variable sized. */
794 is_variable_sized (const_tree expr)
796 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
799 /* Return true if DECL is a reference type. */
802 is_reference (tree decl)
804 return lang_hooks.decls.omp_privatize_by_reference (decl);
807 /* Lookup variables in the decl or field splay trees. The "maybe" form
808 allows for the variable form to not have been entered, otherwise we
809 assert that the variable must have been entered. */
812 lookup_decl (tree var, omp_context *ctx)
815 n = (tree *) pointer_map_contains (ctx->cb.decl_map, var);
820 maybe_lookup_decl (const_tree var, omp_context *ctx)
823 n = (tree *) pointer_map_contains (ctx->cb.decl_map, var);
824 return n ? *n : NULL_TREE;
828 lookup_field (tree var, omp_context *ctx)
831 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
832 return (tree) n->value;
836 lookup_sfield (tree var, omp_context *ctx)
839 n = splay_tree_lookup (ctx->sfield_map
840 ? ctx->sfield_map : ctx->field_map,
841 (splay_tree_key) var);
842 return (tree) n->value;
846 maybe_lookup_field (tree var, omp_context *ctx)
849 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
850 return n ? (tree) n->value : NULL_TREE;
853 /* Return true if DECL should be copied by pointer. SHARED_CTX is
854 the parallel context if DECL is to be shared. */
857 use_pointer_for_field (tree decl, omp_context *shared_ctx)
859 if (AGGREGATE_TYPE_P (TREE_TYPE (decl)))
862 /* We can only use copy-in/copy-out semantics for shared variables
863 when we know the value is not accessible from an outer scope. */
866 /* ??? Trivially accessible from anywhere. But why would we even
867 be passing an address in this case? Should we simply assert
868 this to be false, or should we have a cleanup pass that removes
869 these from the list of mappings? */
870 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
873 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
874 without analyzing the expression whether or not its location
875 is accessible to anyone else. In the case of nested parallel
876 regions it certainly may be. */
877 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
880 /* Do not use copy-in/copy-out for variables that have their
882 if (TREE_ADDRESSABLE (decl))
885 /* lower_send_shared_vars only uses copy-in, but not copy-out
887 if (TREE_READONLY (decl)
888 || ((TREE_CODE (decl) == RESULT_DECL
889 || TREE_CODE (decl) == PARM_DECL)
890 && DECL_BY_REFERENCE (decl)))
893 /* Disallow copy-in/out in nested parallel if
894 decl is shared in outer parallel, otherwise
895 each thread could store the shared variable
896 in its own copy-in location, making the
897 variable no longer really shared. */
898 if (shared_ctx->is_nested)
902 for (up = shared_ctx->outer; up; up = up->outer)
903 if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up))
910 for (c = gimple_omp_taskreg_clauses (up->stmt);
911 c; c = OMP_CLAUSE_CHAIN (c))
912 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
913 && OMP_CLAUSE_DECL (c) == decl)
917 goto maybe_mark_addressable_and_ret;
921 /* For tasks avoid using copy-in/out. As tasks can be
922 deferred or executed in different thread, when GOMP_task
923 returns, the task hasn't necessarily terminated. */
924 if (is_task_ctx (shared_ctx))
927 maybe_mark_addressable_and_ret:
928 outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
929 if (is_gimple_reg (outer))
931 /* Taking address of OUTER in lower_send_shared_vars
932 might need regimplification of everything that uses the
934 if (!task_shared_vars)
935 task_shared_vars = BITMAP_ALLOC (NULL);
936 bitmap_set_bit (task_shared_vars, DECL_UID (outer));
937 TREE_ADDRESSABLE (outer) = 1;
946 /* Construct a new automatic decl similar to VAR. */
949 omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
951 tree copy = copy_var_decl (var, name, type);
953 DECL_CONTEXT (copy) = current_function_decl;
954 DECL_CHAIN (copy) = ctx->block_vars;
955 ctx->block_vars = copy;
961 omp_copy_decl_1 (tree var, omp_context *ctx)
963 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
966 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
969 omp_build_component_ref (tree obj, tree field)
971 tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL);
972 if (TREE_THIS_VOLATILE (field))
973 TREE_THIS_VOLATILE (ret) |= 1;
974 if (TREE_READONLY (field))
975 TREE_READONLY (ret) |= 1;
979 /* Build tree nodes to access the field for VAR on the receiver side. */
982 build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
984 tree x, field = lookup_field (var, ctx);
986 /* If the receiver record type was remapped in the child function,
987 remap the field into the new record type. */
988 x = maybe_lookup_field (field, ctx);
992 x = build_simple_mem_ref (ctx->receiver_decl);
993 x = omp_build_component_ref (x, field);
995 x = build_simple_mem_ref (x);
1000 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1001 of a parallel, this is a component reference; for workshare constructs
1002 this is some variable. */
1005 build_outer_var_ref (tree var, omp_context *ctx)
1009 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
1011 else if (is_variable_sized (var))
1013 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
1014 x = build_outer_var_ref (x, ctx);
1015 x = build_simple_mem_ref (x);
1017 else if (is_taskreg_ctx (ctx))
1019 bool by_ref = use_pointer_for_field (var, NULL);
1020 x = build_receiver_ref (var, by_ref, ctx);
1022 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1023 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
1025 /* #pragma omp simd isn't a worksharing construct, and can reference even
1026 private vars in its linear etc. clauses. */
1028 if (ctx->outer && is_taskreg_ctx (ctx))
1029 x = lookup_decl (var, ctx->outer);
1030 else if (ctx->outer)
1031 x = maybe_lookup_decl_in_outer_ctx (var, ctx);
1035 else if (ctx->outer)
1036 x = lookup_decl (var, ctx->outer);
1037 else if (is_reference (var))
1038 /* This can happen with orphaned constructs. If var is reference, it is
1039 possible it is shared and as such valid. */
1044 if (is_reference (var))
1045 x = build_simple_mem_ref (x);
1050 /* Build tree nodes to access the field for VAR on the sender side. */
1053 build_sender_ref (tree var, omp_context *ctx)
1055 tree field = lookup_sfield (var, ctx);
1056 return omp_build_component_ref (ctx->sender_decl, field);
1059 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1062 install_var_field (tree var, bool by_ref, int mask, omp_context *ctx)
1064 tree field, type, sfield = NULL_TREE;
1066 gcc_assert ((mask & 1) == 0
1067 || !splay_tree_lookup (ctx->field_map, (splay_tree_key) var));
1068 gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
1069 || !splay_tree_lookup (ctx->sfield_map, (splay_tree_key) var));
1071 type = TREE_TYPE (var);
1074 gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
1075 type = build_pointer_type (build_pointer_type (type));
1078 type = build_pointer_type (type);
1079 else if ((mask & 3) == 1 && is_reference (var))
1080 type = TREE_TYPE (type);
1082 field = build_decl (DECL_SOURCE_LOCATION (var),
1083 FIELD_DECL, DECL_NAME (var), type);
1085 /* Remember what variable this field was created for. This does have a
1086 side effect of making dwarf2out ignore this member, so for helpful
1087 debugging we clear it later in delete_omp_context. */
1088 DECL_ABSTRACT_ORIGIN (field) = var;
1089 if (type == TREE_TYPE (var))
1091 DECL_ALIGN (field) = DECL_ALIGN (var);
1092 DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
1093 TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
1096 DECL_ALIGN (field) = TYPE_ALIGN (type);
1098 if ((mask & 3) == 3)
1100 insert_field_into_struct (ctx->record_type, field);
1101 if (ctx->srecord_type)
1103 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1104 FIELD_DECL, DECL_NAME (var), type);
1105 DECL_ABSTRACT_ORIGIN (sfield) = var;
1106 DECL_ALIGN (sfield) = DECL_ALIGN (field);
1107 DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
1108 TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
1109 insert_field_into_struct (ctx->srecord_type, sfield);
1114 if (ctx->srecord_type == NULL_TREE)
1118 ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
1119 ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1120 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
1122 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1123 FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
1124 DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
1125 insert_field_into_struct (ctx->srecord_type, sfield);
1126 splay_tree_insert (ctx->sfield_map,
1127 (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
1128 (splay_tree_value) sfield);
1132 insert_field_into_struct ((mask & 1) ? ctx->record_type
1133 : ctx->srecord_type, field);
1137 splay_tree_insert (ctx->field_map, (splay_tree_key) var,
1138 (splay_tree_value) field);
1139 if ((mask & 2) && ctx->sfield_map)
1140 splay_tree_insert (ctx->sfield_map, (splay_tree_key) var,
1141 (splay_tree_value) sfield);
1145 install_var_local (tree var, omp_context *ctx)
1147 tree new_var = omp_copy_decl_1 (var, ctx);
1148 insert_decl_map (&ctx->cb, var, new_var);
1152 /* Adjust the replacement for DECL in CTX for the new context. This means
1153 copying the DECL_VALUE_EXPR, and fixing up the type. */
1156 fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
1158 tree new_decl, size;
1160 new_decl = lookup_decl (decl, ctx);
1162 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
1164 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
1165 && DECL_HAS_VALUE_EXPR_P (decl))
1167 tree ve = DECL_VALUE_EXPR (decl);
1168 walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
1169 SET_DECL_VALUE_EXPR (new_decl, ve);
1170 DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
1173 if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
1175 size = remap_decl (DECL_SIZE (decl), &ctx->cb);
1176 if (size == error_mark_node)
1177 size = TYPE_SIZE (TREE_TYPE (new_decl));
1178 DECL_SIZE (new_decl) = size;
1180 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
1181 if (size == error_mark_node)
1182 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
1183 DECL_SIZE_UNIT (new_decl) = size;
1187 /* The callback for remap_decl. Search all containing contexts for a
1188 mapping of the variable; this avoids having to duplicate the splay
1189 tree ahead of time. We know a mapping doesn't already exist in the
1190 given context. Create new mappings to implement default semantics. */
1193 omp_copy_decl (tree var, copy_body_data *cb)
1195 omp_context *ctx = (omp_context *) cb;
1198 if (TREE_CODE (var) == LABEL_DECL)
1200 new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
1201 DECL_CONTEXT (new_var) = current_function_decl;
1202 insert_decl_map (&ctx->cb, var, new_var);
1206 while (!is_taskreg_ctx (ctx))
1211 new_var = maybe_lookup_decl (var, ctx);
1216 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
1219 return error_mark_node;
1223 /* Debugging dumps for parallel regions. */
1224 void dump_omp_region (FILE *, struct omp_region *, int);
1225 void debug_omp_region (struct omp_region *);
1226 void debug_all_omp_regions (void);
1228 /* Dump the parallel region tree rooted at REGION. */
1231 dump_omp_region (FILE *file, struct omp_region *region, int indent)
1233 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
1234 gimple_code_name[region->type]);
1237 dump_omp_region (file, region->inner, indent + 4);
1241 fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "",
1242 region->cont->index);
1246 fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "",
1247 region->exit->index);
1249 fprintf (file, "%*s[no exit marker]\n", indent, "");
1252 dump_omp_region (file, region->next, indent);
1256 debug_omp_region (struct omp_region *region)
1258 dump_omp_region (stderr, region, 0);
1262 debug_all_omp_regions (void)
1264 dump_omp_region (stderr, root_omp_region, 0);
1268 /* Create a new parallel region starting at STMT inside region PARENT. */
1270 static struct omp_region *
1271 new_omp_region (basic_block bb, enum gimple_code type,
1272 struct omp_region *parent)
1274 struct omp_region *region = XCNEW (struct omp_region);
1276 region->outer = parent;
1278 region->type = type;
1282 /* This is a nested region. Add it to the list of inner
1283 regions in PARENT. */
1284 region->next = parent->inner;
1285 parent->inner = region;
1289 /* This is a toplevel region. Add it to the list of toplevel
1290 regions in ROOT_OMP_REGION. */
1291 region->next = root_omp_region;
1292 root_omp_region = region;
1298 /* Release the memory associated with the region tree rooted at REGION. */
1301 free_omp_region_1 (struct omp_region *region)
1303 struct omp_region *i, *n;
1305 for (i = region->inner; i ; i = n)
1308 free_omp_region_1 (i);
1314 /* Release the memory for the entire omp region tree. */
1317 free_omp_regions (void)
1319 struct omp_region *r, *n;
1320 for (r = root_omp_region; r ; r = n)
1323 free_omp_region_1 (r);
1325 root_omp_region = NULL;
1329 /* Create a new context, with OUTER_CTX being the surrounding context. */
1331 static omp_context *
1332 new_omp_context (gimple stmt, omp_context *outer_ctx)
1334 omp_context *ctx = XCNEW (omp_context);
1336 splay_tree_insert (all_contexts, (splay_tree_key) stmt,
1337 (splay_tree_value) ctx);
1342 ctx->outer = outer_ctx;
1343 ctx->cb = outer_ctx->cb;
1344 ctx->cb.block = NULL;
1345 ctx->depth = outer_ctx->depth + 1;
1349 ctx->cb.src_fn = current_function_decl;
1350 ctx->cb.dst_fn = current_function_decl;
1351 ctx->cb.src_node = cgraph_get_node (current_function_decl);
1352 gcc_checking_assert (ctx->cb.src_node);
1353 ctx->cb.dst_node = ctx->cb.src_node;
1354 ctx->cb.src_cfun = cfun;
1355 ctx->cb.copy_decl = omp_copy_decl;
1356 ctx->cb.eh_lp_nr = 0;
1357 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1361 ctx->cb.decl_map = pointer_map_create ();
1366 static gimple_seq maybe_catch_exception (gimple_seq);
1368 /* Finalize task copyfn. */
1371 finalize_task_copyfn (gimple task_stmt)
1373 struct function *child_cfun;
1375 gimple_seq seq = NULL, new_seq;
1378 child_fn = gimple_omp_task_copy_fn (task_stmt);
1379 if (child_fn == NULL_TREE)
1382 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1383 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1385 push_cfun (child_cfun);
1386 bind = gimplify_body (child_fn, false);
1387 gimple_seq_add_stmt (&seq, bind);
1388 new_seq = maybe_catch_exception (seq);
1391 bind = gimple_build_bind (NULL, new_seq, NULL);
1393 gimple_seq_add_stmt (&seq, bind);
1395 gimple_set_body (child_fn, seq);
1398 /* Inform the callgraph about the new function. */
1399 cgraph_add_new_function (child_fn, false);
1402 /* Destroy a omp_context data structures. Called through the splay tree
1403 value delete callback. */
1406 delete_omp_context (splay_tree_value value)
1408 omp_context *ctx = (omp_context *) value;
1410 pointer_map_destroy (ctx->cb.decl_map);
1413 splay_tree_delete (ctx->field_map);
1414 if (ctx->sfield_map)
1415 splay_tree_delete (ctx->sfield_map);
1417 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1418 it produces corrupt debug information. */
1419 if (ctx->record_type)
1422 for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1423 DECL_ABSTRACT_ORIGIN (t) = NULL;
1425 if (ctx->srecord_type)
1428 for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1429 DECL_ABSTRACT_ORIGIN (t) = NULL;
1432 if (is_task_ctx (ctx))
1433 finalize_task_copyfn (ctx->stmt);
1438 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1442 fixup_child_record_type (omp_context *ctx)
1444 tree f, type = ctx->record_type;
1446 /* ??? It isn't sufficient to just call remap_type here, because
1447 variably_modified_type_p doesn't work the way we expect for
1448 record types. Testing each field for whether it needs remapping
1449 and creating a new record by hand works, however. */
1450 for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1451 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1455 tree name, new_fields = NULL;
1457 type = lang_hooks.types.make_type (RECORD_TYPE);
1458 name = DECL_NAME (TYPE_NAME (ctx->record_type));
1459 name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1460 TYPE_DECL, name, type);
1461 TYPE_NAME (type) = name;
1463 for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1465 tree new_f = copy_node (f);
1466 DECL_CONTEXT (new_f) = type;
1467 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1468 DECL_CHAIN (new_f) = new_fields;
1469 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1470 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1472 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1476 /* Arrange to be able to look up the receiver field
1477 given the sender field. */
1478 splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1479 (splay_tree_value) new_f);
1481 TYPE_FIELDS (type) = nreverse (new_fields);
1485 TREE_TYPE (ctx->receiver_decl) = build_pointer_type (type);
1488 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1489 specified by CLAUSES. */
1492 scan_sharing_clauses (tree clauses, omp_context *ctx)
1495 bool scan_array_reductions = false;
1497 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1501 switch (OMP_CLAUSE_CODE (c))
1503 case OMP_CLAUSE_PRIVATE:
1504 decl = OMP_CLAUSE_DECL (c);
1505 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1507 else if (!is_variable_sized (decl))
1508 install_var_local (decl, ctx);
1511 case OMP_CLAUSE_SHARED:
1512 /* Ignore shared directives in teams construct. */
1513 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1515 gcc_assert (is_taskreg_ctx (ctx));
1516 decl = OMP_CLAUSE_DECL (c);
1517 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1518 || !is_variable_sized (decl));
1519 /* Global variables don't need to be copied,
1520 the receiver side will use them directly. */
1521 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1523 by_ref = use_pointer_for_field (decl, ctx);
1524 if (! TREE_READONLY (decl)
1525 || TREE_ADDRESSABLE (decl)
1527 || is_reference (decl))
1529 install_var_field (decl, by_ref, 3, ctx);
1530 install_var_local (decl, ctx);
1533 /* We don't need to copy const scalar vars back. */
1534 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1537 case OMP_CLAUSE_LASTPRIVATE:
1538 /* Let the corresponding firstprivate clause create
1540 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1544 case OMP_CLAUSE_FIRSTPRIVATE:
1545 case OMP_CLAUSE_REDUCTION:
1546 case OMP_CLAUSE_LINEAR:
1547 decl = OMP_CLAUSE_DECL (c);
1549 if (is_variable_sized (decl))
1551 if (is_task_ctx (ctx))
1552 install_var_field (decl, false, 1, ctx);
1555 else if (is_taskreg_ctx (ctx))
1558 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1559 by_ref = use_pointer_for_field (decl, NULL);
1561 if (is_task_ctx (ctx)
1562 && (global || by_ref || is_reference (decl)))
1564 install_var_field (decl, false, 1, ctx);
1566 install_var_field (decl, by_ref, 2, ctx);
1569 install_var_field (decl, by_ref, 3, ctx);
1571 install_var_local (decl, ctx);
1574 case OMP_CLAUSE__LOOPTEMP_:
1575 gcc_assert (is_parallel_ctx (ctx));
1576 decl = OMP_CLAUSE_DECL (c);
1577 install_var_field (decl, false, 3, ctx);
1578 install_var_local (decl, ctx);
1581 case OMP_CLAUSE_COPYPRIVATE:
1582 case OMP_CLAUSE_COPYIN:
1583 decl = OMP_CLAUSE_DECL (c);
1584 by_ref = use_pointer_for_field (decl, NULL);
1585 install_var_field (decl, by_ref, 3, ctx);
1588 case OMP_CLAUSE_DEFAULT:
1589 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c);
1592 case OMP_CLAUSE_FINAL:
1594 case OMP_CLAUSE_NUM_THREADS:
1595 case OMP_CLAUSE_NUM_TEAMS:
1596 case OMP_CLAUSE_THREAD_LIMIT:
1597 case OMP_CLAUSE_DEVICE:
1598 case OMP_CLAUSE_SCHEDULE:
1599 case OMP_CLAUSE_DIST_SCHEDULE:
1600 case OMP_CLAUSE_DEPEND:
1602 scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1606 case OMP_CLAUSE_FROM:
1607 case OMP_CLAUSE_MAP:
1609 scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1610 decl = OMP_CLAUSE_DECL (c);
1611 /* Global variables with "omp declare target" attribute
1612 don't need to be copied, the receiver side will use them
1614 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1616 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1617 && lookup_attribute ("omp declare target",
1618 DECL_ATTRIBUTES (decl)))
1620 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1621 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER)
1623 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1624 #pragma omp target data, there is nothing to map for
1626 if (gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_DATA
1627 && !POINTER_TYPE_P (TREE_TYPE (decl)))
1632 if (DECL_SIZE (decl)
1633 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1635 tree decl2 = DECL_VALUE_EXPR (decl);
1636 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
1637 decl2 = TREE_OPERAND (decl2, 0);
1638 gcc_assert (DECL_P (decl2));
1639 install_var_field (decl2, true, 3, ctx);
1640 install_var_local (decl2, ctx);
1641 install_var_local (decl, ctx);
1645 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1646 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
1647 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1648 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1649 install_var_field (decl, true, 7, ctx);
1651 install_var_field (decl, true, 3, ctx);
1652 if (gimple_omp_target_kind (ctx->stmt)
1653 == GF_OMP_TARGET_KIND_REGION)
1654 install_var_local (decl, ctx);
1659 tree base = get_base_address (decl);
1660 tree nc = OMP_CLAUSE_CHAIN (c);
1663 && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1664 && OMP_CLAUSE_DECL (nc) == base
1665 && OMP_CLAUSE_MAP_KIND (nc) == OMP_CLAUSE_MAP_POINTER
1666 && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1668 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1669 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1673 gcc_assert (!splay_tree_lookup (ctx->field_map,
1674 (splay_tree_key) decl));
1676 = build_decl (OMP_CLAUSE_LOCATION (c),
1677 FIELD_DECL, NULL_TREE, ptr_type_node);
1678 DECL_ALIGN (field) = TYPE_ALIGN (ptr_type_node);
1679 insert_field_into_struct (ctx->record_type, field);
1680 splay_tree_insert (ctx->field_map, (splay_tree_key) decl,
1681 (splay_tree_value) field);
1686 case OMP_CLAUSE_NOWAIT:
1687 case OMP_CLAUSE_ORDERED:
1688 case OMP_CLAUSE_COLLAPSE:
1689 case OMP_CLAUSE_UNTIED:
1690 case OMP_CLAUSE_MERGEABLE:
1691 case OMP_CLAUSE_PROC_BIND:
1692 case OMP_CLAUSE_SAFELEN:
1695 case OMP_CLAUSE_ALIGNED:
1696 decl = OMP_CLAUSE_DECL (c);
1697 if (is_global_var (decl)
1698 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1699 install_var_local (decl, ctx);
1707 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1709 switch (OMP_CLAUSE_CODE (c))
1711 case OMP_CLAUSE_LASTPRIVATE:
1712 /* Let the corresponding firstprivate clause create
1714 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1715 scan_array_reductions = true;
1716 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1720 case OMP_CLAUSE_PRIVATE:
1721 case OMP_CLAUSE_FIRSTPRIVATE:
1722 case OMP_CLAUSE_REDUCTION:
1723 case OMP_CLAUSE_LINEAR:
1724 decl = OMP_CLAUSE_DECL (c);
1725 if (is_variable_sized (decl))
1726 install_var_local (decl, ctx);
1727 fixup_remapped_decl (decl, ctx,
1728 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1729 && OMP_CLAUSE_PRIVATE_DEBUG (c));
1730 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1731 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1732 scan_array_reductions = true;
1733 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
1734 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
1735 scan_array_reductions = true;
1738 case OMP_CLAUSE_SHARED:
1739 /* Ignore shared directives in teams construct. */
1740 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1742 decl = OMP_CLAUSE_DECL (c);
1743 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1744 fixup_remapped_decl (decl, ctx, false);
1747 case OMP_CLAUSE_MAP:
1748 if (gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_DATA)
1750 decl = OMP_CLAUSE_DECL (c);
1752 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1753 && lookup_attribute ("omp declare target",
1754 DECL_ATTRIBUTES (decl)))
1758 if (OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
1759 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
1760 && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
1762 tree new_decl = lookup_decl (decl, ctx);
1763 TREE_TYPE (new_decl)
1764 = remap_type (TREE_TYPE (decl), &ctx->cb);
1766 else if (DECL_SIZE (decl)
1767 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1769 tree decl2 = DECL_VALUE_EXPR (decl);
1770 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
1771 decl2 = TREE_OPERAND (decl2, 0);
1772 gcc_assert (DECL_P (decl2));
1773 fixup_remapped_decl (decl2, ctx, false);
1774 fixup_remapped_decl (decl, ctx, true);
1777 fixup_remapped_decl (decl, ctx, false);
1781 case OMP_CLAUSE_COPYPRIVATE:
1782 case OMP_CLAUSE_COPYIN:
1783 case OMP_CLAUSE_DEFAULT:
1785 case OMP_CLAUSE_NUM_THREADS:
1786 case OMP_CLAUSE_NUM_TEAMS:
1787 case OMP_CLAUSE_THREAD_LIMIT:
1788 case OMP_CLAUSE_DEVICE:
1789 case OMP_CLAUSE_SCHEDULE:
1790 case OMP_CLAUSE_DIST_SCHEDULE:
1791 case OMP_CLAUSE_NOWAIT:
1792 case OMP_CLAUSE_ORDERED:
1793 case OMP_CLAUSE_COLLAPSE:
1794 case OMP_CLAUSE_UNTIED:
1795 case OMP_CLAUSE_FINAL:
1796 case OMP_CLAUSE_MERGEABLE:
1797 case OMP_CLAUSE_PROC_BIND:
1798 case OMP_CLAUSE_SAFELEN:
1799 case OMP_CLAUSE_ALIGNED:
1800 case OMP_CLAUSE_DEPEND:
1801 case OMP_CLAUSE__LOOPTEMP_:
1803 case OMP_CLAUSE_FROM:
1811 if (scan_array_reductions)
1812 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1813 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1814 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1816 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx);
1817 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
1819 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
1820 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1821 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
1822 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
1823 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
1824 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
1827 /* Create a new name for omp child function. Returns an identifier. */
1830 create_omp_child_function_name (bool task_copy)
1832 return (clone_function_name (current_function_decl,
1833 task_copy ? "_omp_cpyfn" : "_omp_fn"));
1836 /* Build a decl for the omp child function. It'll not contain a body
1837 yet, just the bare decl. */
1840 create_omp_child_function (omp_context *ctx, bool task_copy)
1842 tree decl, type, name, t;
1844 name = create_omp_child_function_name (task_copy);
1846 type = build_function_type_list (void_type_node, ptr_type_node,
1847 ptr_type_node, NULL_TREE);
1849 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
1851 decl = build_decl (gimple_location (ctx->stmt),
1852 FUNCTION_DECL, name, type);
1855 ctx->cb.dst_fn = decl;
1857 gimple_omp_task_set_copy_fn (ctx->stmt, decl);
1859 TREE_STATIC (decl) = 1;
1860 TREE_USED (decl) = 1;
1861 DECL_ARTIFICIAL (decl) = 1;
1862 DECL_NAMELESS (decl) = 1;
1863 DECL_IGNORED_P (decl) = 0;
1864 TREE_PUBLIC (decl) = 0;
1865 DECL_UNINLINABLE (decl) = 1;
1866 DECL_EXTERNAL (decl) = 0;
1867 DECL_CONTEXT (decl) = NULL_TREE;
1868 DECL_INITIAL (decl) = make_node (BLOCK);
1869 bool target_p = false;
1870 if (lookup_attribute ("omp declare target",
1871 DECL_ATTRIBUTES (current_function_decl)))
1876 for (octx = ctx; octx; octx = octx->outer)
1877 if (gimple_code (octx->stmt) == GIMPLE_OMP_TARGET
1878 && gimple_omp_target_kind (octx->stmt)
1879 == GF_OMP_TARGET_KIND_REGION)
1886 DECL_ATTRIBUTES (decl)
1887 = tree_cons (get_identifier ("omp declare target"),
1888 NULL_TREE, DECL_ATTRIBUTES (decl));
1890 t = build_decl (DECL_SOURCE_LOCATION (decl),
1891 RESULT_DECL, NULL_TREE, void_type_node);
1892 DECL_ARTIFICIAL (t) = 1;
1893 DECL_IGNORED_P (t) = 1;
1894 DECL_CONTEXT (t) = decl;
1895 DECL_RESULT (decl) = t;
1897 t = build_decl (DECL_SOURCE_LOCATION (decl),
1898 PARM_DECL, get_identifier (".omp_data_i"), ptr_type_node);
1899 DECL_ARTIFICIAL (t) = 1;
1900 DECL_NAMELESS (t) = 1;
1901 DECL_ARG_TYPE (t) = ptr_type_node;
1902 DECL_CONTEXT (t) = current_function_decl;
1904 DECL_ARGUMENTS (decl) = t;
1906 ctx->receiver_decl = t;
1909 t = build_decl (DECL_SOURCE_LOCATION (decl),
1910 PARM_DECL, get_identifier (".omp_data_o"),
1912 DECL_ARTIFICIAL (t) = 1;
1913 DECL_NAMELESS (t) = 1;
1914 DECL_ARG_TYPE (t) = ptr_type_node;
1915 DECL_CONTEXT (t) = current_function_decl;
1917 TREE_ADDRESSABLE (t) = 1;
1918 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
1919 DECL_ARGUMENTS (decl) = t;
1922 /* Allocate memory for the function structure. The call to
1923 allocate_struct_function clobbers CFUN, so we need to restore
1925 push_struct_function (decl);
1926 cfun->function_end_locus = gimple_location (ctx->stmt);
1930 /* Callback for walk_gimple_seq. Check if combined parallel
1931 contains gimple_omp_for_combined_into_p OMP_FOR. */
1934 find_combined_for (gimple_stmt_iterator *gsi_p,
1935 bool *handled_ops_p,
1936 struct walk_stmt_info *wi)
1938 gimple stmt = gsi_stmt (*gsi_p);
1940 *handled_ops_p = true;
1941 switch (gimple_code (stmt))
1945 case GIMPLE_OMP_FOR:
1946 if (gimple_omp_for_combined_into_p (stmt)
1947 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
1950 return integer_zero_node;
1959 /* Scan an OpenMP parallel directive. */
1962 scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
1966 gimple stmt = gsi_stmt (*gsi);
1968 /* Ignore parallel directives with empty bodies, unless there
1969 are copyin clauses. */
1971 && empty_body_p (gimple_omp_body (stmt))
1972 && find_omp_clause (gimple_omp_parallel_clauses (stmt),
1973 OMP_CLAUSE_COPYIN) == NULL)
1975 gsi_replace (gsi, gimple_build_nop (), false);
1979 if (gimple_omp_parallel_combined_p (stmt))
1982 struct walk_stmt_info wi;
1984 memset (&wi, 0, sizeof (wi));
1986 walk_gimple_seq (gimple_omp_body (stmt),
1987 find_combined_for, NULL, &wi);
1988 for_stmt = (gimple) wi.info;
1991 struct omp_for_data fd;
1992 extract_omp_for_data (for_stmt, &fd, NULL);
1993 /* We need two temporaries with fd.loop.v type (istart/iend)
1994 and then (fd.collapse - 1) temporaries with the same
1995 type for count2 ... countN-1 vars if not constant. */
1996 size_t count = 2, i;
1997 tree type = fd.iter_type;
1999 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
2000 count += fd.collapse - 1;
2001 for (i = 0; i < count; i++)
2003 tree temp = create_tmp_var (type, NULL);
2004 tree c = build_omp_clause (UNKNOWN_LOCATION,
2005 OMP_CLAUSE__LOOPTEMP_);
2006 OMP_CLAUSE_DECL (c) = temp;
2007 OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
2008 gimple_omp_parallel_set_clauses (stmt, c);
2013 ctx = new_omp_context (stmt, outer_ctx);
2014 if (taskreg_nesting_level > 1)
2015 ctx->is_nested = true;
2016 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2017 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2018 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2019 name = create_tmp_var_name (".omp_data_s");
2020 name = build_decl (gimple_location (stmt),
2021 TYPE_DECL, name, ctx->record_type);
2022 DECL_ARTIFICIAL (name) = 1;
2023 DECL_NAMELESS (name) = 1;
2024 TYPE_NAME (ctx->record_type) = name;
2025 create_omp_child_function (ctx, false);
2026 gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2028 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2029 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2031 if (TYPE_FIELDS (ctx->record_type) == NULL)
2032 ctx->record_type = ctx->receiver_decl = NULL;
2035 layout_type (ctx->record_type);
2036 fixup_child_record_type (ctx);
2040 /* Scan an OpenMP task directive. */
2043 scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2047 gimple stmt = gsi_stmt (*gsi);
2048 location_t loc = gimple_location (stmt);
2050 /* Ignore task directives with empty bodies. */
2052 && empty_body_p (gimple_omp_body (stmt)))
2054 gsi_replace (gsi, gimple_build_nop (), false);
2058 ctx = new_omp_context (stmt, outer_ctx);
2059 if (taskreg_nesting_level > 1)
2060 ctx->is_nested = true;
2061 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2062 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2063 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2064 name = create_tmp_var_name (".omp_data_s");
2065 name = build_decl (gimple_location (stmt),
2066 TYPE_DECL, name, ctx->record_type);
2067 DECL_ARTIFICIAL (name) = 1;
2068 DECL_NAMELESS (name) = 1;
2069 TYPE_NAME (ctx->record_type) = name;
2070 create_omp_child_function (ctx, false);
2071 gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2073 scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2075 if (ctx->srecord_type)
2077 name = create_tmp_var_name (".omp_data_a");
2078 name = build_decl (gimple_location (stmt),
2079 TYPE_DECL, name, ctx->srecord_type);
2080 DECL_ARTIFICIAL (name) = 1;
2081 DECL_NAMELESS (name) = 1;
2082 TYPE_NAME (ctx->srecord_type) = name;
2083 create_omp_child_function (ctx, true);
2086 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2088 if (TYPE_FIELDS (ctx->record_type) == NULL)
2090 ctx->record_type = ctx->receiver_decl = NULL;
2091 t = build_int_cst (long_integer_type_node, 0);
2092 gimple_omp_task_set_arg_size (stmt, t);
2093 t = build_int_cst (long_integer_type_node, 1);
2094 gimple_omp_task_set_arg_align (stmt, t);
2098 tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2099 /* Move VLA fields to the end. */
2100 p = &TYPE_FIELDS (ctx->record_type);
2102 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2103 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2106 *p = TREE_CHAIN (*p);
2107 TREE_CHAIN (*q) = NULL_TREE;
2108 q = &TREE_CHAIN (*q);
2111 p = &DECL_CHAIN (*p);
2113 layout_type (ctx->record_type);
2114 fixup_child_record_type (ctx);
2115 if (ctx->srecord_type)
2116 layout_type (ctx->srecord_type);
2117 t = fold_convert_loc (loc, long_integer_type_node,
2118 TYPE_SIZE_UNIT (ctx->record_type));
2119 gimple_omp_task_set_arg_size (stmt, t);
2120 t = build_int_cst (long_integer_type_node,
2121 TYPE_ALIGN_UNIT (ctx->record_type));
2122 gimple_omp_task_set_arg_align (stmt, t);
2127 /* Scan an OpenMP loop directive. */
2130 scan_omp_for (gimple stmt, omp_context *outer_ctx)
2135 ctx = new_omp_context (stmt, outer_ctx);
2137 scan_sharing_clauses (gimple_omp_for_clauses (stmt), ctx);
2139 scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
2140 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
2142 scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
2143 scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
2144 scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
2145 scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
2147 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2150 /* Scan an OpenMP sections directive. */
2153 scan_omp_sections (gimple stmt, omp_context *outer_ctx)
2157 ctx = new_omp_context (stmt, outer_ctx);
2158 scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
2159 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2162 /* Scan an OpenMP single directive. */
2165 scan_omp_single (gimple stmt, omp_context *outer_ctx)
2170 ctx = new_omp_context (stmt, outer_ctx);
2171 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2172 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2173 name = create_tmp_var_name (".omp_copy_s");
2174 name = build_decl (gimple_location (stmt),
2175 TYPE_DECL, name, ctx->record_type);
2176 TYPE_NAME (ctx->record_type) = name;
2178 scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
2179 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2181 if (TYPE_FIELDS (ctx->record_type) == NULL)
2182 ctx->record_type = NULL;
2184 layout_type (ctx->record_type);
2187 /* Scan an OpenMP target{, data, update} directive. */
2190 scan_omp_target (gimple stmt, omp_context *outer_ctx)
2194 int kind = gimple_omp_target_kind (stmt);
2196 ctx = new_omp_context (stmt, outer_ctx);
2197 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2198 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2199 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2200 name = create_tmp_var_name (".omp_data_t");
2201 name = build_decl (gimple_location (stmt),
2202 TYPE_DECL, name, ctx->record_type);
2203 DECL_ARTIFICIAL (name) = 1;
2204 DECL_NAMELESS (name) = 1;
2205 TYPE_NAME (ctx->record_type) = name;
2206 if (kind == GF_OMP_TARGET_KIND_REGION)
2208 create_omp_child_function (ctx, false);
2209 gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
2212 scan_sharing_clauses (gimple_omp_target_clauses (stmt), ctx);
2213 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2215 if (TYPE_FIELDS (ctx->record_type) == NULL)
2216 ctx->record_type = ctx->receiver_decl = NULL;
2219 TYPE_FIELDS (ctx->record_type)
2220 = nreverse (TYPE_FIELDS (ctx->record_type));
2221 #ifdef ENABLE_CHECKING
2223 unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
2224 for (field = TYPE_FIELDS (ctx->record_type);
2226 field = DECL_CHAIN (field))
2227 gcc_assert (DECL_ALIGN (field) == align);
2229 layout_type (ctx->record_type);
2230 if (kind == GF_OMP_TARGET_KIND_REGION)
2231 fixup_child_record_type (ctx);
2235 /* Scan an OpenMP teams directive. */
2238 scan_omp_teams (gimple stmt, omp_context *outer_ctx)
2240 omp_context *ctx = new_omp_context (stmt, outer_ctx);
2241 scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
2242 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2245 /* Check OpenMP nesting restrictions. */
2247 check_omp_nesting_restrictions (gimple stmt, omp_context *ctx)
2251 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
2252 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
2254 error_at (gimple_location (stmt),
2255 "OpenMP constructs may not be nested inside simd region");
2258 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2260 if ((gimple_code (stmt) != GIMPLE_OMP_FOR
2261 || (gimple_omp_for_kind (stmt)
2262 != GF_OMP_FOR_KIND_DISTRIBUTE))
2263 && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
2265 error_at (gimple_location (stmt),
2266 "only distribute or parallel constructs are allowed to "
2267 "be closely nested inside teams construct");
2272 switch (gimple_code (stmt))
2274 case GIMPLE_OMP_FOR:
2275 if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_SIMD)
2277 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
2279 if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
2281 error_at (gimple_location (stmt),
2282 "distribute construct must be closely nested inside "
2290 if (is_gimple_call (stmt)
2291 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2292 == BUILT_IN_GOMP_CANCEL
2293 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2294 == BUILT_IN_GOMP_CANCELLATION_POINT))
2296 const char *bad = NULL;
2297 const char *kind = NULL;
2300 error_at (gimple_location (stmt), "orphaned %qs construct",
2301 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2302 == BUILT_IN_GOMP_CANCEL
2303 ? "#pragma omp cancel"
2304 : "#pragma omp cancellation point");
2307 switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
2308 ? tree_to_shwi (gimple_call_arg (stmt, 0))
2312 if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
2313 bad = "#pragma omp parallel";
2314 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2315 == BUILT_IN_GOMP_CANCEL
2316 && !integer_zerop (gimple_call_arg (stmt, 1)))
2317 ctx->cancellable = true;
2321 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
2322 || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
2323 bad = "#pragma omp for";
2324 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2325 == BUILT_IN_GOMP_CANCEL
2326 && !integer_zerop (gimple_call_arg (stmt, 1)))
2328 ctx->cancellable = true;
2329 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2331 warning_at (gimple_location (stmt), 0,
2332 "%<#pragma omp cancel for%> inside "
2333 "%<nowait%> for construct");
2334 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2335 OMP_CLAUSE_ORDERED))
2336 warning_at (gimple_location (stmt), 0,
2337 "%<#pragma omp cancel for%> inside "
2338 "%<ordered%> for construct");
2343 if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
2344 && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
2345 bad = "#pragma omp sections";
2346 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2347 == BUILT_IN_GOMP_CANCEL
2348 && !integer_zerop (gimple_call_arg (stmt, 1)))
2350 if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
2352 ctx->cancellable = true;
2353 if (find_omp_clause (gimple_omp_sections_clauses
2356 warning_at (gimple_location (stmt), 0,
2357 "%<#pragma omp cancel sections%> inside "
2358 "%<nowait%> sections construct");
2362 gcc_assert (ctx->outer
2363 && gimple_code (ctx->outer->stmt)
2364 == GIMPLE_OMP_SECTIONS);
2365 ctx->outer->cancellable = true;
2366 if (find_omp_clause (gimple_omp_sections_clauses
2369 warning_at (gimple_location (stmt), 0,
2370 "%<#pragma omp cancel sections%> inside "
2371 "%<nowait%> sections construct");
2377 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK)
2378 bad = "#pragma omp task";
2380 ctx->cancellable = true;
2384 error_at (gimple_location (stmt), "invalid arguments");
2389 error_at (gimple_location (stmt),
2390 "%<%s %s%> construct not closely nested inside of %qs",
2391 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2392 == BUILT_IN_GOMP_CANCEL
2393 ? "#pragma omp cancel"
2394 : "#pragma omp cancellation point", kind, bad);
2399 case GIMPLE_OMP_SECTIONS:
2400 case GIMPLE_OMP_SINGLE:
2401 for (; ctx != NULL; ctx = ctx->outer)
2402 switch (gimple_code (ctx->stmt))
2404 case GIMPLE_OMP_FOR:
2405 case GIMPLE_OMP_SECTIONS:
2406 case GIMPLE_OMP_SINGLE:
2407 case GIMPLE_OMP_ORDERED:
2408 case GIMPLE_OMP_MASTER:
2409 case GIMPLE_OMP_TASK:
2410 case GIMPLE_OMP_CRITICAL:
2411 if (is_gimple_call (stmt))
2413 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2414 != BUILT_IN_GOMP_BARRIER)
2416 error_at (gimple_location (stmt),
2417 "barrier region may not be closely nested inside "
2418 "of work-sharing, critical, ordered, master or "
2419 "explicit task region");
2422 error_at (gimple_location (stmt),
2423 "work-sharing region may not be closely nested inside "
2424 "of work-sharing, critical, ordered, master or explicit "
2427 case GIMPLE_OMP_PARALLEL:
2433 case GIMPLE_OMP_MASTER:
2434 for (; ctx != NULL; ctx = ctx->outer)
2435 switch (gimple_code (ctx->stmt))
2437 case GIMPLE_OMP_FOR:
2438 case GIMPLE_OMP_SECTIONS:
2439 case GIMPLE_OMP_SINGLE:
2440 case GIMPLE_OMP_TASK:
2441 error_at (gimple_location (stmt),
2442 "master region may not be closely nested inside "
2443 "of work-sharing or explicit task region");
2445 case GIMPLE_OMP_PARALLEL:
2451 case GIMPLE_OMP_ORDERED:
2452 for (; ctx != NULL; ctx = ctx->outer)
2453 switch (gimple_code (ctx->stmt))
2455 case GIMPLE_OMP_CRITICAL:
2456 case GIMPLE_OMP_TASK:
2457 error_at (gimple_location (stmt),
2458 "ordered region may not be closely nested inside "
2459 "of critical or explicit task region");
2461 case GIMPLE_OMP_FOR:
2462 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2463 OMP_CLAUSE_ORDERED) == NULL)
2465 error_at (gimple_location (stmt),
2466 "ordered region must be closely nested inside "
2467 "a loop region with an ordered clause");
2471 case GIMPLE_OMP_PARALLEL:
2472 error_at (gimple_location (stmt),
2473 "ordered region must be closely nested inside "
2474 "a loop region with an ordered clause");
2480 case GIMPLE_OMP_CRITICAL:
2481 for (; ctx != NULL; ctx = ctx->outer)
2482 if (gimple_code (ctx->stmt) == GIMPLE_OMP_CRITICAL
2483 && (gimple_omp_critical_name (stmt)
2484 == gimple_omp_critical_name (ctx->stmt)))
2486 error_at (gimple_location (stmt),
2487 "critical region may not be nested inside a critical "
2488 "region with the same name");
2492 case GIMPLE_OMP_TEAMS:
2494 || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
2495 || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION)
2497 error_at (gimple_location (stmt),
2498 "teams construct not closely nested inside of target "
2510 /* Helper function scan_omp.
2512 Callback for walk_tree or operators in walk_gimple_stmt used to
2513 scan for OpenMP directives in TP. */
2516 scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
2518 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
2519 omp_context *ctx = (omp_context *) wi->info;
2522 switch (TREE_CODE (t))
2529 *tp = remap_decl (t, &ctx->cb);
2533 if (ctx && TYPE_P (t))
2534 *tp = remap_type (t, &ctx->cb);
2535 else if (!DECL_P (t))
2540 tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
2541 if (tem != TREE_TYPE (t))
2543 if (TREE_CODE (t) == INTEGER_CST)
2544 *tp = wide_int_to_tree (tem, t);
2546 TREE_TYPE (t) = tem;
2556 /* Return true if FNDECL is a setjmp or a longjmp. */
2559 setjmp_or_longjmp_p (const_tree fndecl)
2561 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
2562 && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP
2563 || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP))
2566 tree declname = DECL_NAME (fndecl);
2569 const char *name = IDENTIFIER_POINTER (declname);
2570 return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
2574 /* Helper function for scan_omp.
2576 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2577 the current statement in GSI. */
2580 scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
2581 struct walk_stmt_info *wi)
2583 gimple stmt = gsi_stmt (*gsi);
2584 omp_context *ctx = (omp_context *) wi->info;
2586 if (gimple_has_location (stmt))
2587 input_location = gimple_location (stmt);
2589 /* Check the OpenMP nesting restrictions. */
2590 bool remove = false;
2591 if (is_gimple_omp (stmt))
2592 remove = !check_omp_nesting_restrictions (stmt, ctx);
2593 else if (is_gimple_call (stmt))
2595 tree fndecl = gimple_call_fndecl (stmt);
2598 if (setjmp_or_longjmp_p (fndecl)
2600 && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
2601 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
2604 error_at (gimple_location (stmt),
2605 "setjmp/longjmp inside simd construct");
2607 else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
2608 switch (DECL_FUNCTION_CODE (fndecl))
2610 case BUILT_IN_GOMP_BARRIER:
2611 case BUILT_IN_GOMP_CANCEL:
2612 case BUILT_IN_GOMP_CANCELLATION_POINT:
2613 case BUILT_IN_GOMP_TASKYIELD:
2614 case BUILT_IN_GOMP_TASKWAIT:
2615 case BUILT_IN_GOMP_TASKGROUP_START:
2616 case BUILT_IN_GOMP_TASKGROUP_END:
2617 remove = !check_omp_nesting_restrictions (stmt, ctx);
2626 stmt = gimple_build_nop ();
2627 gsi_replace (gsi, stmt, false);
2630 *handled_ops_p = true;
2632 switch (gimple_code (stmt))
2634 case GIMPLE_OMP_PARALLEL:
2635 taskreg_nesting_level++;
2636 scan_omp_parallel (gsi, ctx);
2637 taskreg_nesting_level--;
2640 case GIMPLE_OMP_TASK:
2641 taskreg_nesting_level++;
2642 scan_omp_task (gsi, ctx);
2643 taskreg_nesting_level--;
2646 case GIMPLE_OMP_FOR:
2647 scan_omp_for (stmt, ctx);
2650 case GIMPLE_OMP_SECTIONS:
2651 scan_omp_sections (stmt, ctx);
2654 case GIMPLE_OMP_SINGLE:
2655 scan_omp_single (stmt, ctx);
2658 case GIMPLE_OMP_SECTION:
2659 case GIMPLE_OMP_MASTER:
2660 case GIMPLE_OMP_TASKGROUP:
2661 case GIMPLE_OMP_ORDERED:
2662 case GIMPLE_OMP_CRITICAL:
2663 ctx = new_omp_context (stmt, ctx);
2664 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2667 case GIMPLE_OMP_TARGET:
2668 scan_omp_target (stmt, ctx);
2671 case GIMPLE_OMP_TEAMS:
2672 scan_omp_teams (stmt, ctx);
2679 *handled_ops_p = false;
2681 for (var = gimple_bind_vars (stmt); var ; var = DECL_CHAIN (var))
2682 insert_decl_map (&ctx->cb, var, var);
2686 *handled_ops_p = false;
2694 /* Scan all the statements starting at the current statement. CTX
2695 contains context information about the OpenMP directives and
2696 clauses found during the scan. */
2699 scan_omp (gimple_seq *body_p, omp_context *ctx)
2701 location_t saved_location;
2702 struct walk_stmt_info wi;
2704 memset (&wi, 0, sizeof (wi));
2706 wi.want_locations = true;
2708 saved_location = input_location;
2709 walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
2710 input_location = saved_location;
2713 /* Re-gimplification and code generation routines. */
2715 /* Build a call to GOMP_barrier. */
2718 build_omp_barrier (tree lhs)
2720 tree fndecl = builtin_decl_explicit (lhs ? BUILT_IN_GOMP_BARRIER_CANCEL
2721 : BUILT_IN_GOMP_BARRIER);
2722 gimple g = gimple_build_call (fndecl, 0);
2724 gimple_call_set_lhs (g, lhs);
2728 /* If a context was created for STMT when it was scanned, return it. */
2730 static omp_context *
2731 maybe_lookup_ctx (gimple stmt)
2734 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
2735 return n ? (omp_context *) n->value : NULL;
2739 /* Find the mapping for DECL in CTX or the immediately enclosing
2740 context that has a mapping for DECL.
2742 If CTX is a nested parallel directive, we may have to use the decl
2743 mappings created in CTX's parent context. Suppose that we have the
2744 following parallel nesting (variable UIDs showed for clarity):
2747 #omp parallel shared(iD.1562) -> outer parallel
2748 iD.1562 = iD.1562 + 1;
2750 #omp parallel shared (iD.1562) -> inner parallel
2751 iD.1562 = iD.1562 - 1;
2753 Each parallel structure will create a distinct .omp_data_s structure
2754 for copying iD.1562 in/out of the directive:
2756 outer parallel .omp_data_s.1.i -> iD.1562
2757 inner parallel .omp_data_s.2.i -> iD.1562
2759 A shared variable mapping will produce a copy-out operation before
2760 the parallel directive and a copy-in operation after it. So, in
2761 this case we would have:
2764 .omp_data_o.1.i = iD.1562;
2765 #omp parallel shared(iD.1562) -> outer parallel
2766 .omp_data_i.1 = &.omp_data_o.1
2767 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2769 .omp_data_o.2.i = iD.1562; -> **
2770 #omp parallel shared(iD.1562) -> inner parallel
2771 .omp_data_i.2 = &.omp_data_o.2
2772 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2775 ** This is a problem. The symbol iD.1562 cannot be referenced
2776 inside the body of the outer parallel region. But since we are
2777 emitting this copy operation while expanding the inner parallel
2778 directive, we need to access the CTX structure of the outer
2779 parallel directive to get the correct mapping:
2781 .omp_data_o.2.i = .omp_data_i.1->i
2783 Since there may be other workshare or parallel directives enclosing
2784 the parallel directive, it may be necessary to walk up the context
2785 parent chain. This is not a problem in general because nested
2786 parallelism happens only rarely. */
2789 lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
2794 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
2795 t = maybe_lookup_decl (decl, up);
2797 gcc_assert (!ctx->is_nested || t || is_global_var (decl));
2799 return t ? t : decl;
2803 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2804 in outer contexts. */
2807 maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
2812 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
2813 t = maybe_lookup_decl (decl, up);
2815 return t ? t : decl;
2819 /* Construct the initialization value for reduction CLAUSE. */
2822 omp_reduction_init (tree clause, tree type)
2824 location_t loc = OMP_CLAUSE_LOCATION (clause);
2825 switch (OMP_CLAUSE_REDUCTION_CODE (clause))
2832 case TRUTH_ORIF_EXPR:
2833 case TRUTH_XOR_EXPR:
2835 return build_zero_cst (type);
2838 case TRUTH_AND_EXPR:
2839 case TRUTH_ANDIF_EXPR:
2841 return fold_convert_loc (loc, type, integer_one_node);
2844 return fold_convert_loc (loc, type, integer_minus_one_node);
2847 if (SCALAR_FLOAT_TYPE_P (type))
2849 REAL_VALUE_TYPE max, min;
2850 if (HONOR_INFINITIES (TYPE_MODE (type)))
2853 real_arithmetic (&min, NEGATE_EXPR, &max, NULL);
2856 real_maxval (&min, 1, TYPE_MODE (type));
2857 return build_real (type, min);
2861 gcc_assert (INTEGRAL_TYPE_P (type));
2862 return TYPE_MIN_VALUE (type);
2866 if (SCALAR_FLOAT_TYPE_P (type))
2868 REAL_VALUE_TYPE max;
2869 if (HONOR_INFINITIES (TYPE_MODE (type)))
2872 real_maxval (&max, 0, TYPE_MODE (type));
2873 return build_real (type, max);
2877 gcc_assert (INTEGRAL_TYPE_P (type));
2878 return TYPE_MAX_VALUE (type);
2886 /* Return alignment to be assumed for var in CLAUSE, which should be
2887 OMP_CLAUSE_ALIGNED. */
2890 omp_clause_aligned_alignment (tree clause)
2892 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
2893 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
2895 /* Otherwise return implementation defined alignment. */
2896 unsigned int al = 1;
2897 enum machine_mode mode, vmode;
2898 int vs = targetm.vectorize.autovectorize_vector_sizes ();
2900 vs = 1 << floor_log2 (vs);
2901 static enum mode_class classes[]
2902 = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
2903 for (int i = 0; i < 4; i += 2)
2904 for (mode = GET_CLASS_NARROWEST_MODE (classes[i]);
2906 mode = GET_MODE_WIDER_MODE (mode))
2908 vmode = targetm.vectorize.preferred_simd_mode (mode);
2909 if (GET_MODE_CLASS (vmode) != classes[i + 1])
2912 && GET_MODE_SIZE (vmode) < vs
2913 && GET_MODE_2XWIDER_MODE (vmode) != VOIDmode)
2914 vmode = GET_MODE_2XWIDER_MODE (vmode);
2916 tree type = lang_hooks.types.type_for_mode (mode, 1);
2917 if (type == NULL_TREE || TYPE_MODE (type) != mode)
2919 type = build_vector_type (type, GET_MODE_SIZE (vmode)
2920 / GET_MODE_SIZE (mode));
2921 if (TYPE_MODE (type) != vmode)
2923 if (TYPE_ALIGN_UNIT (type) > al)
2924 al = TYPE_ALIGN_UNIT (type);
2926 return build_int_cst (integer_type_node, al);
2929 /* Return maximum possible vectorization factor for the target. */
2936 || !flag_tree_loop_optimize
2937 || (!flag_tree_loop_vectorize
2938 && (global_options_set.x_flag_tree_loop_vectorize
2939 || global_options_set.x_flag_tree_vectorize)))
2942 int vs = targetm.vectorize.autovectorize_vector_sizes ();
2945 vs = 1 << floor_log2 (vs);
2948 enum machine_mode vqimode = targetm.vectorize.preferred_simd_mode (QImode);
2949 if (GET_MODE_CLASS (vqimode) == MODE_VECTOR_INT)
2950 return GET_MODE_NUNITS (vqimode);
2954 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2958 lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, int &max_vf,
2959 tree &idx, tree &lane, tree &ivar, tree &lvar)
2963 max_vf = omp_max_vf ();
2966 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2967 OMP_CLAUSE_SAFELEN);
2969 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c), max_vf) == -1)
2970 max_vf = tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c));
2974 idx = create_tmp_var (unsigned_type_node, NULL);
2975 lane = create_tmp_var (unsigned_type_node, NULL);
2981 tree atype = build_array_type_nelts (TREE_TYPE (new_var), max_vf);
2982 tree avar = create_tmp_var_raw (atype, NULL);
2983 if (TREE_ADDRESSABLE (new_var))
2984 TREE_ADDRESSABLE (avar) = 1;
2985 DECL_ATTRIBUTES (avar)
2986 = tree_cons (get_identifier ("omp simd array"), NULL,
2987 DECL_ATTRIBUTES (avar));
2988 gimple_add_tmp_var (avar);
2989 ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, idx,
2990 NULL_TREE, NULL_TREE);
2991 lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, lane,
2992 NULL_TREE, NULL_TREE);
2993 if (DECL_P (new_var))
2995 SET_DECL_VALUE_EXPR (new_var, lvar);
2996 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3001 /* Helper function of lower_rec_input_clauses. For a reference
3002 in simd reduction, add an underlying variable it will reference. */
3005 handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
3007 tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
3008 if (TREE_CONSTANT (z))
3010 const char *name = NULL;
3011 if (DECL_NAME (new_vard))
3012 name = IDENTIFIER_POINTER (DECL_NAME (new_vard));
3014 z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)), name);
3015 gimple_add_tmp_var (z);
3016 TREE_ADDRESSABLE (z) = 1;
3017 z = build_fold_addr_expr_loc (loc, z);
3018 gimplify_assign (new_vard, z, ilist);
3022 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3023 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3024 private variables. Initialization statements go in ILIST, while calls
3025 to destructors go in DLIST. */
3028 lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
3029 omp_context *ctx, struct omp_for_data *fd)
3031 tree c, dtor, copyin_seq, x, ptr;
3032 bool copyin_by_ref = false;
3033 bool lastprivate_firstprivate = false;
3034 bool reduction_omp_orig_ref = false;
3036 bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3037 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD);
3039 tree lane = NULL_TREE, idx = NULL_TREE;
3040 tree ivar = NULL_TREE, lvar = NULL_TREE;
3041 gimple_seq llist[2] = { NULL, NULL };
3045 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3046 with data sharing clauses referencing variable sized vars. That
3047 is unnecessarily hard to support and very unlikely to result in
3048 vectorized code anyway. */
3050 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3051 switch (OMP_CLAUSE_CODE (c))
3053 case OMP_CLAUSE_REDUCTION:
3054 case OMP_CLAUSE_PRIVATE:
3055 case OMP_CLAUSE_FIRSTPRIVATE:
3056 case OMP_CLAUSE_LASTPRIVATE:
3057 case OMP_CLAUSE_LINEAR:
3058 if (is_variable_sized (OMP_CLAUSE_DECL (c)))
3065 /* Do all the fixed sized types in the first pass, and the variable sized
3066 types in the second pass. This makes sure that the scalar arguments to
3067 the variable sized types are processed before we use them in the
3068 variable sized operations. */
3069 for (pass = 0; pass < 2; ++pass)
3071 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3073 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
3076 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3080 case OMP_CLAUSE_PRIVATE:
3081 if (OMP_CLAUSE_PRIVATE_DEBUG (c))
3084 case OMP_CLAUSE_SHARED:
3085 /* Ignore shared directives in teams construct. */
3086 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3088 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
3090 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c)));
3093 case OMP_CLAUSE_FIRSTPRIVATE:
3094 case OMP_CLAUSE_COPYIN:
3095 case OMP_CLAUSE_LINEAR:
3097 case OMP_CLAUSE_REDUCTION:
3098 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
3099 reduction_omp_orig_ref = true;
3101 case OMP_CLAUSE__LOOPTEMP_:
3102 /* Handle _looptemp_ clauses only on parallel. */
3106 case OMP_CLAUSE_LASTPRIVATE:
3107 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3109 lastprivate_firstprivate = true;
3114 case OMP_CLAUSE_ALIGNED:
3117 var = OMP_CLAUSE_DECL (c);
3118 if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
3119 && !is_global_var (var))
3121 new_var = maybe_lookup_decl (var, ctx);
3122 if (new_var == NULL_TREE)
3123 new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
3124 x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3125 x = build_call_expr_loc (clause_loc, x, 2, new_var,
3126 omp_clause_aligned_alignment (c));
3127 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3128 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
3129 gimplify_and_add (x, ilist);
3131 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
3132 && is_global_var (var))
3134 tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
3135 new_var = lookup_decl (var, ctx);
3136 t = maybe_lookup_decl_in_outer_ctx (var, ctx);
3137 t = build_fold_addr_expr_loc (clause_loc, t);
3138 t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3139 t = build_call_expr_loc (clause_loc, t2, 2, t,
3140 omp_clause_aligned_alignment (c));
3141 t = fold_convert_loc (clause_loc, ptype, t);
3142 x = create_tmp_var (ptype, NULL);
3143 t = build2 (MODIFY_EXPR, ptype, x, t);
3144 gimplify_and_add (t, ilist);
3145 t = build_simple_mem_ref_loc (clause_loc, x);
3146 SET_DECL_VALUE_EXPR (new_var, t);
3147 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3154 new_var = var = OMP_CLAUSE_DECL (c);
3155 if (c_kind != OMP_CLAUSE_COPYIN)
3156 new_var = lookup_decl (var, ctx);
3158 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
3163 else if (is_variable_sized (var))
3165 /* For variable sized types, we need to allocate the
3166 actual storage here. Call alloca and store the
3167 result in the pointer decl that we created elsewhere. */
3171 if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
3176 ptr = DECL_VALUE_EXPR (new_var);
3177 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
3178 ptr = TREE_OPERAND (ptr, 0);
3179 gcc_assert (DECL_P (ptr));
3180 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
3182 /* void *tmp = __builtin_alloca */
3183 atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3184 stmt = gimple_build_call (atmp, 1, x);
3185 tmp = create_tmp_var_raw (ptr_type_node, NULL);
3186 gimple_add_tmp_var (tmp);
3187 gimple_call_set_lhs (stmt, tmp);
3189 gimple_seq_add_stmt (ilist, stmt);
3191 x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
3192 gimplify_assign (ptr, x, ilist);
3195 else if (is_reference (var))
3197 /* For references that are being privatized for Fortran,
3198 allocate new backing storage for the new pointer
3199 variable. This allows us to avoid changing all the
3200 code that expects a pointer to something that expects
3201 a direct variable. */
3205 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
3206 if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
3208 x = build_receiver_ref (var, false, ctx);
3209 x = build_fold_addr_expr_loc (clause_loc, x);
3211 else if (TREE_CONSTANT (x))
3213 /* For reduction in SIMD loop, defer adding the
3214 initialization of the reference, because if we decide
3215 to use SIMD array for it, the initilization could cause
3217 if (c_kind == OMP_CLAUSE_REDUCTION && is_simd)
3221 const char *name = NULL;
3222 if (DECL_NAME (var))
3223 name = IDENTIFIER_POINTER (DECL_NAME (new_var));
3225 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
3227 gimple_add_tmp_var (x);
3228 TREE_ADDRESSABLE (x) = 1;
3229 x = build_fold_addr_expr_loc (clause_loc, x);
3234 tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3235 x = build_call_expr_loc (clause_loc, atmp, 1, x);
3240 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3241 gimplify_assign (new_var, x, ilist);
3244 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3246 else if (c_kind == OMP_CLAUSE_REDUCTION
3247 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3255 switch (OMP_CLAUSE_CODE (c))
3257 case OMP_CLAUSE_SHARED:
3258 /* Ignore shared directives in teams construct. */
3259 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3261 /* Shared global vars are just accessed directly. */
3262 if (is_global_var (new_var))
3264 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3265 needs to be delayed until after fixup_child_record_type so
3266 that we get the correct type during the dereference. */
3267 by_ref = use_pointer_for_field (var, ctx);
3268 x = build_receiver_ref (var, by_ref, ctx);
3269 SET_DECL_VALUE_EXPR (new_var, x);
3270 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3272 /* ??? If VAR is not passed by reference, and the variable
3273 hasn't been initialized yet, then we'll get a warning for
3274 the store into the omp_data_s structure. Ideally, we'd be
3275 able to notice this and not store anything at all, but
3276 we're generating code too early. Suppress the warning. */
3278 TREE_NO_WARNING (var) = 1;
3281 case OMP_CLAUSE_LASTPRIVATE:
3282 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3286 case OMP_CLAUSE_PRIVATE:
3287 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
3288 x = build_outer_var_ref (var, ctx);
3289 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
3291 if (is_task_ctx (ctx))
3292 x = build_receiver_ref (var, false, ctx);
3294 x = build_outer_var_ref (var, ctx);
3300 nx = lang_hooks.decls.omp_clause_default_ctor (c, new_var, x);
3303 tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
3304 if ((TREE_ADDRESSABLE (new_var) || nx || y
3305 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
3306 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3307 idx, lane, ivar, lvar))
3310 x = lang_hooks.decls.omp_clause_default_ctor
3311 (c, unshare_expr (ivar), x);
3313 gimplify_and_add (x, &llist[0]);
3316 y = lang_hooks.decls.omp_clause_dtor (c, ivar);
3319 gimple_seq tseq = NULL;
3322 gimplify_stmt (&dtor, &tseq);
3323 gimple_seq_add_seq (&llist[1], tseq);
3330 gimplify_and_add (nx, ilist);
3334 x = lang_hooks.decls.omp_clause_dtor (c, new_var);
3337 gimple_seq tseq = NULL;
3340 gimplify_stmt (&dtor, &tseq);
3341 gimple_seq_add_seq (dlist, tseq);
3345 case OMP_CLAUSE_LINEAR:
3346 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
3347 goto do_firstprivate;
3348 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
3351 x = build_outer_var_ref (var, ctx);
3354 case OMP_CLAUSE_FIRSTPRIVATE:
3355 if (is_task_ctx (ctx))
3357 if (is_reference (var) || is_variable_sized (var))
3359 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
3361 || use_pointer_for_field (var, NULL))
3363 x = build_receiver_ref (var, false, ctx);
3364 SET_DECL_VALUE_EXPR (new_var, x);
3365 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3370 x = build_outer_var_ref (var, ctx);
3373 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
3374 && gimple_omp_for_combined_into_p (ctx->stmt))
3376 tree stept = POINTER_TYPE_P (TREE_TYPE (x))
3377 ? sizetype : TREE_TYPE (x);
3378 tree t = fold_convert (stept,
3379 OMP_CLAUSE_LINEAR_STEP (c));
3380 tree c = find_omp_clause (clauses,
3381 OMP_CLAUSE__LOOPTEMP_);
3383 tree l = OMP_CLAUSE_DECL (c);
3384 if (fd->collapse == 1)
3386 tree n1 = fd->loop.n1;
3387 tree step = fd->loop.step;
3388 tree itype = TREE_TYPE (l);
3389 if (POINTER_TYPE_P (itype))
3390 itype = signed_type_for (itype);
3391 l = fold_build2 (MINUS_EXPR, itype, l, n1);
3392 if (TYPE_UNSIGNED (itype)
3393 && fd->loop.cond_code == GT_EXPR)
3394 l = fold_build2 (TRUNC_DIV_EXPR, itype,
3395 fold_build1 (NEGATE_EXPR,
3397 fold_build1 (NEGATE_EXPR,
3400 l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
3402 t = fold_build2 (MULT_EXPR, stept,
3403 fold_convert (stept, l), t);
3404 if (POINTER_TYPE_P (TREE_TYPE (x)))
3405 x = fold_build2 (POINTER_PLUS_EXPR,
3406 TREE_TYPE (x), x, t);
3408 x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, t);
3411 if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
3412 || TREE_ADDRESSABLE (new_var))
3413 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3414 idx, lane, ivar, lvar))
3416 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
3418 tree iv = create_tmp_var (TREE_TYPE (new_var), NULL);
3419 x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
3420 gimplify_and_add (x, ilist);
3421 gimple_stmt_iterator gsi
3422 = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
3424 = gimple_build_assign (unshare_expr (lvar), iv);
3425 gsi_insert_before_without_update (&gsi, g,
3427 tree stept = POINTER_TYPE_P (TREE_TYPE (iv))
3428 ? sizetype : TREE_TYPE (iv);
3429 tree t = fold_convert (stept,
3430 OMP_CLAUSE_LINEAR_STEP (c));
3431 enum tree_code code = PLUS_EXPR;
3432 if (POINTER_TYPE_P (TREE_TYPE (new_var)))
3433 code = POINTER_PLUS_EXPR;
3434 g = gimple_build_assign_with_ops (code, iv, iv, t);
3435 gsi_insert_before_without_update (&gsi, g,
3439 x = lang_hooks.decls.omp_clause_copy_ctor
3440 (c, unshare_expr (ivar), x);
3441 gimplify_and_add (x, &llist[0]);
3442 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
3445 gimple_seq tseq = NULL;
3448 gimplify_stmt (&dtor, &tseq);
3449 gimple_seq_add_seq (&llist[1], tseq);
3454 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
3455 gimplify_and_add (x, ilist);
3458 case OMP_CLAUSE__LOOPTEMP_:
3459 gcc_assert (is_parallel_ctx (ctx));
3460 x = build_outer_var_ref (var, ctx);
3461 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
3462 gimplify_and_add (x, ilist);
3465 case OMP_CLAUSE_COPYIN:
3466 by_ref = use_pointer_for_field (var, NULL);
3467 x = build_receiver_ref (var, by_ref, ctx);
3468 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
3469 append_to_statement_list (x, ©in_seq);
3470 copyin_by_ref |= by_ref;
3473 case OMP_CLAUSE_REDUCTION:
3474 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3476 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
3478 x = build_outer_var_ref (var, ctx);
3480 if (is_reference (var)
3481 && !useless_type_conversion_p (TREE_TYPE (placeholder),
3483 x = build_fold_addr_expr_loc (clause_loc, x);
3484 SET_DECL_VALUE_EXPR (placeholder, x);
3485 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
3486 tree new_vard = new_var;
3487 if (is_reference (var))
3489 gcc_assert (TREE_CODE (new_var) == MEM_REF);
3490 new_vard = TREE_OPERAND (new_var, 0);
3491 gcc_assert (DECL_P (new_vard));
3494 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3495 idx, lane, ivar, lvar))
3497 if (new_vard == new_var)
3499 gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
3500 SET_DECL_VALUE_EXPR (new_var, ivar);
3504 SET_DECL_VALUE_EXPR (new_vard,
3505 build_fold_addr_expr (ivar));
3506 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
3508 x = lang_hooks.decls.omp_clause_default_ctor
3509 (c, unshare_expr (ivar),
3510 build_outer_var_ref (var, ctx));
3512 gimplify_and_add (x, &llist[0]);
3513 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
3515 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
3516 lower_omp (&tseq, ctx);
3517 gimple_seq_add_seq (&llist[0], tseq);
3519 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
3520 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
3521 lower_omp (&tseq, ctx);
3522 gimple_seq_add_seq (&llist[1], tseq);
3523 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
3524 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
3525 if (new_vard == new_var)
3526 SET_DECL_VALUE_EXPR (new_var, lvar);
3528 SET_DECL_VALUE_EXPR (new_vard,
3529 build_fold_addr_expr (lvar));
3530 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
3535 gimplify_stmt (&dtor, &tseq);
3536 gimple_seq_add_seq (&llist[1], tseq);
3540 /* If this is a reference to constant size reduction var
3541 with placeholder, we haven't emitted the initializer
3542 for it because it is undesirable if SIMD arrays are used.
3543 But if they aren't used, we need to emit the deferred
3544 initialization now. */
3545 else if (is_reference (var) && is_simd)
3546 handle_simd_reference (clause_loc, new_vard, ilist);
3547 x = lang_hooks.decls.omp_clause_default_ctor
3548 (c, new_var, unshare_expr (x));
3550 gimplify_and_add (x, ilist);
3551 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
3553 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
3554 lower_omp (&tseq, ctx);
3555 gimple_seq_add_seq (ilist, tseq);
3557 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
3560 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
3561 lower_omp (&tseq, ctx);
3562 gimple_seq_add_seq (dlist, tseq);
3563 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
3565 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
3570 x = omp_reduction_init (c, TREE_TYPE (new_var));
3571 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
3572 enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
3574 /* reduction(-:var) sums up the partial results, so it
3575 acts identically to reduction(+:var). */
3576 if (code == MINUS_EXPR)
3579 tree new_vard = new_var;
3580 if (is_simd && is_reference (var))
3582 gcc_assert (TREE_CODE (new_var) == MEM_REF);
3583 new_vard = TREE_OPERAND (new_var, 0);
3584 gcc_assert (DECL_P (new_vard));
3587 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3588 idx, lane, ivar, lvar))
3590 tree ref = build_outer_var_ref (var, ctx);
3592 gimplify_assign (unshare_expr (ivar), x, &llist[0]);
3594 x = build2 (code, TREE_TYPE (ref), ref, ivar);
3595 ref = build_outer_var_ref (var, ctx);
3596 gimplify_assign (ref, x, &llist[1]);
3598 if (new_vard != new_var)
3600 SET_DECL_VALUE_EXPR (new_vard,
3601 build_fold_addr_expr (lvar));
3602 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
3607 if (is_reference (var) && is_simd)
3608 handle_simd_reference (clause_loc, new_vard, ilist);
3609 gimplify_assign (new_var, x, ilist);
3612 tree ref = build_outer_var_ref (var, ctx);
3614 x = build2 (code, TREE_TYPE (ref), ref, new_var);
3615 ref = build_outer_var_ref (var, ctx);
3616 gimplify_assign (ref, x, dlist);
3630 tree uid = create_tmp_var (ptr_type_node, "simduid");
3631 /* Don't want uninit warnings on simduid, it is always uninitialized,
3632 but we use it not for the value, but for the DECL_UID only. */
3633 TREE_NO_WARNING (uid) = 1;
3635 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid);
3636 gimple_call_set_lhs (g, lane);
3637 gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
3638 gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
3639 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
3640 OMP_CLAUSE__SIMDUID__DECL (c) = uid;
3641 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
3642 gimple_omp_for_set_clauses (ctx->stmt, c);
3643 g = gimple_build_assign_with_ops (INTEGER_CST, lane,
3644 build_int_cst (unsigned_type_node, 0),
3646 gimple_seq_add_stmt (ilist, g);
3647 for (int i = 0; i < 2; i++)
3650 tree vf = create_tmp_var (unsigned_type_node, NULL);
3651 g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
3652 gimple_call_set_lhs (g, vf);
3653 gimple_seq *seq = i == 0 ? ilist : dlist;
3654 gimple_seq_add_stmt (seq, g);
3655 tree t = build_int_cst (unsigned_type_node, 0);
3656 g = gimple_build_assign_with_ops (INTEGER_CST, idx, t, NULL_TREE);
3657 gimple_seq_add_stmt (seq, g);
3658 tree body = create_artificial_label (UNKNOWN_LOCATION);
3659 tree header = create_artificial_label (UNKNOWN_LOCATION);
3660 tree end = create_artificial_label (UNKNOWN_LOCATION);
3661 gimple_seq_add_stmt (seq, gimple_build_goto (header));
3662 gimple_seq_add_stmt (seq, gimple_build_label (body));
3663 gimple_seq_add_seq (seq, llist[i]);
3664 t = build_int_cst (unsigned_type_node, 1);
3665 g = gimple_build_assign_with_ops (PLUS_EXPR, idx, idx, t);
3666 gimple_seq_add_stmt (seq, g);
3667 gimple_seq_add_stmt (seq, gimple_build_label (header));
3668 g = gimple_build_cond (LT_EXPR, idx, vf, body, end);
3669 gimple_seq_add_stmt (seq, g);
3670 gimple_seq_add_stmt (seq, gimple_build_label (end));
3674 /* The copyin sequence is not to be executed by the main thread, since
3675 that would result in self-copies. Perhaps not visible to scalars,
3676 but it certainly is to C++ operator=. */
3679 x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
3681 x = build2 (NE_EXPR, boolean_type_node, x,
3682 build_int_cst (TREE_TYPE (x), 0));
3683 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
3684 gimplify_and_add (x, ilist);
3687 /* If any copyin variable is passed by reference, we must ensure the
3688 master thread doesn't modify it before it is copied over in all
3689 threads. Similarly for variables in both firstprivate and
3690 lastprivate clauses we need to ensure the lastprivate copying
3691 happens after firstprivate copying in all threads. And similarly
3692 for UDRs if initializer expression refers to omp_orig. */
3693 if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref)
3695 /* Don't add any barrier for #pragma omp simd or
3696 #pragma omp distribute. */
3697 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3698 || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR)
3699 gimple_seq_add_stmt (ilist, build_omp_barrier (NULL_TREE));
3702 /* If max_vf is non-zero, then we can use only a vectorization factor
3703 up to the max_vf we chose. So stick it into the safelen clause. */
3706 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3707 OMP_CLAUSE_SAFELEN);
3709 || compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
3712 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
3713 OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
3715 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
3716 gimple_omp_for_set_clauses (ctx->stmt, c);
3722 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3723 both parallel and workshare constructs. PREDICATE may be NULL if it's
3727 lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list,
3730 tree x, c, label = NULL, orig_clauses = clauses;
3731 bool par_clauses = false;
3732 tree simduid = NULL, lastlane = NULL;
3734 /* Early exit if there are no lastprivate or linear clauses. */
3735 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
3736 if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
3737 || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
3738 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
3740 if (clauses == NULL)
3742 /* If this was a workshare clause, see if it had been combined
3743 with its parallel. In that case, look for the clauses on the
3744 parallel statement itself. */
3745 if (is_parallel_ctx (ctx))
3749 if (ctx == NULL || !is_parallel_ctx (ctx))
3752 clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
3753 OMP_CLAUSE_LASTPRIVATE);
3754 if (clauses == NULL)
3762 tree label_true, arm1, arm2;
3764 label = create_artificial_label (UNKNOWN_LOCATION);
3765 label_true = create_artificial_label (UNKNOWN_LOCATION);
3766 arm1 = TREE_OPERAND (predicate, 0);
3767 arm2 = TREE_OPERAND (predicate, 1);
3768 gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
3769 gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
3770 stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2,
3772 gimple_seq_add_stmt (stmt_list, stmt);
3773 gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
3776 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3777 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
3779 simduid = find_omp_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
3781 simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
3784 for (c = clauses; c ;)
3787 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3789 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
3790 || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
3791 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
3793 var = OMP_CLAUSE_DECL (c);
3794 new_var = lookup_decl (var, ctx);
3796 if (simduid && DECL_HAS_VALUE_EXPR_P (new_var))
3798 tree val = DECL_VALUE_EXPR (new_var);
3799 if (TREE_CODE (val) == ARRAY_REF
3800 && VAR_P (TREE_OPERAND (val, 0))
3801 && lookup_attribute ("omp simd array",
3802 DECL_ATTRIBUTES (TREE_OPERAND (val,
3805 if (lastlane == NULL)
3807 lastlane = create_tmp_var (unsigned_type_node, NULL);
3809 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
3811 TREE_OPERAND (val, 1));
3812 gimple_call_set_lhs (g, lastlane);
3813 gimple_seq_add_stmt (stmt_list, g);
3815 new_var = build4 (ARRAY_REF, TREE_TYPE (val),
3816 TREE_OPERAND (val, 0), lastlane,
3817 NULL_TREE, NULL_TREE);
3821 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
3822 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
3824 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
3825 gimple_seq_add_seq (stmt_list,
3826 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
3827 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
3829 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
3830 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
3832 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
3833 gimple_seq_add_seq (stmt_list,
3834 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
3835 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
3838 x = build_outer_var_ref (var, ctx);
3839 if (is_reference (var))
3840 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3841 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
3842 gimplify_and_add (x, stmt_list);
3844 c = OMP_CLAUSE_CHAIN (c);
3845 if (c == NULL && !par_clauses)
3847 /* If this was a workshare clause, see if it had been combined
3848 with its parallel. In that case, continue looking for the
3849 clauses also on the parallel statement itself. */
3850 if (is_parallel_ctx (ctx))
3854 if (ctx == NULL || !is_parallel_ctx (ctx))
3857 c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
3858 OMP_CLAUSE_LASTPRIVATE);
3864 gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
3868 /* Generate code to implement the REDUCTION clauses. */
3871 lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx)
3873 gimple_seq sub_seq = NULL;
3878 /* SIMD reductions are handled in lower_rec_input_clauses. */
3879 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3880 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
3883 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3884 update in that case, otherwise use a lock. */
3885 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
3886 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
3888 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3890 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3900 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3902 tree var, ref, new_var;
3903 enum tree_code code;
3904 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3906 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
3909 var = OMP_CLAUSE_DECL (c);
3910 new_var = lookup_decl (var, ctx);
3911 if (is_reference (var))
3912 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3913 ref = build_outer_var_ref (var, ctx);
3914 code = OMP_CLAUSE_REDUCTION_CODE (c);
3916 /* reduction(-:var) sums up the partial results, so it acts
3917 identically to reduction(+:var). */
3918 if (code == MINUS_EXPR)
3923 tree addr = build_fold_addr_expr_loc (clause_loc, ref);
3925 addr = save_expr (addr);
3926 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
3927 x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var);
3928 x = build2 (OMP_ATOMIC, void_type_node, addr, x);
3929 gimplify_and_add (x, stmt_seqp);
3933 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3935 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
3937 if (is_reference (var)
3938 && !useless_type_conversion_p (TREE_TYPE (placeholder),
3940 ref = build_fold_addr_expr_loc (clause_loc, ref);
3941 SET_DECL_VALUE_EXPR (placeholder, ref);
3942 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
3943 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
3944 gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
3945 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
3946 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
3950 x = build2 (code, TREE_TYPE (ref), ref, new_var);
3951 ref = build_outer_var_ref (var, ctx);
3952 gimplify_assign (ref, x, &sub_seq);
3956 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START),
3958 gimple_seq_add_stmt (stmt_seqp, stmt);
3960 gimple_seq_add_seq (stmt_seqp, sub_seq);
3962 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
3964 gimple_seq_add_stmt (stmt_seqp, stmt);
3968 /* Generate code to implement the COPYPRIVATE clauses. */
3971 lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
3976 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3978 tree var, new_var, ref, x;
3980 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3982 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
3985 var = OMP_CLAUSE_DECL (c);
3986 by_ref = use_pointer_for_field (var, NULL);
3988 ref = build_sender_ref (var, ctx);
3989 x = new_var = lookup_decl_in_outer_ctx (var, ctx);
3992 x = build_fold_addr_expr_loc (clause_loc, new_var);
3993 x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
3995 gimplify_assign (ref, x, slist);
3997 ref = build_receiver_ref (var, false, ctx);
4000 ref = fold_convert_loc (clause_loc,
4001 build_pointer_type (TREE_TYPE (new_var)),
4003 ref = build_fold_indirect_ref_loc (clause_loc, ref);
4005 if (is_reference (var))
4007 ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
4008 ref = build_simple_mem_ref_loc (clause_loc, ref);
4009 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4011 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
4012 gimplify_and_add (x, rlist);
4017 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4018 and REDUCTION from the sender (aka parent) side. */
4021 lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
4026 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4028 tree val, ref, x, var;
4029 bool by_ref, do_in = false, do_out = false;
4030 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4032 switch (OMP_CLAUSE_CODE (c))
4034 case OMP_CLAUSE_PRIVATE:
4035 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
4038 case OMP_CLAUSE_FIRSTPRIVATE:
4039 case OMP_CLAUSE_COPYIN:
4040 case OMP_CLAUSE_LASTPRIVATE:
4041 case OMP_CLAUSE_REDUCTION:
4042 case OMP_CLAUSE__LOOPTEMP_:
4048 val = OMP_CLAUSE_DECL (c);
4049 var = lookup_decl_in_outer_ctx (val, ctx);
4051 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
4052 && is_global_var (var))
4054 if (is_variable_sized (val))
4056 by_ref = use_pointer_for_field (val, NULL);
4058 switch (OMP_CLAUSE_CODE (c))
4060 case OMP_CLAUSE_PRIVATE:
4061 case OMP_CLAUSE_FIRSTPRIVATE:
4062 case OMP_CLAUSE_COPYIN:
4063 case OMP_CLAUSE__LOOPTEMP_:
4067 case OMP_CLAUSE_LASTPRIVATE:
4068 if (by_ref || is_reference (val))
4070 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
4077 if (lang_hooks.decls.omp_private_outer_ref (val))
4082 case OMP_CLAUSE_REDUCTION:
4084 do_out = !(by_ref || is_reference (val));
4093 ref = build_sender_ref (val, ctx);
4094 x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
4095 gimplify_assign (ref, x, ilist);
4096 if (is_task_ctx (ctx))
4097 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
4102 ref = build_sender_ref (val, ctx);
4103 gimplify_assign (var, ref, olist);
4108 /* Generate code to implement SHARED from the sender (aka parent)
4109 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4110 list things that got automatically shared. */
4113 lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
4115 tree var, ovar, nvar, f, x, record_type;
4117 if (ctx->record_type == NULL)
4120 record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
4121 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
4123 ovar = DECL_ABSTRACT_ORIGIN (f);
4124 nvar = maybe_lookup_decl (ovar, ctx);
4125 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar))
4128 /* If CTX is a nested parallel directive. Find the immediately
4129 enclosing parallel or workshare construct that contains a
4130 mapping for OVAR. */
4131 var = lookup_decl_in_outer_ctx (ovar, ctx);
4133 if (use_pointer_for_field (ovar, ctx))
4135 x = build_sender_ref (ovar, ctx);
4136 var = build_fold_addr_expr (var);
4137 gimplify_assign (x, var, ilist);
4141 x = build_sender_ref (ovar, ctx);
4142 gimplify_assign (x, var, ilist);
4144 if (!TREE_READONLY (var)
4145 /* We don't need to receive a new reference to a result
4146 or parm decl. In fact we may not store to it as we will
4147 invalidate any pending RSO and generate wrong gimple
4149 && !((TREE_CODE (var) == RESULT_DECL
4150 || TREE_CODE (var) == PARM_DECL)
4151 && DECL_BY_REFERENCE (var)))
4153 x = build_sender_ref (ovar, ctx);
4154 gimplify_assign (var, x, olist);
4161 /* A convenience function to build an empty GIMPLE_COND with just the
4165 gimple_build_cond_empty (tree cond)
4167 enum tree_code pred_code;
4170 gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs);
4171 return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE);
4175 /* Build the function calls to GOMP_parallel_start etc to actually
4176 generate the parallel operation. REGION is the parallel region
4177 being expanded. BB is the block where to insert the code. WS_ARGS
4178 will be set if this is a call to a combined parallel+workshare
4179 construct, it contains the list of additional arguments needed by
4180 the workshare construct. */
4183 expand_parallel_call (struct omp_region *region, basic_block bb,
4184 gimple entry_stmt, vec<tree, va_gc> *ws_args)
4186 tree t, t1, t2, val, cond, c, clauses, flags;
4187 gimple_stmt_iterator gsi;
4189 enum built_in_function start_ix;
4191 location_t clause_loc;
4192 vec<tree, va_gc> *args;
4194 clauses = gimple_omp_parallel_clauses (entry_stmt);
4196 /* Determine what flavor of GOMP_parallel we will be
4198 start_ix = BUILT_IN_GOMP_PARALLEL;
4199 if (is_combined_parallel (region))
4201 switch (region->inner->type)
4203 case GIMPLE_OMP_FOR:
4204 gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
4205 start_ix2 = ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4206 + (region->inner->sched_kind
4207 == OMP_CLAUSE_SCHEDULE_RUNTIME
4208 ? 3 : region->inner->sched_kind));
4209 start_ix = (enum built_in_function)start_ix2;
4211 case GIMPLE_OMP_SECTIONS:
4212 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS;
4219 /* By default, the value of NUM_THREADS is zero (selected at run time)
4220 and there is no conditional. */
4222 val = build_int_cst (unsigned_type_node, 0);
4223 flags = build_int_cst (unsigned_type_node, 0);
4225 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
4227 cond = OMP_CLAUSE_IF_EXPR (c);
4229 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS);
4232 val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
4233 clause_loc = OMP_CLAUSE_LOCATION (c);
4236 clause_loc = gimple_location (entry_stmt);
4238 c = find_omp_clause (clauses, OMP_CLAUSE_PROC_BIND);
4240 flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c));
4242 /* Ensure 'val' is of the correct type. */
4243 val = fold_convert_loc (clause_loc, unsigned_type_node, val);
4245 /* If we found the clause 'if (cond)', build either
4246 (cond != 0) or (cond ? val : 1u). */
4249 cond = gimple_boolify (cond);
4251 if (integer_zerop (val))
4252 val = fold_build2_loc (clause_loc,
4253 EQ_EXPR, unsigned_type_node, cond,
4254 build_int_cst (TREE_TYPE (cond), 0));
4257 basic_block cond_bb, then_bb, else_bb;
4258 edge e, e_then, e_else;
4259 tree tmp_then, tmp_else, tmp_join, tmp_var;
4261 tmp_var = create_tmp_var (TREE_TYPE (val), NULL);
4262 if (gimple_in_ssa_p (cfun))
4264 tmp_then = make_ssa_name (tmp_var, NULL);
4265 tmp_else = make_ssa_name (tmp_var, NULL);
4266 tmp_join = make_ssa_name (tmp_var, NULL);
4275 e = split_block (bb, NULL);
4280 then_bb = create_empty_bb (cond_bb);
4281 else_bb = create_empty_bb (then_bb);
4282 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
4283 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
4285 stmt = gimple_build_cond_empty (cond);
4286 gsi = gsi_start_bb (cond_bb);
4287 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4289 gsi = gsi_start_bb (then_bb);
4290 stmt = gimple_build_assign (tmp_then, val);
4291 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4293 gsi = gsi_start_bb (else_bb);
4294 stmt = gimple_build_assign
4295 (tmp_else, build_int_cst (unsigned_type_node, 1));
4296 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4298 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
4299 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
4300 add_bb_to_loop (then_bb, cond_bb->loop_father);
4301 add_bb_to_loop (else_bb, cond_bb->loop_father);
4302 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
4303 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
4305 if (gimple_in_ssa_p (cfun))
4307 gimple phi = create_phi_node (tmp_join, bb);
4308 add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION);
4309 add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION);
4315 gsi = gsi_start_bb (bb);
4316 val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE,
4317 false, GSI_CONTINUE_LINKING);
4320 gsi = gsi_last_bb (bb);
4321 t = gimple_omp_parallel_data_arg (entry_stmt);
4323 t1 = null_pointer_node;
4325 t1 = build_fold_addr_expr (t);
4326 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
4328 vec_alloc (args, 4 + vec_safe_length (ws_args));
4329 args->quick_push (t2);
4330 args->quick_push (t1);
4331 args->quick_push (val);
4333 args->splice (*ws_args);
4334 args->quick_push (flags);
4336 t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
4337 builtin_decl_explicit (start_ix), args);
4339 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4340 false, GSI_CONTINUE_LINKING);
4344 /* Build the function call to GOMP_task to actually
4345 generate the task operation. BB is the block where to insert the code. */
4348 expand_task_call (basic_block bb, gimple entry_stmt)
4350 tree t, t1, t2, t3, flags, cond, c, c2, clauses, depend;
4351 gimple_stmt_iterator gsi;
4352 location_t loc = gimple_location (entry_stmt);
4354 clauses = gimple_omp_task_clauses (entry_stmt);
4356 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
4358 cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (c));
4360 cond = boolean_true_node;
4362 c = find_omp_clause (clauses, OMP_CLAUSE_UNTIED);
4363 c2 = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE);
4364 depend = find_omp_clause (clauses, OMP_CLAUSE_DEPEND);
4365 flags = build_int_cst (unsigned_type_node,
4366 (c ? 1 : 0) + (c2 ? 4 : 0) + (depend ? 8 : 0));
4368 c = find_omp_clause (clauses, OMP_CLAUSE_FINAL);
4371 c = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c));
4372 c = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, c,
4373 build_int_cst (unsigned_type_node, 2),
4374 build_int_cst (unsigned_type_node, 0));
4375 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, c);
4378 depend = OMP_CLAUSE_DECL (depend);
4380 depend = build_int_cst (ptr_type_node, 0);
4382 gsi = gsi_last_bb (bb);
4383 t = gimple_omp_task_data_arg (entry_stmt);
4385 t2 = null_pointer_node;
4387 t2 = build_fold_addr_expr_loc (loc, t);
4388 t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt));
4389 t = gimple_omp_task_copy_fn (entry_stmt);
4391 t3 = null_pointer_node;
4393 t3 = build_fold_addr_expr_loc (loc, t);
4395 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
4397 gimple_omp_task_arg_size (entry_stmt),
4398 gimple_omp_task_arg_align (entry_stmt), cond, flags,
4401 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4402 false, GSI_CONTINUE_LINKING);
4406 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4407 catch handler and return it. This prevents programs from violating the
4408 structured block semantics with throws. */
4411 maybe_catch_exception (gimple_seq body)
4416 if (!flag_exceptions)
4419 if (lang_hooks.eh_protect_cleanup_actions != NULL)
4420 decl = lang_hooks.eh_protect_cleanup_actions ();
4422 decl = builtin_decl_explicit (BUILT_IN_TRAP);
4424 g = gimple_build_eh_must_not_throw (decl);
4425 g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
4428 return gimple_seq_alloc_with_stmt (g);
4431 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4434 vec2chain (vec<tree, va_gc> *v)
4436 tree chain = NULL_TREE, t;
4439 FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t)
4441 DECL_CHAIN (t) = chain;
4449 /* Remove barriers in REGION->EXIT's block. Note that this is only
4450 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4451 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4452 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4456 remove_exit_barrier (struct omp_region *region)
4458 gimple_stmt_iterator gsi;
4459 basic_block exit_bb;
4463 int any_addressable_vars = -1;
4465 exit_bb = region->exit;
4467 /* If the parallel region doesn't return, we don't have REGION->EXIT
4472 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4473 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4474 statements that can appear in between are extremely limited -- no
4475 memory operations at all. Here, we allow nothing at all, so the
4476 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4477 gsi = gsi_last_bb (exit_bb);
4478 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
4480 if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
4483 FOR_EACH_EDGE (e, ei, exit_bb->preds)
4485 gsi = gsi_last_bb (e->src);
4486 if (gsi_end_p (gsi))
4488 stmt = gsi_stmt (gsi);
4489 if (gimple_code (stmt) == GIMPLE_OMP_RETURN
4490 && !gimple_omp_return_nowait_p (stmt))
4492 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4493 in many cases. If there could be tasks queued, the barrier
4494 might be needed to let the tasks run before some local
4495 variable of the parallel that the task uses as shared
4496 runs out of scope. The task can be spawned either
4497 from within current function (this would be easy to check)
4498 or from some function it calls and gets passed an address
4499 of such a variable. */
4500 if (any_addressable_vars < 0)
4502 gimple parallel_stmt = last_stmt (region->entry);
4503 tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt);
4504 tree local_decls, block, decl;
4507 any_addressable_vars = 0;
4508 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl)
4509 if (TREE_ADDRESSABLE (decl))
4511 any_addressable_vars = 1;
4514 for (block = gimple_block (stmt);
4515 !any_addressable_vars
4517 && TREE_CODE (block) == BLOCK;
4518 block = BLOCK_SUPERCONTEXT (block))
4520 for (local_decls = BLOCK_VARS (block);
4522 local_decls = DECL_CHAIN (local_decls))
4523 if (TREE_ADDRESSABLE (local_decls))
4525 any_addressable_vars = 1;
4528 if (block == gimple_block (parallel_stmt))
4532 if (!any_addressable_vars)
4533 gimple_omp_return_set_nowait (stmt);
4539 remove_exit_barriers (struct omp_region *region)
4541 if (region->type == GIMPLE_OMP_PARALLEL)
4542 remove_exit_barrier (region);
4546 region = region->inner;
4547 remove_exit_barriers (region);
4548 while (region->next)
4550 region = region->next;
4551 remove_exit_barriers (region);
4556 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4557 calls. These can't be declared as const functions, but
4558 within one parallel body they are constant, so they can be
4559 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4560 which are declared const. Similarly for task body, except
4561 that in untied task omp_get_thread_num () can change at any task
4562 scheduling point. */
4565 optimize_omp_library_calls (gimple entry_stmt)
4568 gimple_stmt_iterator gsi;
4569 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
4570 tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree);
4571 tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
4572 tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree);
4573 bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
4574 && find_omp_clause (gimple_omp_task_clauses (entry_stmt),
4575 OMP_CLAUSE_UNTIED) != NULL);
4577 FOR_EACH_BB_FN (bb, cfun)
4578 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4580 gimple call = gsi_stmt (gsi);
4583 if (is_gimple_call (call)
4584 && (decl = gimple_call_fndecl (call))
4585 && DECL_EXTERNAL (decl)
4586 && TREE_PUBLIC (decl)
4587 && DECL_INITIAL (decl) == NULL)
4591 if (DECL_NAME (decl) == thr_num_id)
4593 /* In #pragma omp task untied omp_get_thread_num () can change
4594 during the execution of the task region. */
4597 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
4599 else if (DECL_NAME (decl) == num_thr_id)
4600 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
4604 if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in)
4605 || gimple_call_num_args (call) != 0)
4608 if (flag_exceptions && !TREE_NOTHROW (decl))
4611 if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE
4612 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)),
4613 TREE_TYPE (TREE_TYPE (built_in))))
4616 gimple_call_set_fndecl (call, built_in);
4621 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4625 expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *)
4629 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4630 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t))
4633 if (TREE_CODE (t) == ADDR_EXPR)
4634 recompute_tree_invariant_for_addr_expr (t);
4636 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
4640 /* Prepend TO = FROM assignment before *GSI_P. */
4643 expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from)
4645 bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to);
4646 from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE,
4647 true, GSI_SAME_STMT);
4648 gimple stmt = gimple_build_assign (to, from);
4649 gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT);
4650 if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL)
4651 || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL))
4653 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
4654 gimple_regimplify_operands (stmt, &gsi);
4658 /* Expand the OpenMP parallel or task directive starting at REGION. */
4661 expand_omp_taskreg (struct omp_region *region)
4663 basic_block entry_bb, exit_bb, new_bb;
4664 struct function *child_cfun;
4665 tree child_fn, block, t;
4666 gimple_stmt_iterator gsi;
4667 gimple entry_stmt, stmt;
4669 vec<tree, va_gc> *ws_args;
4671 entry_stmt = last_stmt (region->entry);
4672 child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
4673 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
4675 entry_bb = region->entry;
4676 exit_bb = region->exit;
4678 if (is_combined_parallel (region))
4679 ws_args = region->ws_args;
4683 if (child_cfun->cfg)
4685 /* Due to inlining, it may happen that we have already outlined
4686 the region, in which case all we need to do is make the
4687 sub-graph unreachable and emit the parallel call. */
4688 edge entry_succ_e, exit_succ_e;
4690 entry_succ_e = single_succ_edge (entry_bb);
4692 gsi = gsi_last_bb (entry_bb);
4693 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
4694 || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
4695 gsi_remove (&gsi, true);
4700 exit_succ_e = single_succ_edge (exit_bb);
4701 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
4703 remove_edge_and_dominated_blocks (entry_succ_e);
4707 unsigned srcidx, dstidx, num;
4709 /* If the parallel region needs data sent from the parent
4710 function, then the very first statement (except possible
4711 tree profile counter updates) of the parallel body
4712 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4713 &.OMP_DATA_O is passed as an argument to the child function,
4714 we need to replace it with the argument as seen by the child
4717 In most cases, this will end up being the identity assignment
4718 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4719 a function call that has been inlined, the original PARM_DECL
4720 .OMP_DATA_I may have been converted into a different local
4721 variable. In which case, we need to keep the assignment. */
4722 if (gimple_omp_taskreg_data_arg (entry_stmt))
4724 basic_block entry_succ_bb = single_succ (entry_bb);
4726 gimple parcopy_stmt = NULL;
4728 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
4732 gcc_assert (!gsi_end_p (gsi));
4733 stmt = gsi_stmt (gsi);
4734 if (gimple_code (stmt) != GIMPLE_ASSIGN)
4737 if (gimple_num_ops (stmt) == 2)
4739 tree arg = gimple_assign_rhs1 (stmt);
4741 /* We're ignore the subcode because we're
4742 effectively doing a STRIP_NOPS. */
4744 if (TREE_CODE (arg) == ADDR_EXPR
4745 && TREE_OPERAND (arg, 0)
4746 == gimple_omp_taskreg_data_arg (entry_stmt))
4748 parcopy_stmt = stmt;
4754 gcc_assert (parcopy_stmt != NULL);
4755 arg = DECL_ARGUMENTS (child_fn);
4757 if (!gimple_in_ssa_p (cfun))
4759 if (gimple_assign_lhs (parcopy_stmt) == arg)
4760 gsi_remove (&gsi, true);
4763 /* ?? Is setting the subcode really necessary ?? */
4764 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg));
4765 gimple_assign_set_rhs1 (parcopy_stmt, arg);
4770 /* If we are in ssa form, we must load the value from the default
4771 definition of the argument. That should not be defined now,
4772 since the argument is not used uninitialized. */
4773 gcc_assert (ssa_default_def (cfun, arg) == NULL);
4774 narg = make_ssa_name (arg, gimple_build_nop ());
4775 set_ssa_default_def (cfun, arg, narg);
4776 /* ?? Is setting the subcode really necessary ?? */
4777 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (narg));
4778 gimple_assign_set_rhs1 (parcopy_stmt, narg);
4779 update_stmt (parcopy_stmt);
4783 /* Declare local variables needed in CHILD_CFUN. */
4784 block = DECL_INITIAL (child_fn);
4785 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
4786 /* The gimplifier could record temporaries in parallel/task block
4787 rather than in containing function's local_decls chain,
4788 which would mean cgraph missed finalizing them. Do it now. */
4789 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
4790 if (TREE_CODE (t) == VAR_DECL
4792 && !DECL_EXTERNAL (t))
4793 varpool_finalize_decl (t);
4794 DECL_SAVED_TREE (child_fn) = NULL;
4795 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4796 gimple_set_body (child_fn, NULL);
4797 TREE_USED (block) = 1;
4799 /* Reset DECL_CONTEXT on function arguments. */
4800 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
4801 DECL_CONTEXT (t) = child_fn;
4803 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4804 so that it can be moved to the child function. */
4805 gsi = gsi_last_bb (entry_bb);
4806 stmt = gsi_stmt (gsi);
4807 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
4808 || gimple_code (stmt) == GIMPLE_OMP_TASK));
4809 gsi_remove (&gsi, true);
4810 e = split_block (entry_bb, stmt);
4812 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
4814 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4817 gsi = gsi_last_bb (exit_bb);
4818 gcc_assert (!gsi_end_p (gsi)
4819 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
4820 stmt = gimple_build_return (NULL);
4821 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
4822 gsi_remove (&gsi, true);
4825 /* Move the parallel region into CHILD_CFUN. */
4827 if (gimple_in_ssa_p (cfun))
4829 init_tree_ssa (child_cfun);
4830 init_ssa_operands (child_cfun);
4831 child_cfun->gimple_df->in_ssa_p = true;
4835 block = gimple_block (entry_stmt);
4837 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
4839 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
4840 /* When the OMP expansion process cannot guarantee an up-to-date
4841 loop tree arrange for the child function to fixup loops. */
4842 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
4843 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
4845 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4846 num = vec_safe_length (child_cfun->local_decls);
4847 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
4849 t = (*child_cfun->local_decls)[srcidx];
4850 if (DECL_CONTEXT (t) == cfun->decl)
4852 if (srcidx != dstidx)
4853 (*child_cfun->local_decls)[dstidx] = t;
4857 vec_safe_truncate (child_cfun->local_decls, dstidx);
4859 /* Inform the callgraph about the new function. */
4860 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
4861 cgraph_add_new_function (child_fn, true);
4863 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4864 fixed in a following pass. */
4865 push_cfun (child_cfun);
4867 optimize_omp_library_calls (entry_stmt);
4868 rebuild_cgraph_edges ();
4870 /* Some EH regions might become dead, see PR34608. If
4871 pass_cleanup_cfg isn't the first pass to happen with the
4872 new child, these dead EH edges might cause problems.
4873 Clean them up now. */
4874 if (flag_exceptions)
4877 bool changed = false;
4879 FOR_EACH_BB_FN (bb, cfun)
4880 changed |= gimple_purge_dead_eh_edges (bb);
4882 cleanup_tree_cfg ();
4884 if (gimple_in_ssa_p (cfun))
4885 update_ssa (TODO_update_ssa);
4889 /* Emit a library call to launch the children threads. */
4890 if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
4891 expand_parallel_call (region, new_bb, entry_stmt, ws_args);
4893 expand_task_call (new_bb, entry_stmt);
4894 if (gimple_in_ssa_p (cfun))
4895 update_ssa (TODO_update_ssa_only_virtuals);
4899 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4900 of the combined collapse > 1 loop constructs, generate code like:
4901 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4906 count3 = (adj + N32 - N31) / STEP3;
4907 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4912 count2 = (adj + N22 - N21) / STEP2;
4913 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4918 count1 = (adj + N12 - N11) / STEP1;
4919 count = count1 * count2 * count3;
4920 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4922 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4923 of the combined loop constructs, just initialize COUNTS array
4924 from the _looptemp_ clauses. */
4926 /* NOTE: It *could* be better to moosh all of the BBs together,
4927 creating one larger BB with all the computation and the unexpected
4928 jump at the end. I.e.
4930 bool zero3, zero2, zero1, zero;
4933 count3 = (N32 - N31) /[cl] STEP3;
4935 count2 = (N22 - N21) /[cl] STEP2;
4937 count1 = (N12 - N11) /[cl] STEP1;
4938 zero = zero3 || zero2 || zero1;
4939 count = count1 * count2 * count3;
4940 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4942 After all, we expect the zero=false, and thus we expect to have to
4943 evaluate all of the comparison expressions, so short-circuiting
4944 oughtn't be a win. Since the condition isn't protecting a
4945 denominator, we're not concerned about divide-by-zero, so we can
4946 fully evaluate count even if a numerator turned out to be wrong.
4948 It seems like putting this all together would create much better
4949 scheduling opportunities, and less pressure on the chip's branch
4953 expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
4954 basic_block &entry_bb, tree *counts,
4955 basic_block &zero_iter_bb, int &first_zero_iter,
4956 basic_block &l2_dom_bb)
4958 tree t, type = TREE_TYPE (fd->loop.v);
4963 /* Collapsed loops need work for expansion into SSA form. */
4964 gcc_assert (!gimple_in_ssa_p (cfun));
4966 if (gimple_omp_for_combined_into_p (fd->for_stmt)
4967 && TREE_CODE (fd->loop.n2) != INTEGER_CST)
4969 /* First two _looptemp_ clauses are for istart/iend, counts[0]
4970 isn't supposed to be handled, as the inner loop doesn't
4972 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
4973 OMP_CLAUSE__LOOPTEMP_);
4974 gcc_assert (innerc);
4975 for (i = 0; i < fd->collapse; i++)
4977 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
4978 OMP_CLAUSE__LOOPTEMP_);
4979 gcc_assert (innerc);
4981 counts[i] = OMP_CLAUSE_DECL (innerc);
4983 counts[0] = NULL_TREE;
4988 for (i = 0; i < fd->collapse; i++)
4990 tree itype = TREE_TYPE (fd->loops[i].v);
4992 if (SSA_VAR_P (fd->loop.n2)
4993 && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
4994 fold_convert (itype, fd->loops[i].n1),
4995 fold_convert (itype, fd->loops[i].n2)))
4996 == NULL_TREE || !integer_onep (t)))
4999 n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1));
5000 n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE,
5001 true, GSI_SAME_STMT);
5002 n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
5003 n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE,
5004 true, GSI_SAME_STMT);
5005 stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
5006 NULL_TREE, NULL_TREE);
5007 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
5008 if (walk_tree (gimple_cond_lhs_ptr (stmt),
5009 expand_omp_regimplify_p, NULL, NULL)
5010 || walk_tree (gimple_cond_rhs_ptr (stmt),
5011 expand_omp_regimplify_p, NULL, NULL))
5013 *gsi = gsi_for_stmt (stmt);
5014 gimple_regimplify_operands (stmt, gsi);
5016 e = split_block (entry_bb, stmt);
5017 if (zero_iter_bb == NULL)
5019 first_zero_iter = i;
5020 zero_iter_bb = create_empty_bb (entry_bb);
5021 add_bb_to_loop (zero_iter_bb, entry_bb->loop_father);
5022 *gsi = gsi_after_labels (zero_iter_bb);
5023 stmt = gimple_build_assign (fd->loop.n2,
5024 build_zero_cst (type));
5025 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
5026 set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb,
5029 ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
5030 ne->probability = REG_BR_PROB_BASE / 2000 - 1;
5031 e->flags = EDGE_TRUE_VALUE;
5032 e->probability = REG_BR_PROB_BASE - ne->probability;
5033 if (l2_dom_bb == NULL)
5034 l2_dom_bb = entry_bb;
5036 *gsi = gsi_last_bb (entry_bb);
5039 if (POINTER_TYPE_P (itype))
5040 itype = signed_type_for (itype);
5041 t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
5043 t = fold_build2 (PLUS_EXPR, itype,
5044 fold_convert (itype, fd->loops[i].step), t);
5045 t = fold_build2 (PLUS_EXPR, itype, t,
5046 fold_convert (itype, fd->loops[i].n2));
5047 t = fold_build2 (MINUS_EXPR, itype, t,
5048 fold_convert (itype, fd->loops[i].n1));
5049 /* ?? We could probably use CEIL_DIV_EXPR instead of
5050 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5051 generate the same code in the end because generically we
5052 don't know that the values involved must be negative for
5054 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
5055 t = fold_build2 (TRUNC_DIV_EXPR, itype,
5056 fold_build1 (NEGATE_EXPR, itype, t),
5057 fold_build1 (NEGATE_EXPR, itype,
5058 fold_convert (itype,
5059 fd->loops[i].step)));
5061 t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
5062 fold_convert (itype, fd->loops[i].step));
5063 t = fold_convert (type, t);
5064 if (TREE_CODE (t) == INTEGER_CST)
5068 counts[i] = create_tmp_reg (type, ".count");
5069 expand_omp_build_assign (gsi, counts[i], t);
5071 if (SSA_VAR_P (fd->loop.n2))
5076 t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]);
5077 expand_omp_build_assign (gsi, fd->loop.n2, t);
5083 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5085 V3 = N31 + (T % count3) * STEP3;
5087 V2 = N21 + (T % count2) * STEP2;
5089 V1 = N11 + T * STEP1;
5090 if this loop doesn't have an inner loop construct combined with it.
5091 If it does have an inner loop construct combined with it and the
5092 iteration count isn't known constant, store values from counts array
5093 into its _looptemp_ temporaries instead. */
5096 expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
5097 tree *counts, gimple inner_stmt, tree startvar)
5100 if (gimple_omp_for_combined_p (fd->for_stmt))
5102 /* If fd->loop.n2 is constant, then no propagation of the counts
5103 is needed, they are constant. */
5104 if (TREE_CODE (fd->loop.n2) == INTEGER_CST)
5107 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
5108 ? gimple_omp_parallel_clauses (inner_stmt)
5109 : gimple_omp_for_clauses (inner_stmt);
5110 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5111 isn't supposed to be handled, as the inner loop doesn't
5113 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
5114 gcc_assert (innerc);
5115 for (i = 0; i < fd->collapse; i++)
5117 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5118 OMP_CLAUSE__LOOPTEMP_);
5119 gcc_assert (innerc);
5122 tree tem = OMP_CLAUSE_DECL (innerc);
5123 tree t = fold_convert (TREE_TYPE (tem), counts[i]);
5124 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
5125 false, GSI_CONTINUE_LINKING);
5126 gimple stmt = gimple_build_assign (tem, t);
5127 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5133 tree type = TREE_TYPE (fd->loop.v);
5134 tree tem = create_tmp_reg (type, ".tem");
5135 gimple stmt = gimple_build_assign (tem, startvar);
5136 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5138 for (i = fd->collapse - 1; i >= 0; i--)
5140 tree vtype = TREE_TYPE (fd->loops[i].v), itype, t;
5142 if (POINTER_TYPE_P (vtype))
5143 itype = signed_type_for (vtype);
5145 t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]);
5148 t = fold_convert (itype, t);
5149 t = fold_build2 (MULT_EXPR, itype, t,
5150 fold_convert (itype, fd->loops[i].step));
5151 if (POINTER_TYPE_P (vtype))
5152 t = fold_build_pointer_plus (fd->loops[i].n1, t);
5154 t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
5155 t = force_gimple_operand_gsi (gsi, t,
5156 DECL_P (fd->loops[i].v)
5157 && TREE_ADDRESSABLE (fd->loops[i].v),
5159 GSI_CONTINUE_LINKING);
5160 stmt = gimple_build_assign (fd->loops[i].v, t);
5161 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5164 t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]);
5165 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
5166 false, GSI_CONTINUE_LINKING);
5167 stmt = gimple_build_assign (tem, t);
5168 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5174 /* Helper function for expand_omp_for_*. Generate code like:
5177 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5181 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5188 extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb,
5189 basic_block body_bb)
5191 basic_block last_bb, bb, collapse_bb = NULL;
5193 gimple_stmt_iterator gsi;
5199 for (i = fd->collapse - 1; i >= 0; i--)
5201 tree vtype = TREE_TYPE (fd->loops[i].v);
5203 bb = create_empty_bb (last_bb);
5204 add_bb_to_loop (bb, last_bb->loop_father);
5205 gsi = gsi_start_bb (bb);
5207 if (i < fd->collapse - 1)
5209 e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
5210 e->probability = REG_BR_PROB_BASE / 8;
5212 t = fd->loops[i + 1].n1;
5213 t = force_gimple_operand_gsi (&gsi, t,
5214 DECL_P (fd->loops[i + 1].v)
5215 && TREE_ADDRESSABLE (fd->loops[i
5218 GSI_CONTINUE_LINKING);
5219 stmt = gimple_build_assign (fd->loops[i + 1].v, t);
5220 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5225 set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
5227 if (POINTER_TYPE_P (vtype))
5228 t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
5230 t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step);
5231 t = force_gimple_operand_gsi (&gsi, t,
5232 DECL_P (fd->loops[i].v)
5233 && TREE_ADDRESSABLE (fd->loops[i].v),
5234 NULL_TREE, false, GSI_CONTINUE_LINKING);
5235 stmt = gimple_build_assign (fd->loops[i].v, t);
5236 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5240 t = fd->loops[i].n2;
5241 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5242 false, GSI_CONTINUE_LINKING);
5243 tree v = fd->loops[i].v;
5244 if (DECL_P (v) && TREE_ADDRESSABLE (v))
5245 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
5246 false, GSI_CONTINUE_LINKING);
5247 t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
5248 stmt = gimple_build_cond_empty (t);
5249 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5250 e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
5251 e->probability = REG_BR_PROB_BASE * 7 / 8;
5254 make_edge (bb, body_bb, EDGE_FALLTHRU);
5262 /* A subroutine of expand_omp_for. Generate code for a parallel
5263 loop with any schedule. Given parameters:
5265 for (V = N1; V cond N2; V += STEP) BODY;
5267 where COND is "<" or ">", we generate pseudocode
5269 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5270 if (more) goto L0; else goto L3;
5277 if (V cond iend) goto L1; else goto L2;
5279 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5282 If this is a combined omp parallel loop, instead of the call to
5283 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5284 If this is gimple_omp_for_combined_p loop, then instead of assigning
5285 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5286 inner GIMPLE_OMP_FOR and V += STEP; and
5287 if (V cond iend) goto L1; else goto L2; are removed.
5289 For collapsed loops, given parameters:
5291 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5292 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5293 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5296 we generate pseudocode
5298 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5303 count3 = (adj + N32 - N31) / STEP3;
5304 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5309 count2 = (adj + N22 - N21) / STEP2;
5310 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5315 count1 = (adj + N12 - N11) / STEP1;
5316 count = count1 * count2 * count3;
5321 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5322 if (more) goto L0; else goto L3;
5326 V3 = N31 + (T % count3) * STEP3;
5328 V2 = N21 + (T % count2) * STEP2;
5330 V1 = N11 + T * STEP1;
5335 if (V < iend) goto L10; else goto L2;
5338 if (V3 cond3 N32) goto L1; else goto L11;
5342 if (V2 cond2 N22) goto L1; else goto L12;
5348 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5354 expand_omp_for_generic (struct omp_region *region,
5355 struct omp_for_data *fd,
5356 enum built_in_function start_fn,
5357 enum built_in_function next_fn,
5360 tree type, istart0, iend0, iend;
5361 tree t, vmain, vback, bias = NULL_TREE;
5362 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
5363 basic_block l2_bb = NULL, l3_bb = NULL;
5364 gimple_stmt_iterator gsi;
5366 bool in_combined_parallel = is_combined_parallel (region);
5367 bool broken_loop = region->cont == NULL;
5369 tree *counts = NULL;
5372 gcc_assert (!broken_loop || !in_combined_parallel);
5373 gcc_assert (fd->iter_type == long_integer_type_node
5374 || !in_combined_parallel);
5376 type = TREE_TYPE (fd->loop.v);
5377 istart0 = create_tmp_var (fd->iter_type, ".istart0");
5378 iend0 = create_tmp_var (fd->iter_type, ".iend0");
5379 TREE_ADDRESSABLE (istart0) = 1;
5380 TREE_ADDRESSABLE (iend0) = 1;
5382 /* See if we need to bias by LLONG_MIN. */
5383 if (fd->iter_type == long_long_unsigned_type_node
5384 && TREE_CODE (type) == INTEGER_TYPE
5385 && !TYPE_UNSIGNED (type))
5389 if (fd->loop.cond_code == LT_EXPR)
5392 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
5396 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
5399 if (TREE_CODE (n1) != INTEGER_CST
5400 || TREE_CODE (n2) != INTEGER_CST
5401 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
5402 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
5405 entry_bb = region->entry;
5406 cont_bb = region->cont;
5408 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
5409 gcc_assert (broken_loop
5410 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
5411 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
5412 l1_bb = single_succ (l0_bb);
5415 l2_bb = create_empty_bb (cont_bb);
5416 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb);
5417 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
5421 l3_bb = BRANCH_EDGE (entry_bb)->dest;
5422 exit_bb = region->exit;
5424 gsi = gsi_last_bb (entry_bb);
5426 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5427 if (fd->collapse > 1)
5429 int first_zero_iter = -1;
5430 basic_block zero_iter_bb = NULL, l2_dom_bb = NULL;
5432 counts = XALLOCAVEC (tree, fd->collapse);
5433 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
5434 zero_iter_bb, first_zero_iter,
5439 /* Some counts[i] vars might be uninitialized if
5440 some loop has zero iterations. But the body shouldn't
5441 be executed in that case, so just avoid uninit warnings. */
5442 for (i = first_zero_iter; i < fd->collapse; i++)
5443 if (SSA_VAR_P (counts[i]))
5444 TREE_NO_WARNING (counts[i]) = 1;
5446 e = split_block (entry_bb, gsi_stmt (gsi));
5448 make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU);
5449 gsi = gsi_last_bb (entry_bb);
5450 set_immediate_dominator (CDI_DOMINATORS, entry_bb,
5451 get_immediate_dominator (CDI_DOMINATORS,
5455 if (in_combined_parallel)
5457 /* In a combined parallel loop, emit a call to
5458 GOMP_loop_foo_next. */
5459 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
5460 build_fold_addr_expr (istart0),
5461 build_fold_addr_expr (iend0));
5465 tree t0, t1, t2, t3, t4;
5466 /* If this is not a combined parallel loop, emit a call to
5467 GOMP_loop_foo_start in ENTRY_BB. */
5468 t4 = build_fold_addr_expr (iend0);
5469 t3 = build_fold_addr_expr (istart0);
5470 t2 = fold_convert (fd->iter_type, fd->loop.step);
5473 if (gimple_omp_for_combined_into_p (fd->for_stmt))
5475 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
5476 OMP_CLAUSE__LOOPTEMP_);
5477 gcc_assert (innerc);
5478 t0 = OMP_CLAUSE_DECL (innerc);
5479 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5480 OMP_CLAUSE__LOOPTEMP_);
5481 gcc_assert (innerc);
5482 t1 = OMP_CLAUSE_DECL (innerc);
5484 if (POINTER_TYPE_P (TREE_TYPE (t0))
5485 && TYPE_PRECISION (TREE_TYPE (t0))
5486 != TYPE_PRECISION (fd->iter_type))
5488 /* Avoid casting pointers to integer of a different size. */
5489 tree itype = signed_type_for (type);
5490 t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
5491 t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
5495 t1 = fold_convert (fd->iter_type, t1);
5496 t0 = fold_convert (fd->iter_type, t0);
5500 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
5501 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
5503 if (fd->iter_type == long_integer_type_node)
5507 t = fold_convert (fd->iter_type, fd->chunk_size);
5508 t = build_call_expr (builtin_decl_explicit (start_fn),
5509 6, t0, t1, t2, t, t3, t4);
5512 t = build_call_expr (builtin_decl_explicit (start_fn),
5513 5, t0, t1, t2, t3, t4);
5521 /* The GOMP_loop_ull_*start functions have additional boolean
5522 argument, true for < loops and false for > loops.
5523 In Fortran, the C bool type can be different from
5524 boolean_type_node. */
5525 bfn_decl = builtin_decl_explicit (start_fn);
5526 c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl));
5527 t5 = build_int_cst (c_bool_type,
5528 fd->loop.cond_code == LT_EXPR ? 1 : 0);
5531 tree bfn_decl = builtin_decl_explicit (start_fn);
5532 t = fold_convert (fd->iter_type, fd->chunk_size);
5533 t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
5536 t = build_call_expr (builtin_decl_explicit (start_fn),
5537 6, t5, t0, t1, t2, t3, t4);
5540 if (TREE_TYPE (t) != boolean_type_node)
5541 t = fold_build2 (NE_EXPR, boolean_type_node,
5542 t, build_int_cst (TREE_TYPE (t), 0));
5543 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5544 true, GSI_SAME_STMT);
5545 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5547 /* Remove the GIMPLE_OMP_FOR statement. */
5548 gsi_remove (&gsi, true);
5550 /* Iteration setup for sequential loop goes in L0_BB. */
5551 tree startvar = fd->loop.v;
5552 tree endvar = NULL_TREE;
5554 if (gimple_omp_for_combined_p (fd->for_stmt))
5556 gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR
5557 && gimple_omp_for_kind (inner_stmt)
5558 == GF_OMP_FOR_KIND_SIMD);
5559 tree innerc = find_omp_clause (gimple_omp_for_clauses (inner_stmt),
5560 OMP_CLAUSE__LOOPTEMP_);
5561 gcc_assert (innerc);
5562 startvar = OMP_CLAUSE_DECL (innerc);
5563 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5564 OMP_CLAUSE__LOOPTEMP_);
5565 gcc_assert (innerc);
5566 endvar = OMP_CLAUSE_DECL (innerc);
5569 gsi = gsi_start_bb (l0_bb);
5572 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
5573 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
5574 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
5575 t = fold_convert (TREE_TYPE (startvar), t);
5576 t = force_gimple_operand_gsi (&gsi, t,
5578 && TREE_ADDRESSABLE (startvar),
5579 NULL_TREE, false, GSI_CONTINUE_LINKING);
5580 stmt = gimple_build_assign (startvar, t);
5581 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5585 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
5586 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
5587 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
5588 t = fold_convert (TREE_TYPE (startvar), t);
5589 iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5590 false, GSI_CONTINUE_LINKING);
5593 stmt = gimple_build_assign (endvar, iend);
5594 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5595 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (iend)))
5596 stmt = gimple_build_assign (fd->loop.v, iend);
5598 stmt = gimple_build_assign_with_ops (NOP_EXPR, fd->loop.v, iend,
5600 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5602 if (fd->collapse > 1)
5603 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
5607 /* Code to control the increment and predicate for the sequential
5608 loop goes in the CONT_BB. */
5609 gsi = gsi_last_bb (cont_bb);
5610 stmt = gsi_stmt (gsi);
5611 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
5612 vmain = gimple_omp_continue_control_use (stmt);
5613 vback = gimple_omp_continue_control_def (stmt);
5615 if (!gimple_omp_for_combined_p (fd->for_stmt))
5617 if (POINTER_TYPE_P (type))
5618 t = fold_build_pointer_plus (vmain, fd->loop.step);
5620 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
5621 t = force_gimple_operand_gsi (&gsi, t,
5623 && TREE_ADDRESSABLE (vback),
5624 NULL_TREE, true, GSI_SAME_STMT);
5625 stmt = gimple_build_assign (vback, t);
5626 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5628 t = build2 (fd->loop.cond_code, boolean_type_node,
5629 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
5631 stmt = gimple_build_cond_empty (t);
5632 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5635 /* Remove GIMPLE_OMP_CONTINUE. */
5636 gsi_remove (&gsi, true);
5638 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
5639 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, l1_bb);
5641 /* Emit code to get the next parallel iteration in L2_BB. */
5642 gsi = gsi_start_bb (l2_bb);
5644 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
5645 build_fold_addr_expr (istart0),
5646 build_fold_addr_expr (iend0));
5647 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5648 false, GSI_CONTINUE_LINKING);
5649 if (TREE_TYPE (t) != boolean_type_node)
5650 t = fold_build2 (NE_EXPR, boolean_type_node,
5651 t, build_int_cst (TREE_TYPE (t), 0));
5652 stmt = gimple_build_cond_empty (t);
5653 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5656 /* Add the loop cleanup function. */
5657 gsi = gsi_last_bb (exit_bb);
5658 if (gimple_omp_return_nowait_p (gsi_stmt (gsi)))
5659 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
5660 else if (gimple_omp_return_lhs (gsi_stmt (gsi)))
5661 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
5663 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
5664 stmt = gimple_build_call (t, 0);
5665 if (gimple_omp_return_lhs (gsi_stmt (gsi)))
5666 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
5667 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
5668 gsi_remove (&gsi, true);
5670 /* Connect the new blocks. */
5671 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
5672 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
5678 e = find_edge (cont_bb, l3_bb);
5679 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
5681 phis = phi_nodes (l3_bb);
5682 for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi))
5684 gimple phi = gsi_stmt (gsi);
5685 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
5686 PHI_ARG_DEF_FROM_EDGE (phi, e));
5690 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
5691 add_bb_to_loop (l2_bb, cont_bb->loop_father);
5692 e = find_edge (cont_bb, l1_bb);
5693 if (gimple_omp_for_combined_p (fd->for_stmt))
5698 else if (fd->collapse > 1)
5701 e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
5704 e->flags = EDGE_TRUE_VALUE;
5707 e->probability = REG_BR_PROB_BASE * 7 / 8;
5708 find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8;
5712 e = find_edge (cont_bb, l2_bb);
5713 e->flags = EDGE_FALLTHRU;
5715 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
5717 set_immediate_dominator (CDI_DOMINATORS, l2_bb,
5718 recompute_dominator (CDI_DOMINATORS, l2_bb));
5719 set_immediate_dominator (CDI_DOMINATORS, l3_bb,
5720 recompute_dominator (CDI_DOMINATORS, l3_bb));
5721 set_immediate_dominator (CDI_DOMINATORS, l0_bb,
5722 recompute_dominator (CDI_DOMINATORS, l0_bb));
5723 set_immediate_dominator (CDI_DOMINATORS, l1_bb,
5724 recompute_dominator (CDI_DOMINATORS, l1_bb));
5726 struct loop *outer_loop = alloc_loop ();
5727 outer_loop->header = l0_bb;
5728 outer_loop->latch = l2_bb;
5729 add_loop (outer_loop, l0_bb->loop_father);
5731 if (!gimple_omp_for_combined_p (fd->for_stmt))
5733 struct loop *loop = alloc_loop ();
5734 loop->header = l1_bb;
5735 /* The loop may have multiple latches. */
5736 add_loop (loop, outer_loop);
5742 /* A subroutine of expand_omp_for. Generate code for a parallel
5743 loop with static schedule and no specified chunk size. Given
5746 for (V = N1; V cond N2; V += STEP) BODY;
5748 where COND is "<" or ">", we generate pseudocode
5750 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5755 if ((__typeof (V)) -1 > 0 && cond is >)
5756 n = -(adj + N2 - N1) / -STEP;
5758 n = (adj + N2 - N1) / STEP;
5761 if (threadid < tt) goto L3; else goto L4;
5766 s0 = q * threadid + tt;
5769 if (s0 >= e0) goto L2; else goto L0;
5775 if (V cond e) goto L1;
5780 expand_omp_for_static_nochunk (struct omp_region *region,
5781 struct omp_for_data *fd,
5784 tree n, q, s0, e0, e, t, tt, nthreads, threadid;
5785 tree type, itype, vmain, vback;
5786 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
5787 basic_block body_bb, cont_bb, collapse_bb = NULL;
5789 gimple_stmt_iterator gsi;
5792 enum built_in_function get_num_threads = BUILT_IN_OMP_GET_NUM_THREADS;
5793 enum built_in_function get_thread_num = BUILT_IN_OMP_GET_THREAD_NUM;
5794 bool broken_loop = region->cont == NULL;
5795 tree *counts = NULL;
5798 itype = type = TREE_TYPE (fd->loop.v);
5799 if (POINTER_TYPE_P (type))
5800 itype = signed_type_for (type);
5802 entry_bb = region->entry;
5803 cont_bb = region->cont;
5804 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
5805 fin_bb = BRANCH_EDGE (entry_bb)->dest;
5806 gcc_assert (broken_loop
5807 || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
5808 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
5809 body_bb = single_succ (seq_start_bb);
5812 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
5813 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
5815 exit_bb = region->exit;
5817 /* Iteration space partitioning goes in ENTRY_BB. */
5818 gsi = gsi_last_bb (entry_bb);
5819 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5821 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
5823 get_num_threads = BUILT_IN_OMP_GET_NUM_TEAMS;
5824 get_thread_num = BUILT_IN_OMP_GET_TEAM_NUM;
5827 if (fd->collapse > 1)
5829 int first_zero_iter = -1;
5830 basic_block l2_dom_bb = NULL;
5832 counts = XALLOCAVEC (tree, fd->collapse);
5833 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
5834 fin_bb, first_zero_iter,
5838 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
5839 t = integer_one_node;
5841 t = fold_binary (fd->loop.cond_code, boolean_type_node,
5842 fold_convert (type, fd->loop.n1),
5843 fold_convert (type, fd->loop.n2));
5844 if (fd->collapse == 1
5845 && TYPE_UNSIGNED (type)
5846 && (t == NULL_TREE || !integer_onep (t)))
5848 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
5849 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
5850 true, GSI_SAME_STMT);
5851 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
5852 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
5853 true, GSI_SAME_STMT);
5854 stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
5855 NULL_TREE, NULL_TREE);
5856 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5857 if (walk_tree (gimple_cond_lhs_ptr (stmt),
5858 expand_omp_regimplify_p, NULL, NULL)
5859 || walk_tree (gimple_cond_rhs_ptr (stmt),
5860 expand_omp_regimplify_p, NULL, NULL))
5862 gsi = gsi_for_stmt (stmt);
5863 gimple_regimplify_operands (stmt, &gsi);
5865 ep = split_block (entry_bb, stmt);
5866 ep->flags = EDGE_TRUE_VALUE;
5867 entry_bb = ep->dest;
5868 ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
5869 ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
5870 ep->probability = REG_BR_PROB_BASE / 2000 - 1;
5871 if (gimple_in_ssa_p (cfun))
5873 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
5874 for (gsi = gsi_start_phis (fin_bb);
5875 !gsi_end_p (gsi); gsi_next (&gsi))
5877 gimple phi = gsi_stmt (gsi);
5878 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
5879 ep, UNKNOWN_LOCATION);
5882 gsi = gsi_last_bb (entry_bb);
5885 t = build_call_expr (builtin_decl_explicit (get_num_threads), 0);
5886 t = fold_convert (itype, t);
5887 nthreads = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5888 true, GSI_SAME_STMT);
5890 t = build_call_expr (builtin_decl_explicit (get_thread_num), 0);
5891 t = fold_convert (itype, t);
5892 threadid = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5893 true, GSI_SAME_STMT);
5897 step = fd->loop.step;
5898 if (gimple_omp_for_combined_into_p (fd->for_stmt))
5900 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
5901 OMP_CLAUSE__LOOPTEMP_);
5902 gcc_assert (innerc);
5903 n1 = OMP_CLAUSE_DECL (innerc);
5904 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5905 OMP_CLAUSE__LOOPTEMP_);
5906 gcc_assert (innerc);
5907 n2 = OMP_CLAUSE_DECL (innerc);
5909 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
5910 true, NULL_TREE, true, GSI_SAME_STMT);
5911 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
5912 true, NULL_TREE, true, GSI_SAME_STMT);
5913 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
5914 true, NULL_TREE, true, GSI_SAME_STMT);
5916 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
5917 t = fold_build2 (PLUS_EXPR, itype, step, t);
5918 t = fold_build2 (PLUS_EXPR, itype, t, n2);
5919 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
5920 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
5921 t = fold_build2 (TRUNC_DIV_EXPR, itype,
5922 fold_build1 (NEGATE_EXPR, itype, t),
5923 fold_build1 (NEGATE_EXPR, itype, step));
5925 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
5926 t = fold_convert (itype, t);
5927 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5929 q = create_tmp_reg (itype, "q");
5930 t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads);
5931 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
5932 gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT);
5934 tt = create_tmp_reg (itype, "tt");
5935 t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads);
5936 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
5937 gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT);
5939 t = build2 (LT_EXPR, boolean_type_node, threadid, tt);
5940 stmt = gimple_build_cond_empty (t);
5941 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5943 second_bb = split_block (entry_bb, stmt)->dest;
5944 gsi = gsi_last_bb (second_bb);
5945 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5947 gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)),
5949 stmt = gimple_build_assign_with_ops (PLUS_EXPR, q, q,
5950 build_int_cst (itype, 1));
5951 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5953 third_bb = split_block (second_bb, stmt)->dest;
5954 gsi = gsi_last_bb (third_bb);
5955 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5957 t = build2 (MULT_EXPR, itype, q, threadid);
5958 t = build2 (PLUS_EXPR, itype, t, tt);
5959 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5961 t = fold_build2 (PLUS_EXPR, itype, s0, q);
5962 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5964 t = build2 (GE_EXPR, boolean_type_node, s0, e0);
5965 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5967 /* Remove the GIMPLE_OMP_FOR statement. */
5968 gsi_remove (&gsi, true);
5970 /* Setup code for sequential iteration goes in SEQ_START_BB. */
5971 gsi = gsi_start_bb (seq_start_bb);
5973 tree startvar = fd->loop.v;
5974 tree endvar = NULL_TREE;
5976 if (gimple_omp_for_combined_p (fd->for_stmt))
5978 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
5979 ? gimple_omp_parallel_clauses (inner_stmt)
5980 : gimple_omp_for_clauses (inner_stmt);
5981 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
5982 gcc_assert (innerc);
5983 startvar = OMP_CLAUSE_DECL (innerc);
5984 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5985 OMP_CLAUSE__LOOPTEMP_);
5986 gcc_assert (innerc);
5987 endvar = OMP_CLAUSE_DECL (innerc);
5989 t = fold_convert (itype, s0);
5990 t = fold_build2 (MULT_EXPR, itype, t, step);
5991 if (POINTER_TYPE_P (type))
5992 t = fold_build_pointer_plus (n1, t);
5994 t = fold_build2 (PLUS_EXPR, type, t, n1);
5995 t = fold_convert (TREE_TYPE (startvar), t);
5996 t = force_gimple_operand_gsi (&gsi, t,
5998 && TREE_ADDRESSABLE (startvar),
5999 NULL_TREE, false, GSI_CONTINUE_LINKING);
6000 stmt = gimple_build_assign (startvar, t);
6001 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6003 t = fold_convert (itype, e0);
6004 t = fold_build2 (MULT_EXPR, itype, t, step);
6005 if (POINTER_TYPE_P (type))
6006 t = fold_build_pointer_plus (n1, t);
6008 t = fold_build2 (PLUS_EXPR, type, t, n1);
6009 t = fold_convert (TREE_TYPE (startvar), t);
6010 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6011 false, GSI_CONTINUE_LINKING);
6014 stmt = gimple_build_assign (endvar, e);
6015 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6016 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
6017 stmt = gimple_build_assign (fd->loop.v, e);
6019 stmt = gimple_build_assign_with_ops (NOP_EXPR, fd->loop.v, e,
6021 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6023 if (fd->collapse > 1)
6024 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
6028 /* The code controlling the sequential loop replaces the
6029 GIMPLE_OMP_CONTINUE. */
6030 gsi = gsi_last_bb (cont_bb);
6031 stmt = gsi_stmt (gsi);
6032 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
6033 vmain = gimple_omp_continue_control_use (stmt);
6034 vback = gimple_omp_continue_control_def (stmt);
6036 if (!gimple_omp_for_combined_p (fd->for_stmt))
6038 if (POINTER_TYPE_P (type))
6039 t = fold_build_pointer_plus (vmain, step);
6041 t = fold_build2 (PLUS_EXPR, type, vmain, step);
6042 t = force_gimple_operand_gsi (&gsi, t,
6044 && TREE_ADDRESSABLE (vback),
6045 NULL_TREE, true, GSI_SAME_STMT);
6046 stmt = gimple_build_assign (vback, t);
6047 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
6049 t = build2 (fd->loop.cond_code, boolean_type_node,
6050 DECL_P (vback) && TREE_ADDRESSABLE (vback)
6052 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6055 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6056 gsi_remove (&gsi, true);
6058 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6059 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
6062 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6063 gsi = gsi_last_bb (exit_bb);
6064 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
6066 t = gimple_omp_return_lhs (gsi_stmt (gsi));
6067 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
6069 gsi_remove (&gsi, true);
6071 /* Connect all the blocks. */
6072 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
6073 ep->probability = REG_BR_PROB_BASE / 4 * 3;
6074 ep = find_edge (entry_bb, second_bb);
6075 ep->flags = EDGE_TRUE_VALUE;
6076 ep->probability = REG_BR_PROB_BASE / 4;
6077 find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
6078 find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
6082 ep = find_edge (cont_bb, body_bb);
6083 if (gimple_omp_for_combined_p (fd->for_stmt))
6088 else if (fd->collapse > 1)
6091 ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6094 ep->flags = EDGE_TRUE_VALUE;
6095 find_edge (cont_bb, fin_bb)->flags
6096 = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
6099 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
6100 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
6101 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb);
6103 set_immediate_dominator (CDI_DOMINATORS, body_bb,
6104 recompute_dominator (CDI_DOMINATORS, body_bb));
6105 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
6106 recompute_dominator (CDI_DOMINATORS, fin_bb));
6108 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
6110 struct loop *loop = alloc_loop ();
6111 loop->header = body_bb;
6112 if (collapse_bb == NULL)
6113 loop->latch = cont_bb;
6114 add_loop (loop, body_bb->loop_father);
6119 /* A subroutine of expand_omp_for. Generate code for a parallel
6120 loop with static schedule and a specified chunk size. Given
6123 for (V = N1; V cond N2; V += STEP) BODY;
6125 where COND is "<" or ">", we generate pseudocode
6127 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6132 if ((__typeof (V)) -1 > 0 && cond is >)
6133 n = -(adj + N2 - N1) / -STEP;
6135 n = (adj + N2 - N1) / STEP;
6137 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6138 here so that V is defined
6139 if the loop is not entered
6141 s0 = (trip * nthreads + threadid) * CHUNK;
6142 e0 = min(s0 + CHUNK, n);
6143 if (s0 < n) goto L1; else goto L4;
6150 if (V cond e) goto L2; else goto L3;
6158 expand_omp_for_static_chunk (struct omp_region *region,
6159 struct omp_for_data *fd, gimple inner_stmt)
6161 tree n, s0, e0, e, t;
6162 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
6163 tree type, itype, vmain, vback, vextra;
6164 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
6165 basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb;
6166 gimple_stmt_iterator gsi;
6169 enum built_in_function get_num_threads = BUILT_IN_OMP_GET_NUM_THREADS;
6170 enum built_in_function get_thread_num = BUILT_IN_OMP_GET_THREAD_NUM;
6171 bool broken_loop = region->cont == NULL;
6172 tree *counts = NULL;
6175 itype = type = TREE_TYPE (fd->loop.v);
6176 if (POINTER_TYPE_P (type))
6177 itype = signed_type_for (type);
6179 entry_bb = region->entry;
6180 se = split_block (entry_bb, last_stmt (entry_bb));
6182 iter_part_bb = se->dest;
6183 cont_bb = region->cont;
6184 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
6185 fin_bb = BRANCH_EDGE (iter_part_bb)->dest;
6186 gcc_assert (broken_loop
6187 || fin_bb == FALLTHRU_EDGE (cont_bb)->dest);
6188 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
6189 body_bb = single_succ (seq_start_bb);
6192 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
6193 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6194 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
6196 exit_bb = region->exit;
6198 /* Trip and adjustment setup goes in ENTRY_BB. */
6199 gsi = gsi_last_bb (entry_bb);
6200 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6202 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
6204 get_num_threads = BUILT_IN_OMP_GET_NUM_TEAMS;
6205 get_thread_num = BUILT_IN_OMP_GET_TEAM_NUM;
6208 if (fd->collapse > 1)
6210 int first_zero_iter = -1;
6211 basic_block l2_dom_bb = NULL;
6213 counts = XALLOCAVEC (tree, fd->collapse);
6214 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6215 fin_bb, first_zero_iter,
6219 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
6220 t = integer_one_node;
6222 t = fold_binary (fd->loop.cond_code, boolean_type_node,
6223 fold_convert (type, fd->loop.n1),
6224 fold_convert (type, fd->loop.n2));
6225 if (fd->collapse == 1
6226 && TYPE_UNSIGNED (type)
6227 && (t == NULL_TREE || !integer_onep (t)))
6229 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
6230 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
6231 true, GSI_SAME_STMT);
6232 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
6233 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
6234 true, GSI_SAME_STMT);
6235 stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
6236 NULL_TREE, NULL_TREE);
6237 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
6238 if (walk_tree (gimple_cond_lhs_ptr (stmt),
6239 expand_omp_regimplify_p, NULL, NULL)
6240 || walk_tree (gimple_cond_rhs_ptr (stmt),
6241 expand_omp_regimplify_p, NULL, NULL))
6243 gsi = gsi_for_stmt (stmt);
6244 gimple_regimplify_operands (stmt, &gsi);
6246 se = split_block (entry_bb, stmt);
6247 se->flags = EDGE_TRUE_VALUE;
6248 entry_bb = se->dest;
6249 se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
6250 se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
6251 se->probability = REG_BR_PROB_BASE / 2000 - 1;
6252 if (gimple_in_ssa_p (cfun))
6254 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
6255 for (gsi = gsi_start_phis (fin_bb);
6256 !gsi_end_p (gsi); gsi_next (&gsi))
6258 gimple phi = gsi_stmt (gsi);
6259 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
6260 se, UNKNOWN_LOCATION);
6263 gsi = gsi_last_bb (entry_bb);
6266 t = build_call_expr (builtin_decl_explicit (get_num_threads), 0);
6267 t = fold_convert (itype, t);
6268 nthreads = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6269 true, GSI_SAME_STMT);
6271 t = build_call_expr (builtin_decl_explicit (get_thread_num), 0);
6272 t = fold_convert (itype, t);
6273 threadid = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6274 true, GSI_SAME_STMT);
6278 step = fd->loop.step;
6279 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6281 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6282 OMP_CLAUSE__LOOPTEMP_);
6283 gcc_assert (innerc);
6284 n1 = OMP_CLAUSE_DECL (innerc);
6285 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6286 OMP_CLAUSE__LOOPTEMP_);
6287 gcc_assert (innerc);
6288 n2 = OMP_CLAUSE_DECL (innerc);
6290 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
6291 true, NULL_TREE, true, GSI_SAME_STMT);
6292 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
6293 true, NULL_TREE, true, GSI_SAME_STMT);
6294 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
6295 true, NULL_TREE, true, GSI_SAME_STMT);
6297 = force_gimple_operand_gsi (&gsi, fold_convert (itype, fd->chunk_size),
6298 true, NULL_TREE, true, GSI_SAME_STMT);
6300 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
6301 t = fold_build2 (PLUS_EXPR, itype, step, t);
6302 t = fold_build2 (PLUS_EXPR, itype, t, n2);
6303 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
6304 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
6305 t = fold_build2 (TRUNC_DIV_EXPR, itype,
6306 fold_build1 (NEGATE_EXPR, itype, t),
6307 fold_build1 (NEGATE_EXPR, itype, step));
6309 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
6310 t = fold_convert (itype, t);
6311 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6312 true, GSI_SAME_STMT);
6314 trip_var = create_tmp_reg (itype, ".trip");
6315 if (gimple_in_ssa_p (cfun))
6317 trip_init = make_ssa_name (trip_var, NULL);
6318 trip_main = make_ssa_name (trip_var, NULL);
6319 trip_back = make_ssa_name (trip_var, NULL);
6323 trip_init = trip_var;
6324 trip_main = trip_var;
6325 trip_back = trip_var;
6328 stmt = gimple_build_assign (trip_init, build_int_cst (itype, 0));
6329 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
6331 t = fold_build2 (MULT_EXPR, itype, threadid, fd->chunk_size);
6332 t = fold_build2 (MULT_EXPR, itype, t, step);
6333 if (POINTER_TYPE_P (type))
6334 t = fold_build_pointer_plus (n1, t);
6336 t = fold_build2 (PLUS_EXPR, type, t, n1);
6337 vextra = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6338 true, GSI_SAME_STMT);
6340 /* Remove the GIMPLE_OMP_FOR. */
6341 gsi_remove (&gsi, true);
6343 /* Iteration space partitioning goes in ITER_PART_BB. */
6344 gsi = gsi_last_bb (iter_part_bb);
6346 t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads);
6347 t = fold_build2 (PLUS_EXPR, itype, t, threadid);
6348 t = fold_build2 (MULT_EXPR, itype, t, fd->chunk_size);
6349 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6350 false, GSI_CONTINUE_LINKING);
6352 t = fold_build2 (PLUS_EXPR, itype, s0, fd->chunk_size);
6353 t = fold_build2 (MIN_EXPR, itype, t, n);
6354 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6355 false, GSI_CONTINUE_LINKING);
6357 t = build2 (LT_EXPR, boolean_type_node, s0, n);
6358 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING);
6360 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6361 gsi = gsi_start_bb (seq_start_bb);
6363 tree startvar = fd->loop.v;
6364 tree endvar = NULL_TREE;
6366 if (gimple_omp_for_combined_p (fd->for_stmt))
6368 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
6369 ? gimple_omp_parallel_clauses (inner_stmt)
6370 : gimple_omp_for_clauses (inner_stmt);
6371 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
6372 gcc_assert (innerc);
6373 startvar = OMP_CLAUSE_DECL (innerc);
6374 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6375 OMP_CLAUSE__LOOPTEMP_);
6376 gcc_assert (innerc);
6377 endvar = OMP_CLAUSE_DECL (innerc);
6380 t = fold_convert (itype, s0);
6381 t = fold_build2 (MULT_EXPR, itype, t, step);
6382 if (POINTER_TYPE_P (type))
6383 t = fold_build_pointer_plus (n1, t);
6385 t = fold_build2 (PLUS_EXPR, type, t, n1);
6386 t = fold_convert (TREE_TYPE (startvar), t);
6387 t = force_gimple_operand_gsi (&gsi, t,
6389 && TREE_ADDRESSABLE (startvar),
6390 NULL_TREE, false, GSI_CONTINUE_LINKING);
6391 stmt = gimple_build_assign (startvar, t);
6392 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6394 t = fold_convert (itype, e0);
6395 t = fold_build2 (MULT_EXPR, itype, t, step);
6396 if (POINTER_TYPE_P (type))
6397 t = fold_build_pointer_plus (n1, t);
6399 t = fold_build2 (PLUS_EXPR, type, t, n1);
6400 t = fold_convert (TREE_TYPE (startvar), t);
6401 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6402 false, GSI_CONTINUE_LINKING);
6405 stmt = gimple_build_assign (endvar, e);
6406 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6407 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
6408 stmt = gimple_build_assign (fd->loop.v, e);
6410 stmt = gimple_build_assign_with_ops (NOP_EXPR, fd->loop.v, e,
6412 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6414 if (fd->collapse > 1)
6415 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
6419 /* The code controlling the sequential loop goes in CONT_BB,
6420 replacing the GIMPLE_OMP_CONTINUE. */
6421 gsi = gsi_last_bb (cont_bb);
6422 stmt = gsi_stmt (gsi);
6423 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
6424 vmain = gimple_omp_continue_control_use (stmt);
6425 vback = gimple_omp_continue_control_def (stmt);
6427 if (!gimple_omp_for_combined_p (fd->for_stmt))
6429 if (POINTER_TYPE_P (type))
6430 t = fold_build_pointer_plus (vmain, step);
6432 t = fold_build2 (PLUS_EXPR, type, vmain, step);
6433 if (DECL_P (vback) && TREE_ADDRESSABLE (vback))
6434 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6435 true, GSI_SAME_STMT);
6436 stmt = gimple_build_assign (vback, t);
6437 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
6439 t = build2 (fd->loop.cond_code, boolean_type_node,
6440 DECL_P (vback) && TREE_ADDRESSABLE (vback)
6442 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6445 /* Remove GIMPLE_OMP_CONTINUE. */
6446 gsi_remove (&gsi, true);
6448 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6449 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
6451 /* Trip update code goes into TRIP_UPDATE_BB. */
6452 gsi = gsi_start_bb (trip_update_bb);
6454 t = build_int_cst (itype, 1);
6455 t = build2 (PLUS_EXPR, itype, trip_main, t);
6456 stmt = gimple_build_assign (trip_back, t);
6457 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6460 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6461 gsi = gsi_last_bb (exit_bb);
6462 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
6464 t = gimple_omp_return_lhs (gsi_stmt (gsi));
6465 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
6467 gsi_remove (&gsi, true);
6469 /* Connect the new blocks. */
6470 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
6471 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
6475 se = find_edge (cont_bb, body_bb);
6476 if (gimple_omp_for_combined_p (fd->for_stmt))
6481 else if (fd->collapse > 1)
6484 se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6487 se->flags = EDGE_TRUE_VALUE;
6488 find_edge (cont_bb, trip_update_bb)->flags
6489 = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
6491 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb);
6494 if (gimple_in_ssa_p (cfun))
6496 gimple_stmt_iterator psi;
6499 edge_var_map_vector *head;
6503 gcc_assert (fd->collapse == 1 && !broken_loop);
6505 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6506 remove arguments of the phi nodes in fin_bb. We need to create
6507 appropriate phi nodes in iter_part_bb instead. */
6508 se = single_pred_edge (fin_bb);
6509 re = single_succ_edge (trip_update_bb);
6510 head = redirect_edge_var_map_vector (re);
6511 ene = single_succ_edge (entry_bb);
6513 psi = gsi_start_phis (fin_bb);
6514 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
6515 gsi_next (&psi), ++i)
6518 source_location locus;
6520 phi = gsi_stmt (psi);
6521 t = gimple_phi_result (phi);
6522 gcc_assert (t == redirect_edge_var_map_result (vm));
6523 nphi = create_phi_node (t, iter_part_bb);
6525 t = PHI_ARG_DEF_FROM_EDGE (phi, se);
6526 locus = gimple_phi_arg_location_from_edge (phi, se);
6528 /* A special case -- fd->loop.v is not yet computed in
6529 iter_part_bb, we need to use vextra instead. */
6530 if (t == fd->loop.v)
6532 add_phi_arg (nphi, t, ene, locus);
6533 locus = redirect_edge_var_map_location (vm);
6534 add_phi_arg (nphi, redirect_edge_var_map_def (vm), re, locus);
6536 gcc_assert (!gsi_end_p (psi) && i == head->length ());
6537 redirect_edge_var_map_clear (re);
6540 psi = gsi_start_phis (fin_bb);
6541 if (gsi_end_p (psi))
6543 remove_phi_node (&psi, false);
6546 /* Make phi node for trip. */
6547 phi = create_phi_node (trip_main, iter_part_bb);
6548 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb),
6550 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb),
6555 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
6556 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
6557 recompute_dominator (CDI_DOMINATORS, iter_part_bb));
6558 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
6559 recompute_dominator (CDI_DOMINATORS, fin_bb));
6560 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
6561 recompute_dominator (CDI_DOMINATORS, seq_start_bb));
6562 set_immediate_dominator (CDI_DOMINATORS, body_bb,
6563 recompute_dominator (CDI_DOMINATORS, body_bb));
6567 struct loop *trip_loop = alloc_loop ();
6568 trip_loop->header = iter_part_bb;
6569 trip_loop->latch = trip_update_bb;
6570 add_loop (trip_loop, iter_part_bb->loop_father);
6572 if (!gimple_omp_for_combined_p (fd->for_stmt))
6574 struct loop *loop = alloc_loop ();
6575 loop->header = body_bb;
6576 if (collapse_bb == NULL)
6577 loop->latch = cont_bb;
6578 add_loop (loop, trip_loop);
6584 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6585 loop. Given parameters:
6587 for (V = N1; V cond N2; V += STEP) BODY;
6589 where COND is "<" or ">", we generate pseudocode
6597 if (V cond N2) goto L0; else goto L2;
6600 For collapsed loops, given parameters:
6602 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6603 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6604 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6607 we generate pseudocode
6613 count3 = (adj + N32 - N31) / STEP3;
6618 count2 = (adj + N22 - N21) / STEP2;
6623 count1 = (adj + N12 - N11) / STEP1;
6624 count = count1 * count2 * count3;
6634 V2 += (V3 cond3 N32) ? 0 : STEP2;
6635 V3 = (V3 cond3 N32) ? V3 : N31;
6636 V1 += (V2 cond2 N22) ? 0 : STEP1;
6637 V2 = (V2 cond2 N22) ? V2 : N21;
6639 if (V < count) goto L0; else goto L2;
6645 expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
6648 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb;
6649 gimple_stmt_iterator gsi;
6651 bool broken_loop = region->cont == NULL;
6653 tree *counts = NULL;
6655 tree safelen = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6656 OMP_CLAUSE_SAFELEN);
6657 tree simduid = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6658 OMP_CLAUSE__SIMDUID_);
6661 type = TREE_TYPE (fd->loop.v);
6662 entry_bb = region->entry;
6663 cont_bb = region->cont;
6664 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6665 gcc_assert (broken_loop
6666 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
6667 l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
6670 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
6671 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6672 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
6673 l2_bb = BRANCH_EDGE (entry_bb)->dest;
6677 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
6678 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
6679 l2_bb = single_succ (l1_bb);
6681 exit_bb = region->exit;
6684 gsi = gsi_last_bb (entry_bb);
6686 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6687 /* Not needed in SSA form right now. */
6688 gcc_assert (!gimple_in_ssa_p (cfun));
6689 if (fd->collapse > 1)
6691 int first_zero_iter = -1;
6692 basic_block zero_iter_bb = l2_bb;
6694 counts = XALLOCAVEC (tree, fd->collapse);
6695 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6696 zero_iter_bb, first_zero_iter,
6699 if (l2_dom_bb == NULL)
6704 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6706 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6707 OMP_CLAUSE__LOOPTEMP_);
6708 gcc_assert (innerc);
6709 n1 = OMP_CLAUSE_DECL (innerc);
6710 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6711 OMP_CLAUSE__LOOPTEMP_);
6712 gcc_assert (innerc);
6713 n2 = OMP_CLAUSE_DECL (innerc);
6714 expand_omp_build_assign (&gsi, fd->loop.v,
6715 fold_convert (type, n1));
6716 if (fd->collapse > 1)
6719 expand_omp_for_init_vars (fd, &gsi, counts, NULL, n1);
6725 expand_omp_build_assign (&gsi, fd->loop.v,
6726 fold_convert (type, fd->loop.n1));
6727 if (fd->collapse > 1)
6728 for (i = 0; i < fd->collapse; i++)
6730 tree itype = TREE_TYPE (fd->loops[i].v);
6731 if (POINTER_TYPE_P (itype))
6732 itype = signed_type_for (itype);
6733 t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1);
6734 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6738 /* Remove the GIMPLE_OMP_FOR statement. */
6739 gsi_remove (&gsi, true);
6743 /* Code to control the increment goes in the CONT_BB. */
6744 gsi = gsi_last_bb (cont_bb);
6745 stmt = gsi_stmt (gsi);
6746 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
6748 if (POINTER_TYPE_P (type))
6749 t = fold_build_pointer_plus (fd->loop.v, fd->loop.step);
6751 t = fold_build2 (PLUS_EXPR, type, fd->loop.v, fd->loop.step);
6752 expand_omp_build_assign (&gsi, fd->loop.v, t);
6754 if (fd->collapse > 1)
6756 i = fd->collapse - 1;
6757 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
6759 t = fold_convert (sizetype, fd->loops[i].step);
6760 t = fold_build_pointer_plus (fd->loops[i].v, t);
6764 t = fold_convert (TREE_TYPE (fd->loops[i].v),
6766 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
6769 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6771 for (i = fd->collapse - 1; i > 0; i--)
6773 tree itype = TREE_TYPE (fd->loops[i].v);
6774 tree itype2 = TREE_TYPE (fd->loops[i - 1].v);
6775 if (POINTER_TYPE_P (itype2))
6776 itype2 = signed_type_for (itype2);
6777 t = build3 (COND_EXPR, itype2,
6778 build2 (fd->loops[i].cond_code, boolean_type_node,
6780 fold_convert (itype, fd->loops[i].n2)),
6781 build_int_cst (itype2, 0),
6782 fold_convert (itype2, fd->loops[i - 1].step));
6783 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i - 1].v)))
6784 t = fold_build_pointer_plus (fd->loops[i - 1].v, t);
6786 t = fold_build2 (PLUS_EXPR, itype2, fd->loops[i - 1].v, t);
6787 expand_omp_build_assign (&gsi, fd->loops[i - 1].v, t);
6789 t = build3 (COND_EXPR, itype,
6790 build2 (fd->loops[i].cond_code, boolean_type_node,
6792 fold_convert (itype, fd->loops[i].n2)),
6794 fold_convert (itype, fd->loops[i].n1));
6795 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6799 /* Remove GIMPLE_OMP_CONTINUE. */
6800 gsi_remove (&gsi, true);
6803 /* Emit the condition in L1_BB. */
6804 gsi = gsi_start_bb (l1_bb);
6806 t = fold_convert (type, n2);
6807 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6808 false, GSI_CONTINUE_LINKING);
6809 t = build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t);
6810 stmt = gimple_build_cond_empty (t);
6811 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6812 if (walk_tree (gimple_cond_lhs_ptr (stmt), expand_omp_regimplify_p,
6814 || walk_tree (gimple_cond_rhs_ptr (stmt), expand_omp_regimplify_p,
6817 gsi = gsi_for_stmt (stmt);
6818 gimple_regimplify_operands (stmt, &gsi);
6821 /* Remove GIMPLE_OMP_RETURN. */
6822 gsi = gsi_last_bb (exit_bb);
6823 gsi_remove (&gsi, true);
6825 /* Connect the new blocks. */
6826 remove_edge (FALLTHRU_EDGE (entry_bb));
6830 remove_edge (BRANCH_EDGE (entry_bb));
6831 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
6833 e = BRANCH_EDGE (l1_bb);
6834 ne = FALLTHRU_EDGE (l1_bb);
6835 e->flags = EDGE_TRUE_VALUE;
6839 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
6841 ne = single_succ_edge (l1_bb);
6842 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
6845 ne->flags = EDGE_FALSE_VALUE;
6846 e->probability = REG_BR_PROB_BASE * 7 / 8;
6847 ne->probability = REG_BR_PROB_BASE / 8;
6849 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
6850 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
6851 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
6855 struct loop *loop = alloc_loop ();
6856 loop->header = l1_bb;
6857 loop->latch = cont_bb;
6858 add_loop (loop, l1_bb->loop_father);
6859 if (safelen == NULL_TREE)
6860 loop->safelen = INT_MAX;
6863 safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen);
6864 if (!tree_fits_uhwi_p (safelen)
6865 || tree_to_uhwi (safelen) > INT_MAX)
6866 loop->safelen = INT_MAX;
6868 loop->safelen = tree_to_uhwi (safelen);
6869 if (loop->safelen == 1)
6874 loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
6875 cfun->has_simduid_loops = true;
6877 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6879 if ((flag_tree_loop_vectorize
6880 || (!global_options_set.x_flag_tree_loop_vectorize
6881 && !global_options_set.x_flag_tree_vectorize))
6882 && flag_tree_loop_optimize
6883 && loop->safelen > 1)
6885 loop->force_vectorize = true;
6886 cfun->has_force_vectorize_loops = true;
6892 /* Expand the OpenMP loop defined by REGION. */
6895 expand_omp_for (struct omp_region *region, gimple inner_stmt)
6897 struct omp_for_data fd;
6898 struct omp_for_data_loop *loops;
6901 = (struct omp_for_data_loop *)
6902 alloca (gimple_omp_for_collapse (last_stmt (region->entry))
6903 * sizeof (struct omp_for_data_loop));
6904 extract_omp_for_data (last_stmt (region->entry), &fd, loops);
6905 region->sched_kind = fd.sched_kind;
6907 gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
6908 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
6909 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
6912 gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
6913 BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
6914 FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
6917 /* If there isn't a continue then this is a degerate case where
6918 the introduction of abnormal edges during lowering will prevent
6919 original loops from being detected. Fix that up. */
6920 loops_state_set (LOOPS_NEED_FIXUP);
6922 if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD)
6923 expand_omp_simd (region, &fd);
6924 else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
6925 && !fd.have_ordered)
6927 if (fd.chunk_size == NULL)
6928 expand_omp_for_static_nochunk (region, &fd, inner_stmt);
6930 expand_omp_for_static_chunk (region, &fd, inner_stmt);
6934 int fn_index, start_ix, next_ix;
6936 gcc_assert (gimple_omp_for_kind (fd.for_stmt)
6937 == GF_OMP_FOR_KIND_FOR);
6938 if (fd.chunk_size == NULL
6939 && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
6940 fd.chunk_size = integer_zero_node;
6941 gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
6942 fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
6943 ? 3 : fd.sched_kind;
6944 fn_index += fd.have_ordered * 4;
6945 start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
6946 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
6947 if (fd.iter_type == long_long_unsigned_type_node)
6949 start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6950 - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
6951 next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6952 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
6954 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
6955 (enum built_in_function) next_ix, inner_stmt);
6958 if (gimple_in_ssa_p (cfun))
6959 update_ssa (TODO_update_ssa_only_virtuals);
6963 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
6965 v = GOMP_sections_start (n);
6982 v = GOMP_sections_next ();
6987 If this is a combined parallel sections, replace the call to
6988 GOMP_sections_start with call to GOMP_sections_next. */
6991 expand_omp_sections (struct omp_region *region)
6993 tree t, u, vin = NULL, vmain, vnext, l2;
6995 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
6996 gimple_stmt_iterator si, switch_si;
6997 gimple sections_stmt, stmt, cont;
7000 struct omp_region *inner;
7002 bool exit_reachable = region->cont != NULL;
7004 gcc_assert (region->exit != NULL);
7005 entry_bb = region->entry;
7006 l0_bb = single_succ (entry_bb);
7007 l1_bb = region->cont;
7008 l2_bb = region->exit;
7009 if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb)
7010 l2 = gimple_block_label (l2_bb);
7013 /* This can happen if there are reductions. */
7014 len = EDGE_COUNT (l0_bb->succs);
7015 gcc_assert (len > 0);
7016 e = EDGE_SUCC (l0_bb, len - 1);
7017 si = gsi_last_bb (e->dest);
7020 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
7021 l2 = gimple_block_label (e->dest);
7023 FOR_EACH_EDGE (e, ei, l0_bb->succs)
7025 si = gsi_last_bb (e->dest);
7027 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
7029 l2 = gimple_block_label (e->dest);
7035 default_bb = create_empty_bb (l1_bb->prev_bb);
7037 default_bb = create_empty_bb (l0_bb);
7039 /* We will build a switch() with enough cases for all the
7040 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7041 and a default case to abort if something goes wrong. */
7042 len = EDGE_COUNT (l0_bb->succs);
7044 /* Use vec::quick_push on label_vec throughout, since we know the size
7046 auto_vec<tree> label_vec (len);
7048 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7049 GIMPLE_OMP_SECTIONS statement. */
7050 si = gsi_last_bb (entry_bb);
7051 sections_stmt = gsi_stmt (si);
7052 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
7053 vin = gimple_omp_sections_control (sections_stmt);
7054 if (!is_combined_parallel (region))
7056 /* If we are not inside a combined parallel+sections region,
7057 call GOMP_sections_start. */
7058 t = build_int_cst (unsigned_type_node, len - 1);
7059 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
7060 stmt = gimple_build_call (u, 1, t);
7064 /* Otherwise, call GOMP_sections_next. */
7065 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
7066 stmt = gimple_build_call (u, 0);
7068 gimple_call_set_lhs (stmt, vin);
7069 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
7070 gsi_remove (&si, true);
7072 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7074 switch_si = gsi_last_bb (l0_bb);
7075 gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH);
7078 cont = last_stmt (l1_bb);
7079 gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE);
7080 vmain = gimple_omp_continue_control_use (cont);
7081 vnext = gimple_omp_continue_control_def (cont);
7089 t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2);
7090 label_vec.quick_push (t);
7093 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7094 for (inner = region->inner, casei = 1;
7096 inner = inner->next, i++, casei++)
7098 basic_block s_entry_bb, s_exit_bb;
7100 /* Skip optional reduction region. */
7101 if (inner->type == GIMPLE_OMP_ATOMIC_LOAD)
7108 s_entry_bb = inner->entry;
7109 s_exit_bb = inner->exit;
7111 t = gimple_block_label (s_entry_bb);
7112 u = build_int_cst (unsigned_type_node, casei);
7113 u = build_case_label (u, NULL, t);
7114 label_vec.quick_push (u);
7116 si = gsi_last_bb (s_entry_bb);
7117 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION);
7118 gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si)));
7119 gsi_remove (&si, true);
7120 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
7122 if (s_exit_bb == NULL)
7125 si = gsi_last_bb (s_exit_bb);
7126 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
7127 gsi_remove (&si, true);
7129 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
7132 /* Error handling code goes in DEFAULT_BB. */
7133 t = gimple_block_label (default_bb);
7134 u = build_case_label (NULL, NULL, t);
7135 make_edge (l0_bb, default_bb, 0);
7136 add_bb_to_loop (default_bb, current_loops->tree_root);
7138 stmt = gimple_build_switch (vmain, u, label_vec);
7139 gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT);
7140 gsi_remove (&switch_si, true);
7142 si = gsi_start_bb (default_bb);
7143 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
7144 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
7150 /* Code to get the next section goes in L1_BB. */
7151 si = gsi_last_bb (l1_bb);
7152 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
7154 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
7155 stmt = gimple_build_call (bfn_decl, 0);
7156 gimple_call_set_lhs (stmt, vnext);
7157 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
7158 gsi_remove (&si, true);
7160 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
7163 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7164 si = gsi_last_bb (l2_bb);
7165 if (gimple_omp_return_nowait_p (gsi_stmt (si)))
7166 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT);
7167 else if (gimple_omp_return_lhs (gsi_stmt (si)))
7168 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL);
7170 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END);
7171 stmt = gimple_build_call (t, 0);
7172 if (gimple_omp_return_lhs (gsi_stmt (si)))
7173 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si)));
7174 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
7175 gsi_remove (&si, true);
7177 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
7181 /* Expand code for an OpenMP single directive. We've already expanded
7182 much of the code, here we simply place the GOMP_barrier call. */
7185 expand_omp_single (struct omp_region *region)
7187 basic_block entry_bb, exit_bb;
7188 gimple_stmt_iterator si;
7190 entry_bb = region->entry;
7191 exit_bb = region->exit;
7193 si = gsi_last_bb (entry_bb);
7194 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE);
7195 gsi_remove (&si, true);
7196 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7198 si = gsi_last_bb (exit_bb);
7199 if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
7201 tree t = gimple_omp_return_lhs (gsi_stmt (si));
7202 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT);
7204 gsi_remove (&si, true);
7205 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
7209 /* Generic expansion for OpenMP synchronization directives: master,
7210 ordered and critical. All we need to do here is remove the entry
7211 and exit markers for REGION. */
7214 expand_omp_synch (struct omp_region *region)
7216 basic_block entry_bb, exit_bb;
7217 gimple_stmt_iterator si;
7219 entry_bb = region->entry;
7220 exit_bb = region->exit;
7222 si = gsi_last_bb (entry_bb);
7223 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
7224 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
7225 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
7226 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
7227 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
7228 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS);
7229 gsi_remove (&si, true);
7230 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7234 si = gsi_last_bb (exit_bb);
7235 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
7236 gsi_remove (&si, true);
7237 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
7241 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7242 operation as a normal volatile load. */
7245 expand_omp_atomic_load (basic_block load_bb, tree addr,
7246 tree loaded_val, int index)
7248 enum built_in_function tmpbase;
7249 gimple_stmt_iterator gsi;
7250 basic_block store_bb;
7253 tree decl, call, type, itype;
7255 gsi = gsi_last_bb (load_bb);
7256 stmt = gsi_stmt (gsi);
7257 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
7258 loc = gimple_location (stmt);
7260 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7261 is smaller than word size, then expand_atomic_load assumes that the load
7262 is atomic. We could avoid the builtin entirely in this case. */
7264 tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
7265 decl = builtin_decl_explicit (tmpbase);
7266 if (decl == NULL_TREE)
7269 type = TREE_TYPE (loaded_val);
7270 itype = TREE_TYPE (TREE_TYPE (decl));
7272 call = build_call_expr_loc (loc, decl, 2, addr,
7273 build_int_cst (NULL,
7274 gimple_omp_atomic_seq_cst_p (stmt)
7276 : MEMMODEL_RELAXED));
7277 if (!useless_type_conversion_p (type, itype))
7278 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
7279 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
7281 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
7282 gsi_remove (&gsi, true);
7284 store_bb = single_succ (load_bb);
7285 gsi = gsi_last_bb (store_bb);
7286 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
7287 gsi_remove (&gsi, true);
7289 if (gimple_in_ssa_p (cfun))
7290 update_ssa (TODO_update_ssa_no_phi);
7295 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7296 operation as a normal volatile store. */
7299 expand_omp_atomic_store (basic_block load_bb, tree addr,
7300 tree loaded_val, tree stored_val, int index)
7302 enum built_in_function tmpbase;
7303 gimple_stmt_iterator gsi;
7304 basic_block store_bb = single_succ (load_bb);
7307 tree decl, call, type, itype;
7308 enum machine_mode imode;
7311 gsi = gsi_last_bb (load_bb);
7312 stmt = gsi_stmt (gsi);
7313 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
7315 /* If the load value is needed, then this isn't a store but an exchange. */
7316 exchange = gimple_omp_atomic_need_value_p (stmt);
7318 gsi = gsi_last_bb (store_bb);
7319 stmt = gsi_stmt (gsi);
7320 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE);
7321 loc = gimple_location (stmt);
7323 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7324 is smaller than word size, then expand_atomic_store assumes that the store
7325 is atomic. We could avoid the builtin entirely in this case. */
7327 tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N);
7328 tmpbase = (enum built_in_function) ((int) tmpbase + index + 1);
7329 decl = builtin_decl_explicit (tmpbase);
7330 if (decl == NULL_TREE)
7333 type = TREE_TYPE (stored_val);
7335 /* Dig out the type of the function's second argument. */
7336 itype = TREE_TYPE (decl);
7337 itype = TYPE_ARG_TYPES (itype);
7338 itype = TREE_CHAIN (itype);
7339 itype = TREE_VALUE (itype);
7340 imode = TYPE_MODE (itype);
7342 if (exchange && !can_atomic_exchange_p (imode, true))
7345 if (!useless_type_conversion_p (itype, type))
7346 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
7347 call = build_call_expr_loc (loc, decl, 3, addr, stored_val,
7348 build_int_cst (NULL,
7349 gimple_omp_atomic_seq_cst_p (stmt)
7351 : MEMMODEL_RELAXED));
7354 if (!useless_type_conversion_p (type, itype))
7355 call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
7356 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
7359 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
7360 gsi_remove (&gsi, true);
7362 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7363 gsi = gsi_last_bb (load_bb);
7364 gsi_remove (&gsi, true);
7366 if (gimple_in_ssa_p (cfun))
7367 update_ssa (TODO_update_ssa_no_phi);
7372 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7373 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7374 size of the data type, and thus usable to find the index of the builtin
7375 decl. Returns false if the expression is not of the proper form. */
7378 expand_omp_atomic_fetch_op (basic_block load_bb,
7379 tree addr, tree loaded_val,
7380 tree stored_val, int index)
7382 enum built_in_function oldbase, newbase, tmpbase;
7383 tree decl, itype, call;
7385 basic_block store_bb = single_succ (load_bb);
7386 gimple_stmt_iterator gsi;
7389 enum tree_code code;
7390 bool need_old, need_new;
7391 enum machine_mode imode;
7394 /* We expect to find the following sequences:
7397 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7400 val = tmp OP something; (or: something OP tmp)
7401 GIMPLE_OMP_STORE (val)
7403 ???FIXME: Allow a more flexible sequence.
7404 Perhaps use data flow to pick the statements.
7408 gsi = gsi_after_labels (store_bb);
7409 stmt = gsi_stmt (gsi);
7410 loc = gimple_location (stmt);
7411 if (!is_gimple_assign (stmt))
7414 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
7416 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
7417 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb));
7418 seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb));
7419 gcc_checking_assert (!need_old || !need_new);
7421 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
7424 /* Check for one of the supported fetch-op operations. */
7425 code = gimple_assign_rhs_code (stmt);
7429 case POINTER_PLUS_EXPR:
7430 oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N;
7431 newbase = BUILT_IN_ATOMIC_ADD_FETCH_N;
7434 oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N;
7435 newbase = BUILT_IN_ATOMIC_SUB_FETCH_N;
7438 oldbase = BUILT_IN_ATOMIC_FETCH_AND_N;
7439 newbase = BUILT_IN_ATOMIC_AND_FETCH_N;
7442 oldbase = BUILT_IN_ATOMIC_FETCH_OR_N;
7443 newbase = BUILT_IN_ATOMIC_OR_FETCH_N;
7446 oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N;
7447 newbase = BUILT_IN_ATOMIC_XOR_FETCH_N;
7453 /* Make sure the expression is of the proper form. */
7454 if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
7455 rhs = gimple_assign_rhs2 (stmt);
7456 else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
7457 && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
7458 rhs = gimple_assign_rhs1 (stmt);
7462 tmpbase = ((enum built_in_function)
7463 ((need_new ? newbase : oldbase) + index + 1));
7464 decl = builtin_decl_explicit (tmpbase);
7465 if (decl == NULL_TREE)
7467 itype = TREE_TYPE (TREE_TYPE (decl));
7468 imode = TYPE_MODE (itype);
7470 /* We could test all of the various optabs involved, but the fact of the
7471 matter is that (with the exception of i486 vs i586 and xadd) all targets
7472 that support any atomic operaton optab also implements compare-and-swap.
7473 Let optabs.c take care of expanding any compare-and-swap loop. */
7474 if (!can_compare_and_swap_p (imode, true))
7477 gsi = gsi_last_bb (load_bb);
7478 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
7480 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7481 It only requires that the operation happen atomically. Thus we can
7482 use the RELAXED memory model. */
7483 call = build_call_expr_loc (loc, decl, 3, addr,
7484 fold_convert_loc (loc, itype, rhs),
7485 build_int_cst (NULL,
7486 seq_cst ? MEMMODEL_SEQ_CST
7487 : MEMMODEL_RELAXED));
7489 if (need_old || need_new)
7491 lhs = need_old ? loaded_val : stored_val;
7492 call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
7493 call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call);
7496 call = fold_convert_loc (loc, void_type_node, call);
7497 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
7498 gsi_remove (&gsi, true);
7500 gsi = gsi_last_bb (store_bb);
7501 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
7502 gsi_remove (&gsi, true);
7503 gsi = gsi_last_bb (store_bb);
7504 gsi_remove (&gsi, true);
7506 if (gimple_in_ssa_p (cfun))
7507 update_ssa (TODO_update_ssa_no_phi);
7512 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7516 newval = rhs; // with oldval replacing *addr in rhs
7517 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7518 if (oldval != newval)
7521 INDEX is log2 of the size of the data type, and thus usable to find the
7522 index of the builtin decl. */
7525 expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
7526 tree addr, tree loaded_val, tree stored_val,
7529 tree loadedi, storedi, initial, new_storedi, old_vali;
7530 tree type, itype, cmpxchg, iaddr;
7531 gimple_stmt_iterator si;
7532 basic_block loop_header = single_succ (load_bb);
7535 enum built_in_function fncode;
7537 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7538 order to use the RELAXED memory model effectively. */
7539 fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7541 cmpxchg = builtin_decl_explicit (fncode);
7542 if (cmpxchg == NULL_TREE)
7544 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
7545 itype = TREE_TYPE (TREE_TYPE (cmpxchg));
7547 if (!can_compare_and_swap_p (TYPE_MODE (itype), true))
7550 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7551 si = gsi_last_bb (load_bb);
7552 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
7554 /* For floating-point values, we'll need to view-convert them to integers
7555 so that we can perform the atomic compare and swap. Simplify the
7556 following code by always setting up the "i"ntegral variables. */
7557 if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
7561 iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode,
7564 = force_gimple_operand_gsi (&si,
7565 fold_convert (TREE_TYPE (iaddr), addr),
7566 false, NULL_TREE, true, GSI_SAME_STMT);
7567 stmt = gimple_build_assign (iaddr, iaddr_val);
7568 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7569 loadedi = create_tmp_var (itype, NULL);
7570 if (gimple_in_ssa_p (cfun))
7571 loadedi = make_ssa_name (loadedi, NULL);
7576 loadedi = loaded_val;
7579 fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
7580 tree loaddecl = builtin_decl_explicit (fncode);
7583 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr)),
7584 build_call_expr (loaddecl, 2, iaddr,
7585 build_int_cst (NULL_TREE,
7586 MEMMODEL_RELAXED)));
7588 initial = build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)), iaddr,
7589 build_int_cst (TREE_TYPE (iaddr), 0));
7592 = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true,
7595 /* Move the value to the LOADEDI temporary. */
7596 if (gimple_in_ssa_p (cfun))
7598 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
7599 phi = create_phi_node (loadedi, loop_header);
7600 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
7604 gsi_insert_before (&si,
7605 gimple_build_assign (loadedi, initial),
7607 if (loadedi != loaded_val)
7609 gimple_stmt_iterator gsi2;
7612 x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
7613 gsi2 = gsi_start_bb (loop_header);
7614 if (gimple_in_ssa_p (cfun))
7617 x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
7618 true, GSI_SAME_STMT);
7619 stmt = gimple_build_assign (loaded_val, x);
7620 gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
7624 x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
7625 force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
7626 true, GSI_SAME_STMT);
7629 gsi_remove (&si, true);
7631 si = gsi_last_bb (store_bb);
7632 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
7635 storedi = stored_val;
7638 force_gimple_operand_gsi (&si,
7639 build1 (VIEW_CONVERT_EXPR, itype,
7640 stored_val), true, NULL_TREE, true,
7643 /* Build the compare&swap statement. */
7644 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
7645 new_storedi = force_gimple_operand_gsi (&si,
7646 fold_convert (TREE_TYPE (loadedi),
7649 true, GSI_SAME_STMT);
7651 if (gimple_in_ssa_p (cfun))
7655 old_vali = create_tmp_var (TREE_TYPE (loadedi), NULL);
7656 stmt = gimple_build_assign (old_vali, loadedi);
7657 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7659 stmt = gimple_build_assign (loadedi, new_storedi);
7660 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7663 /* Note that we always perform the comparison as an integer, even for
7664 floating point. This allows the atomic operation to properly
7665 succeed even with NaNs and -0.0. */
7666 stmt = gimple_build_cond_empty
7667 (build2 (NE_EXPR, boolean_type_node,
7668 new_storedi, old_vali));
7669 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7672 e = single_succ_edge (store_bb);
7673 e->flags &= ~EDGE_FALLTHRU;
7674 e->flags |= EDGE_FALSE_VALUE;
7676 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
7678 /* Copy the new value to loadedi (we already did that before the condition
7679 if we are not in SSA). */
7680 if (gimple_in_ssa_p (cfun))
7682 phi = gimple_seq_first_stmt (phi_nodes (loop_header));
7683 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
7686 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7687 gsi_remove (&si, true);
7689 struct loop *loop = alloc_loop ();
7690 loop->header = loop_header;
7691 loop->latch = store_bb;
7692 add_loop (loop, loop_header->loop_father);
7694 if (gimple_in_ssa_p (cfun))
7695 update_ssa (TODO_update_ssa_no_phi);
7700 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7702 GOMP_atomic_start ();
7706 The result is not globally atomic, but works so long as all parallel
7707 references are within #pragma omp atomic directives. According to
7708 responses received from omp@openmp.org, appears to be within spec.
7709 Which makes sense, since that's how several other compilers handle
7710 this situation as well.
7711 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7712 expanding. STORED_VAL is the operand of the matching
7713 GIMPLE_OMP_ATOMIC_STORE.
7716 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7720 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7725 expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
7726 tree addr, tree loaded_val, tree stored_val)
7728 gimple_stmt_iterator si;
7732 si = gsi_last_bb (load_bb);
7733 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
7735 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
7736 t = build_call_expr (t, 0);
7737 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
7739 stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr));
7740 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7741 gsi_remove (&si, true);
7743 si = gsi_last_bb (store_bb);
7744 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
7746 stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)),
7748 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7750 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
7751 t = build_call_expr (t, 0);
7752 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
7753 gsi_remove (&si, true);
7755 if (gimple_in_ssa_p (cfun))
7756 update_ssa (TODO_update_ssa_no_phi);
7760 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7761 using expand_omp_atomic_fetch_op. If it failed, we try to
7762 call expand_omp_atomic_pipeline, and if it fails too, the
7763 ultimate fallback is wrapping the operation in a mutex
7764 (expand_omp_atomic_mutex). REGION is the atomic region built
7765 by build_omp_regions_1(). */
7768 expand_omp_atomic (struct omp_region *region)
7770 basic_block load_bb = region->entry, store_bb = region->exit;
7771 gimple load = last_stmt (load_bb), store = last_stmt (store_bb);
7772 tree loaded_val = gimple_omp_atomic_load_lhs (load);
7773 tree addr = gimple_omp_atomic_load_rhs (load);
7774 tree stored_val = gimple_omp_atomic_store_val (store);
7775 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
7776 HOST_WIDE_INT index;
7778 /* Make sure the type is one of the supported sizes. */
7779 index = tree_to_uhwi (TYPE_SIZE_UNIT (type));
7780 index = exact_log2 (index);
7781 if (index >= 0 && index <= 4)
7783 unsigned int align = TYPE_ALIGN_UNIT (type);
7785 /* __sync builtins require strict data alignment. */
7786 if (exact_log2 (align) >= index)
7789 if (loaded_val == stored_val
7790 && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
7791 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
7792 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
7793 && expand_omp_atomic_load (load_bb, addr, loaded_val, index))
7797 if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
7798 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
7799 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
7800 && store_bb == single_succ (load_bb)
7801 && first_stmt (store_bb) == store
7802 && expand_omp_atomic_store (load_bb, addr, loaded_val,
7806 /* When possible, use specialized atomic update functions. */
7807 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
7808 && store_bb == single_succ (load_bb)
7809 && expand_omp_atomic_fetch_op (load_bb, addr,
7810 loaded_val, stored_val, index))
7813 /* If we don't have specialized __sync builtins, try and implement
7814 as a compare and swap loop. */
7815 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
7816 loaded_val, stored_val, index))
7821 /* The ultimate fallback is wrapping the operation in a mutex. */
7822 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
7826 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7829 expand_omp_target (struct omp_region *region)
7831 basic_block entry_bb, exit_bb, new_bb;
7832 struct function *child_cfun = NULL;
7833 tree child_fn = NULL_TREE, block, t;
7834 gimple_stmt_iterator gsi;
7835 gimple entry_stmt, stmt;
7838 entry_stmt = last_stmt (region->entry);
7839 new_bb = region->entry;
7840 int kind = gimple_omp_target_kind (entry_stmt);
7841 if (kind == GF_OMP_TARGET_KIND_REGION)
7843 child_fn = gimple_omp_target_child_fn (entry_stmt);
7844 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
7847 entry_bb = region->entry;
7848 exit_bb = region->exit;
7850 if (kind == GF_OMP_TARGET_KIND_REGION)
7852 unsigned srcidx, dstidx, num;
7854 /* If the target region needs data sent from the parent
7855 function, then the very first statement (except possible
7856 tree profile counter updates) of the parallel body
7857 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7858 &.OMP_DATA_O is passed as an argument to the child function,
7859 we need to replace it with the argument as seen by the child
7862 In most cases, this will end up being the identity assignment
7863 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7864 a function call that has been inlined, the original PARM_DECL
7865 .OMP_DATA_I may have been converted into a different local
7866 variable. In which case, we need to keep the assignment. */
7867 if (gimple_omp_target_data_arg (entry_stmt))
7869 basic_block entry_succ_bb = single_succ (entry_bb);
7870 gimple_stmt_iterator gsi;
7872 gimple tgtcopy_stmt = NULL;
7874 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt), 0);
7876 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
7878 gcc_assert (!gsi_end_p (gsi));
7879 stmt = gsi_stmt (gsi);
7880 if (gimple_code (stmt) != GIMPLE_ASSIGN)
7883 if (gimple_num_ops (stmt) == 2)
7885 tree arg = gimple_assign_rhs1 (stmt);
7887 /* We're ignoring the subcode because we're
7888 effectively doing a STRIP_NOPS. */
7890 if (TREE_CODE (arg) == ADDR_EXPR
7891 && TREE_OPERAND (arg, 0) == sender)
7893 tgtcopy_stmt = stmt;
7899 gcc_assert (tgtcopy_stmt != NULL);
7900 arg = DECL_ARGUMENTS (child_fn);
7902 gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg);
7903 gsi_remove (&gsi, true);
7906 /* Declare local variables needed in CHILD_CFUN. */
7907 block = DECL_INITIAL (child_fn);
7908 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
7909 /* The gimplifier could record temporaries in target block
7910 rather than in containing function's local_decls chain,
7911 which would mean cgraph missed finalizing them. Do it now. */
7912 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
7913 if (TREE_CODE (t) == VAR_DECL
7915 && !DECL_EXTERNAL (t))
7916 varpool_finalize_decl (t);
7917 DECL_SAVED_TREE (child_fn) = NULL;
7918 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7919 gimple_set_body (child_fn, NULL);
7920 TREE_USED (block) = 1;
7922 /* Reset DECL_CONTEXT on function arguments. */
7923 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
7924 DECL_CONTEXT (t) = child_fn;
7926 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7927 so that it can be moved to the child function. */
7928 gsi = gsi_last_bb (entry_bb);
7929 stmt = gsi_stmt (gsi);
7930 gcc_assert (stmt && gimple_code (stmt) == GIMPLE_OMP_TARGET
7931 && gimple_omp_target_kind (stmt)
7932 == GF_OMP_TARGET_KIND_REGION);
7933 gsi_remove (&gsi, true);
7934 e = split_block (entry_bb, stmt);
7936 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7938 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7941 gsi = gsi_last_bb (exit_bb);
7942 gcc_assert (!gsi_end_p (gsi)
7943 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
7944 stmt = gimple_build_return (NULL);
7945 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
7946 gsi_remove (&gsi, true);
7949 /* Move the target region into CHILD_CFUN. */
7951 block = gimple_block (entry_stmt);
7953 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
7955 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
7956 /* When the OMP expansion process cannot guarantee an up-to-date
7957 loop tree arrange for the child function to fixup loops. */
7958 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
7959 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
7961 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7962 num = vec_safe_length (child_cfun->local_decls);
7963 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
7965 t = (*child_cfun->local_decls)[srcidx];
7966 if (DECL_CONTEXT (t) == cfun->decl)
7968 if (srcidx != dstidx)
7969 (*child_cfun->local_decls)[dstidx] = t;
7973 vec_safe_truncate (child_cfun->local_decls, dstidx);
7975 /* Inform the callgraph about the new function. */
7976 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
7977 cgraph_add_new_function (child_fn, true);
7979 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7980 fixed in a following pass. */
7981 push_cfun (child_cfun);
7982 rebuild_cgraph_edges ();
7984 /* Some EH regions might become dead, see PR34608. If
7985 pass_cleanup_cfg isn't the first pass to happen with the
7986 new child, these dead EH edges might cause problems.
7987 Clean them up now. */
7988 if (flag_exceptions)
7991 bool changed = false;
7993 FOR_EACH_BB_FN (bb, cfun)
7994 changed |= gimple_purge_dead_eh_edges (bb);
7996 cleanup_tree_cfg ();
8001 /* Emit a library call to launch the target region, or do data
8003 tree t1, t2, t3, t4, device, cond, c, clauses;
8004 enum built_in_function start_ix;
8005 location_t clause_loc;
8007 clauses = gimple_omp_target_clauses (entry_stmt);
8009 if (kind == GF_OMP_TARGET_KIND_REGION)
8010 start_ix = BUILT_IN_GOMP_TARGET;
8011 else if (kind == GF_OMP_TARGET_KIND_DATA)
8012 start_ix = BUILT_IN_GOMP_TARGET_DATA;
8014 start_ix = BUILT_IN_GOMP_TARGET_UPDATE;
8016 /* By default, the value of DEVICE is -1 (let runtime library choose)
8017 and there is no conditional. */
8019 device = build_int_cst (integer_type_node, -1);
8021 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
8023 cond = OMP_CLAUSE_IF_EXPR (c);
8025 c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE);
8028 device = OMP_CLAUSE_DEVICE_ID (c);
8029 clause_loc = OMP_CLAUSE_LOCATION (c);
8032 clause_loc = gimple_location (entry_stmt);
8034 /* Ensure 'device' is of the correct type. */
8035 device = fold_convert_loc (clause_loc, integer_type_node, device);
8037 /* If we found the clause 'if (cond)', build
8038 (cond ? device : -2). */
8041 cond = gimple_boolify (cond);
8043 basic_block cond_bb, then_bb, else_bb;
8047 tmp_var = create_tmp_var (TREE_TYPE (device), NULL);
8048 if (kind != GF_OMP_TARGET_KIND_REGION)
8050 gsi = gsi_last_bb (new_bb);
8052 e = split_block (new_bb, gsi_stmt (gsi));
8055 e = split_block (new_bb, NULL);
8060 then_bb = create_empty_bb (cond_bb);
8061 else_bb = create_empty_bb (then_bb);
8062 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
8063 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
8065 stmt = gimple_build_cond_empty (cond);
8066 gsi = gsi_last_bb (cond_bb);
8067 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
8069 gsi = gsi_start_bb (then_bb);
8070 stmt = gimple_build_assign (tmp_var, device);
8071 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
8073 gsi = gsi_start_bb (else_bb);
8074 stmt = gimple_build_assign (tmp_var,
8075 build_int_cst (integer_type_node, -2));
8076 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
8078 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
8079 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
8080 add_bb_to_loop (then_bb, cond_bb->loop_father);
8081 add_bb_to_loop (else_bb, cond_bb->loop_father);
8082 make_edge (then_bb, new_bb, EDGE_FALLTHRU);
8083 make_edge (else_bb, new_bb, EDGE_FALLTHRU);
8088 gsi = gsi_last_bb (new_bb);
8089 t = gimple_omp_target_data_arg (entry_stmt);
8092 t1 = size_zero_node;
8093 t2 = build_zero_cst (ptr_type_node);
8099 t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1))));
8100 t1 = size_binop (PLUS_EXPR, t1, size_int (1));
8101 t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0));
8102 t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1));
8103 t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2));
8107 /* FIXME: This will be address of
8108 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8109 symbol, as soon as the linker plugin is able to create it for us. */
8110 tree openmp_target = build_zero_cst (ptr_type_node);
8111 if (kind == GF_OMP_TARGET_KIND_REGION)
8113 tree fnaddr = build_fold_addr_expr (child_fn);
8114 g = gimple_build_call (builtin_decl_explicit (start_ix), 7,
8115 device, fnaddr, openmp_target, t1, t2, t3, t4);
8118 g = gimple_build_call (builtin_decl_explicit (start_ix), 6,
8119 device, openmp_target, t1, t2, t3, t4);
8120 gimple_set_location (g, gimple_location (entry_stmt));
8121 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
8122 if (kind != GF_OMP_TARGET_KIND_REGION)
8125 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET);
8126 gsi_remove (&gsi, true);
8128 if (kind == GF_OMP_TARGET_KIND_DATA && region->exit)
8130 gsi = gsi_last_bb (region->exit);
8132 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN);
8133 gsi_remove (&gsi, true);
8138 /* Expand the parallel region tree rooted at REGION. Expansion
8139 proceeds in depth-first order. Innermost regions are expanded
8140 first. This way, parallel regions that require a new function to
8141 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8142 internal dependencies in their body. */
8145 expand_omp (struct omp_region *region)
8149 location_t saved_location;
8150 gimple inner_stmt = NULL;
8152 /* First, determine whether this is a combined parallel+workshare
8154 if (region->type == GIMPLE_OMP_PARALLEL)
8155 determine_parallel_type (region);
8157 if (region->type == GIMPLE_OMP_FOR
8158 && gimple_omp_for_combined_p (last_stmt (region->entry)))
8159 inner_stmt = last_stmt (region->inner->entry);
8162 expand_omp (region->inner);
8164 saved_location = input_location;
8165 if (gimple_has_location (last_stmt (region->entry)))
8166 input_location = gimple_location (last_stmt (region->entry));
8168 switch (region->type)
8170 case GIMPLE_OMP_PARALLEL:
8171 case GIMPLE_OMP_TASK:
8172 expand_omp_taskreg (region);
8175 case GIMPLE_OMP_FOR:
8176 expand_omp_for (region, inner_stmt);
8179 case GIMPLE_OMP_SECTIONS:
8180 expand_omp_sections (region);
8183 case GIMPLE_OMP_SECTION:
8184 /* Individual omp sections are handled together with their
8185 parent GIMPLE_OMP_SECTIONS region. */
8188 case GIMPLE_OMP_SINGLE:
8189 expand_omp_single (region);
8192 case GIMPLE_OMP_MASTER:
8193 case GIMPLE_OMP_TASKGROUP:
8194 case GIMPLE_OMP_ORDERED:
8195 case GIMPLE_OMP_CRITICAL:
8196 case GIMPLE_OMP_TEAMS:
8197 expand_omp_synch (region);
8200 case GIMPLE_OMP_ATOMIC_LOAD:
8201 expand_omp_atomic (region);
8204 case GIMPLE_OMP_TARGET:
8205 expand_omp_target (region);
8212 input_location = saved_location;
8213 region = region->next;
8218 /* Helper for build_omp_regions. Scan the dominator tree starting at
8219 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8220 true, the function ends once a single tree is built (otherwise, whole
8221 forest of OMP constructs may be built). */
8224 build_omp_regions_1 (basic_block bb, struct omp_region *parent,
8227 gimple_stmt_iterator gsi;
8231 gsi = gsi_last_bb (bb);
8232 if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi)))
8234 struct omp_region *region;
8235 enum gimple_code code;
8237 stmt = gsi_stmt (gsi);
8238 code = gimple_code (stmt);
8239 if (code == GIMPLE_OMP_RETURN)
8241 /* STMT is the return point out of region PARENT. Mark it
8242 as the exit point and make PARENT the immediately
8243 enclosing region. */
8244 gcc_assert (parent);
8247 parent = parent->outer;
8249 else if (code == GIMPLE_OMP_ATOMIC_STORE)
8251 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8252 GIMPLE_OMP_RETURN, but matches with
8253 GIMPLE_OMP_ATOMIC_LOAD. */
8254 gcc_assert (parent);
8255 gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD);
8258 parent = parent->outer;
8261 else if (code == GIMPLE_OMP_CONTINUE)
8263 gcc_assert (parent);
8266 else if (code == GIMPLE_OMP_SECTIONS_SWITCH)
8268 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8269 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8272 else if (code == GIMPLE_OMP_TARGET
8273 && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_UPDATE)
8274 new_omp_region (bb, code, parent);
8277 /* Otherwise, this directive becomes the parent for a new
8279 region = new_omp_region (bb, code, parent);
8284 if (single_tree && !parent)
8287 for (son = first_dom_son (CDI_DOMINATORS, bb);
8289 son = next_dom_son (CDI_DOMINATORS, son))
8290 build_omp_regions_1 (son, parent, single_tree);
8293 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8297 build_omp_regions_root (basic_block root)
8299 gcc_assert (root_omp_region == NULL);
8300 build_omp_regions_1 (root, NULL, true);
8301 gcc_assert (root_omp_region != NULL);
8304 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8307 omp_expand_local (basic_block head)
8309 build_omp_regions_root (head);
8310 if (dump_file && (dump_flags & TDF_DETAILS))
8312 fprintf (dump_file, "\nOMP region tree\n\n");
8313 dump_omp_region (dump_file, root_omp_region, 0);
8314 fprintf (dump_file, "\n");
8317 remove_exit_barriers (root_omp_region);
8318 expand_omp (root_omp_region);
8320 free_omp_regions ();
8323 /* Scan the CFG and build a tree of OMP regions. Return the root of
8324 the OMP region tree. */
8327 build_omp_regions (void)
8329 gcc_assert (root_omp_region == NULL);
8330 calculate_dominance_info (CDI_DOMINATORS);
8331 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false);
8334 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8337 execute_expand_omp (void)
8339 build_omp_regions ();
8341 if (!root_omp_region)
8346 fprintf (dump_file, "\nOMP region tree\n\n");
8347 dump_omp_region (dump_file, root_omp_region, 0);
8348 fprintf (dump_file, "\n");
8351 remove_exit_barriers (root_omp_region);
8353 expand_omp (root_omp_region);
8355 cleanup_tree_cfg ();
8357 free_omp_regions ();
8362 /* OMP expansion -- the default pass, run before creation of SSA form. */
8366 const pass_data pass_data_expand_omp =
8368 GIMPLE_PASS, /* type */
8369 "ompexp", /* name */
8370 OPTGROUP_NONE, /* optinfo_flags */
8371 true, /* has_execute */
8372 TV_NONE, /* tv_id */
8373 PROP_gimple_any, /* properties_required */
8374 0, /* properties_provided */
8375 0, /* properties_destroyed */
8376 0, /* todo_flags_start */
8377 0, /* todo_flags_finish */
8380 class pass_expand_omp : public gimple_opt_pass
8383 pass_expand_omp (gcc::context *ctxt)
8384 : gimple_opt_pass (pass_data_expand_omp, ctxt)
8387 /* opt_pass methods: */
8388 virtual bool gate (function *)
8390 return ((flag_openmp != 0 || flag_openmp_simd != 0
8391 || flag_cilkplus != 0) && !seen_error ());
8394 virtual unsigned int execute (function *) { return execute_expand_omp (); }
8396 }; // class pass_expand_omp
8401 make_pass_expand_omp (gcc::context *ctxt)
8403 return new pass_expand_omp (ctxt);
8406 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8408 /* If ctx is a worksharing context inside of a cancellable parallel
8409 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8410 and conditional branch to parallel's cancel_label to handle
8411 cancellation in the implicit barrier. */
8414 maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body)
8416 gimple omp_return = gimple_seq_last_stmt (*body);
8417 gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
8418 if (gimple_omp_return_nowait_p (omp_return))
8421 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL
8422 && ctx->outer->cancellable)
8424 tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
8425 tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
8426 tree lhs = create_tmp_var (c_bool_type, NULL);
8427 gimple_omp_return_set_lhs (omp_return, lhs);
8428 tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
8429 gimple g = gimple_build_cond (NE_EXPR, lhs,
8430 fold_convert (c_bool_type,
8431 boolean_false_node),
8432 ctx->outer->cancel_label, fallthru_label);
8433 gimple_seq_add_stmt (body, g);
8434 gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
8438 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8439 CTX is the enclosing OMP context for the current statement. */
8442 lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8444 tree block, control;
8445 gimple_stmt_iterator tgsi;
8446 gimple stmt, new_stmt, bind, t;
8447 gimple_seq ilist, dlist, olist, new_body;
8449 stmt = gsi_stmt (*gsi_p);
8451 push_gimplify_context ();
8455 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
8456 &ilist, &dlist, ctx, NULL);
8458 new_body = gimple_omp_body (stmt);
8459 gimple_omp_set_body (stmt, NULL);
8460 tgsi = gsi_start (new_body);
8461 for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
8466 sec_start = gsi_stmt (tgsi);
8467 sctx = maybe_lookup_ctx (sec_start);
8470 lower_omp (gimple_omp_body_ptr (sec_start), sctx);
8471 gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
8472 GSI_CONTINUE_LINKING);
8473 gimple_omp_set_body (sec_start, NULL);
8475 if (gsi_one_before_end_p (tgsi))
8477 gimple_seq l = NULL;
8478 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
8480 gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
8481 gimple_omp_section_set_last (sec_start);
8484 gsi_insert_after (&tgsi, gimple_build_omp_return (false),
8485 GSI_CONTINUE_LINKING);
8488 block = make_node (BLOCK);
8489 bind = gimple_build_bind (NULL, new_body, block);
8492 lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx);
8494 block = make_node (BLOCK);
8495 new_stmt = gimple_build_bind (NULL, NULL, block);
8496 gsi_replace (gsi_p, new_stmt, true);
8498 pop_gimplify_context (new_stmt);
8499 gimple_bind_append_vars (new_stmt, ctx->block_vars);
8500 BLOCK_VARS (block) = gimple_bind_vars (bind);
8501 if (BLOCK_VARS (block))
8502 TREE_USED (block) = 1;
8505 gimple_seq_add_seq (&new_body, ilist);
8506 gimple_seq_add_stmt (&new_body, stmt);
8507 gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
8508 gimple_seq_add_stmt (&new_body, bind);
8510 control = create_tmp_var (unsigned_type_node, ".section");
8511 t = gimple_build_omp_continue (control, control);
8512 gimple_omp_sections_set_control (stmt, control);
8513 gimple_seq_add_stmt (&new_body, t);
8515 gimple_seq_add_seq (&new_body, olist);
8516 if (ctx->cancellable)
8517 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
8518 gimple_seq_add_seq (&new_body, dlist);
8520 new_body = maybe_catch_exception (new_body);
8522 t = gimple_build_omp_return
8523 (!!find_omp_clause (gimple_omp_sections_clauses (stmt),
8524 OMP_CLAUSE_NOWAIT));
8525 gimple_seq_add_stmt (&new_body, t);
8526 maybe_add_implicit_barrier_cancel (ctx, &new_body);
8528 gimple_bind_set_body (new_stmt, new_body);
8532 /* A subroutine of lower_omp_single. Expand the simple form of
8533 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8535 if (GOMP_single_start ())
8537 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8539 FIXME. It may be better to delay expanding the logic of this until
8540 pass_expand_omp. The expanded logic may make the job more difficult
8541 to a synchronization analysis pass. */
8544 lower_omp_single_simple (gimple single_stmt, gimple_seq *pre_p)
8546 location_t loc = gimple_location (single_stmt);
8547 tree tlabel = create_artificial_label (loc);
8548 tree flabel = create_artificial_label (loc);
8552 decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START);
8553 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)), NULL);
8554 call = gimple_build_call (decl, 0);
8555 gimple_call_set_lhs (call, lhs);
8556 gimple_seq_add_stmt (pre_p, call);
8558 cond = gimple_build_cond (EQ_EXPR, lhs,
8559 fold_convert_loc (loc, TREE_TYPE (lhs),
8562 gimple_seq_add_stmt (pre_p, cond);
8563 gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
8564 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8565 gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
8569 /* A subroutine of lower_omp_single. Expand the simple form of
8570 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8572 #pragma omp single copyprivate (a, b, c)
8574 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8577 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8583 GOMP_single_copy_end (©out);
8594 FIXME. It may be better to delay expanding the logic of this until
8595 pass_expand_omp. The expanded logic may make the job more difficult
8596 to a synchronization analysis pass. */
8599 lower_omp_single_copy (gimple single_stmt, gimple_seq *pre_p, omp_context *ctx)
8601 tree ptr_type, t, l0, l1, l2, bfn_decl;
8602 gimple_seq copyin_seq;
8603 location_t loc = gimple_location (single_stmt);
8605 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
8607 ptr_type = build_pointer_type (ctx->record_type);
8608 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
8610 l0 = create_artificial_label (loc);
8611 l1 = create_artificial_label (loc);
8612 l2 = create_artificial_label (loc);
8614 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
8615 t = build_call_expr_loc (loc, bfn_decl, 0);
8616 t = fold_convert_loc (loc, ptr_type, t);
8617 gimplify_assign (ctx->receiver_decl, t, pre_p);
8619 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
8620 build_int_cst (ptr_type, 0));
8621 t = build3 (COND_EXPR, void_type_node, t,
8622 build_and_jump (&l0), build_and_jump (&l1));
8623 gimplify_and_add (t, pre_p);
8625 gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
8627 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8630 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
8633 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
8634 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
8635 t = build_call_expr_loc (loc, bfn_decl, 1, t);
8636 gimplify_and_add (t, pre_p);
8638 t = build_and_jump (&l2);
8639 gimplify_and_add (t, pre_p);
8641 gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
8643 gimple_seq_add_seq (pre_p, copyin_seq);
8645 gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
8649 /* Expand code for an OpenMP single directive. */
8652 lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8655 gimple t, bind, single_stmt = gsi_stmt (*gsi_p);
8656 gimple_seq bind_body, bind_body_tail = NULL, dlist;
8658 push_gimplify_context ();
8660 block = make_node (BLOCK);
8661 bind = gimple_build_bind (NULL, NULL, block);
8662 gsi_replace (gsi_p, bind, true);
8665 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
8666 &bind_body, &dlist, ctx, NULL);
8667 lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
8669 gimple_seq_add_stmt (&bind_body, single_stmt);
8671 if (ctx->record_type)
8672 lower_omp_single_copy (single_stmt, &bind_body, ctx);
8674 lower_omp_single_simple (single_stmt, &bind_body);
8676 gimple_omp_set_body (single_stmt, NULL);
8678 gimple_seq_add_seq (&bind_body, dlist);
8680 bind_body = maybe_catch_exception (bind_body);
8682 t = gimple_build_omp_return
8683 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt),
8684 OMP_CLAUSE_NOWAIT));
8685 gimple_seq_add_stmt (&bind_body_tail, t);
8686 maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail);
8687 if (ctx->record_type)
8689 gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
8690 tree clobber = build_constructor (ctx->record_type, NULL);
8691 TREE_THIS_VOLATILE (clobber) = 1;
8692 gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
8693 clobber), GSI_SAME_STMT);
8695 gimple_seq_add_seq (&bind_body, bind_body_tail);
8696 gimple_bind_set_body (bind, bind_body);
8698 pop_gimplify_context (bind);
8700 gimple_bind_append_vars (bind, ctx->block_vars);
8701 BLOCK_VARS (block) = ctx->block_vars;
8702 if (BLOCK_VARS (block))
8703 TREE_USED (block) = 1;
8707 /* Expand code for an OpenMP master directive. */
8710 lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8712 tree block, lab = NULL, x, bfn_decl;
8713 gimple stmt = gsi_stmt (*gsi_p), bind;
8714 location_t loc = gimple_location (stmt);
8717 push_gimplify_context ();
8719 block = make_node (BLOCK);
8720 bind = gimple_build_bind (NULL, NULL, block);
8721 gsi_replace (gsi_p, bind, true);
8722 gimple_bind_add_stmt (bind, stmt);
8724 bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
8725 x = build_call_expr_loc (loc, bfn_decl, 0);
8726 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node);
8727 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
8729 gimplify_and_add (x, &tseq);
8730 gimple_bind_add_seq (bind, tseq);
8732 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8733 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
8734 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8735 gimple_omp_set_body (stmt, NULL);
8737 gimple_bind_add_stmt (bind, gimple_build_label (lab));
8739 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8741 pop_gimplify_context (bind);
8743 gimple_bind_append_vars (bind, ctx->block_vars);
8744 BLOCK_VARS (block) = ctx->block_vars;
8748 /* Expand code for an OpenMP taskgroup directive. */
8751 lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8753 gimple stmt = gsi_stmt (*gsi_p), bind, x;
8754 tree block = make_node (BLOCK);
8756 bind = gimple_build_bind (NULL, NULL, block);
8757 gsi_replace (gsi_p, bind, true);
8758 gimple_bind_add_stmt (bind, stmt);
8760 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
8762 gimple_bind_add_stmt (bind, x);
8764 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8765 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8766 gimple_omp_set_body (stmt, NULL);
8768 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8770 gimple_bind_append_vars (bind, ctx->block_vars);
8771 BLOCK_VARS (block) = ctx->block_vars;
8775 /* Expand code for an OpenMP ordered directive. */
8778 lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8781 gimple stmt = gsi_stmt (*gsi_p), bind, x;
8783 push_gimplify_context ();
8785 block = make_node (BLOCK);
8786 bind = gimple_build_bind (NULL, NULL, block);
8787 gsi_replace (gsi_p, bind, true);
8788 gimple_bind_add_stmt (bind, stmt);
8790 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
8792 gimple_bind_add_stmt (bind, x);
8794 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8795 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
8796 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8797 gimple_omp_set_body (stmt, NULL);
8799 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END), 0);
8800 gimple_bind_add_stmt (bind, x);
8802 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8804 pop_gimplify_context (bind);
8806 gimple_bind_append_vars (bind, ctx->block_vars);
8807 BLOCK_VARS (block) = gimple_bind_vars (bind);
8811 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8812 substitution of a couple of function calls. But in the NAMED case,
8813 requires that languages coordinate a symbol name. It is therefore
8814 best put here in common code. */
8816 static GTY((param1_is (tree), param2_is (tree)))
8817 splay_tree critical_name_mutexes;
8820 lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8823 tree name, lock, unlock;
8824 gimple stmt = gsi_stmt (*gsi_p), bind;
8825 location_t loc = gimple_location (stmt);
8828 name = gimple_omp_critical_name (stmt);
8834 if (!critical_name_mutexes)
8835 critical_name_mutexes
8836 = splay_tree_new_ggc (splay_tree_compare_pointers,
8837 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s,
8838 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s);
8840 n = splay_tree_lookup (critical_name_mutexes, (splay_tree_key) name);
8845 decl = create_tmp_var_raw (ptr_type_node, NULL);
8847 new_str = ACONCAT ((".gomp_critical_user_",
8848 IDENTIFIER_POINTER (name), NULL));
8849 DECL_NAME (decl) = get_identifier (new_str);
8850 TREE_PUBLIC (decl) = 1;
8851 TREE_STATIC (decl) = 1;
8852 DECL_COMMON (decl) = 1;
8853 DECL_ARTIFICIAL (decl) = 1;
8854 DECL_IGNORED_P (decl) = 1;
8855 varpool_finalize_decl (decl);
8857 splay_tree_insert (critical_name_mutexes, (splay_tree_key) name,
8858 (splay_tree_value) decl);
8861 decl = (tree) n->value;
8863 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
8864 lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl));
8866 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
8867 unlock = build_call_expr_loc (loc, unlock, 1,
8868 build_fold_addr_expr_loc (loc, decl));
8872 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
8873 lock = build_call_expr_loc (loc, lock, 0);
8875 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
8876 unlock = build_call_expr_loc (loc, unlock, 0);
8879 push_gimplify_context ();
8881 block = make_node (BLOCK);
8882 bind = gimple_build_bind (NULL, NULL, block);
8883 gsi_replace (gsi_p, bind, true);
8884 gimple_bind_add_stmt (bind, stmt);
8886 tbody = gimple_bind_body (bind);
8887 gimplify_and_add (lock, &tbody);
8888 gimple_bind_set_body (bind, tbody);
8890 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8891 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
8892 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8893 gimple_omp_set_body (stmt, NULL);
8895 tbody = gimple_bind_body (bind);
8896 gimplify_and_add (unlock, &tbody);
8897 gimple_bind_set_body (bind, tbody);
8899 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8901 pop_gimplify_context (bind);
8902 gimple_bind_append_vars (bind, ctx->block_vars);
8903 BLOCK_VARS (block) = gimple_bind_vars (bind);
8907 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8908 for a lastprivate clause. Given a loop control predicate of (V
8909 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8910 is appended to *DLIST, iterator initialization is appended to
8914 lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
8915 gimple_seq *dlist, struct omp_context *ctx)
8917 tree clauses, cond, vinit;
8918 enum tree_code cond_code;
8921 cond_code = fd->loop.cond_code;
8922 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
8924 /* When possible, use a strict equality expression. This can let VRP
8925 type optimizations deduce the value and remove a copy. */
8926 if (tree_fits_shwi_p (fd->loop.step))
8928 HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
8929 if (step == 1 || step == -1)
8930 cond_code = EQ_EXPR;
8933 cond = build2 (cond_code, boolean_type_node, fd->loop.v, fd->loop.n2);
8935 clauses = gimple_omp_for_clauses (fd->for_stmt);
8937 lower_lastprivate_clauses (clauses, cond, &stmts, ctx);
8938 if (!gimple_seq_empty_p (stmts))
8940 gimple_seq_add_seq (&stmts, *dlist);
8943 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8944 vinit = fd->loop.n1;
8945 if (cond_code == EQ_EXPR
8946 && tree_fits_shwi_p (fd->loop.n2)
8947 && ! integer_zerop (fd->loop.n2))
8948 vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
8950 vinit = unshare_expr (vinit);
8952 /* Initialize the iterator variable, so that threads that don't execute
8953 any iterations don't execute the lastprivate clauses by accident. */
8954 gimplify_assign (fd->loop.v, vinit, body_p);
8959 /* Lower code for an OpenMP loop directive. */
8962 lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8965 struct omp_for_data fd, *fdp = NULL;
8966 gimple stmt = gsi_stmt (*gsi_p), new_stmt;
8967 gimple_seq omp_for_body, body, dlist;
8970 push_gimplify_context ();
8972 lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
8974 block = make_node (BLOCK);
8975 new_stmt = gimple_build_bind (NULL, NULL, block);
8976 /* Replace at gsi right away, so that 'stmt' is no member
8977 of a sequence anymore as we're going to add to to a different
8979 gsi_replace (gsi_p, new_stmt, true);
8981 /* Move declaration of temporaries in the loop body before we make
8983 omp_for_body = gimple_omp_body (stmt);
8984 if (!gimple_seq_empty_p (omp_for_body)
8985 && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
8987 gimple inner_bind = gimple_seq_first_stmt (omp_for_body);
8988 tree vars = gimple_bind_vars (inner_bind);
8989 gimple_bind_append_vars (new_stmt, vars);
8990 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
8991 keep them on the inner_bind and it's block. */
8992 gimple_bind_set_vars (inner_bind, NULL_TREE);
8993 if (gimple_bind_block (inner_bind))
8994 BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
8997 if (gimple_omp_for_combined_into_p (stmt))
8999 extract_omp_for_data (stmt, &fd, NULL);
9002 /* We need two temporaries with fd.loop.v type (istart/iend)
9003 and then (fd.collapse - 1) temporaries with the same
9004 type for count2 ... countN-1 vars if not constant. */
9006 tree type = fd.iter_type;
9008 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
9009 count += fd.collapse - 1;
9010 bool parallel_for = gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR;
9011 tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
9015 = find_omp_clause (gimple_omp_parallel_clauses (ctx->outer->stmt),
9016 OMP_CLAUSE__LOOPTEMP_);
9017 for (i = 0; i < count; i++)
9022 gcc_assert (outerc);
9023 temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
9024 outerc = find_omp_clause (OMP_CLAUSE_CHAIN (outerc),
9025 OMP_CLAUSE__LOOPTEMP_);
9028 temp = create_tmp_var (type, NULL);
9029 *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
9030 OMP_CLAUSE_DECL (*pc) = temp;
9031 pc = &OMP_CLAUSE_CHAIN (*pc);
9036 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9039 lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
9041 gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt));
9043 lower_omp (gimple_omp_body_ptr (stmt), ctx);
9045 /* Lower the header expressions. At this point, we can assume that
9046 the header is of the form:
9048 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9050 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9051 using the .omp_data_s mapping, if needed. */
9052 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
9054 rhs_p = gimple_omp_for_initial_ptr (stmt, i);
9055 if (!is_gimple_min_invariant (*rhs_p))
9056 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
9058 rhs_p = gimple_omp_for_final_ptr (stmt, i);
9059 if (!is_gimple_min_invariant (*rhs_p))
9060 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
9062 rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
9063 if (!is_gimple_min_invariant (*rhs_p))
9064 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
9067 /* Once lowered, extract the bounds and clauses. */
9068 extract_omp_for_data (stmt, &fd, NULL);
9070 lower_omp_for_lastprivate (&fd, &body, &dlist, ctx);
9072 gimple_seq_add_stmt (&body, stmt);
9073 gimple_seq_add_seq (&body, gimple_omp_body (stmt));
9075 gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
9078 /* After the loop, add exit clauses. */
9079 lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx);
9081 if (ctx->cancellable)
9082 gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
9084 gimple_seq_add_seq (&body, dlist);
9086 body = maybe_catch_exception (body);
9088 /* Region exit marker goes at the end of the loop body. */
9089 gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait));
9090 maybe_add_implicit_barrier_cancel (ctx, &body);
9091 pop_gimplify_context (new_stmt);
9093 gimple_bind_append_vars (new_stmt, ctx->block_vars);
9094 BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
9095 if (BLOCK_VARS (block))
9096 TREE_USED (block) = 1;
9098 gimple_bind_set_body (new_stmt, body);
9099 gimple_omp_set_body (stmt, NULL);
9100 gimple_omp_for_set_pre_body (stmt, NULL);
9103 /* Callback for walk_stmts. Check if the current statement only contains
9104 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9107 check_combined_parallel (gimple_stmt_iterator *gsi_p,
9108 bool *handled_ops_p,
9109 struct walk_stmt_info *wi)
9111 int *info = (int *) wi->info;
9112 gimple stmt = gsi_stmt (*gsi_p);
9114 *handled_ops_p = true;
9115 switch (gimple_code (stmt))
9119 case GIMPLE_OMP_FOR:
9120 case GIMPLE_OMP_SECTIONS:
9121 *info = *info == 0 ? 1 : -1;
9130 struct omp_taskcopy_context
9132 /* This field must be at the beginning, as we do "inheritance": Some
9133 callback functions for tree-inline.c (e.g., omp_copy_decl)
9134 receive a copy_body_data pointer that is up-casted to an
9135 omp_context pointer. */
9141 task_copyfn_copy_decl (tree var, copy_body_data *cb)
9143 struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
9145 if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
9146 return create_tmp_var (TREE_TYPE (var), NULL);
9152 task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
9154 tree name, new_fields = NULL, type, f;
9156 type = lang_hooks.types.make_type (RECORD_TYPE);
9157 name = DECL_NAME (TYPE_NAME (orig_type));
9158 name = build_decl (gimple_location (tcctx->ctx->stmt),
9159 TYPE_DECL, name, type);
9160 TYPE_NAME (type) = name;
9162 for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
9164 tree new_f = copy_node (f);
9165 DECL_CONTEXT (new_f) = type;
9166 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
9167 TREE_CHAIN (new_f) = new_fields;
9168 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
9169 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
9170 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
9173 *pointer_map_insert (tcctx->cb.decl_map, f) = new_f;
9175 TYPE_FIELDS (type) = nreverse (new_fields);
9180 /* Create task copyfn. */
9183 create_task_copyfn (gimple task_stmt, omp_context *ctx)
9185 struct function *child_cfun;
9186 tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
9187 tree record_type, srecord_type, bind, list;
9188 bool record_needs_remap = false, srecord_needs_remap = false;
9190 struct omp_taskcopy_context tcctx;
9191 location_t loc = gimple_location (task_stmt);
9193 child_fn = gimple_omp_task_copy_fn (task_stmt);
9194 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
9195 gcc_assert (child_cfun->cfg == NULL);
9196 DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
9198 /* Reset DECL_CONTEXT on function arguments. */
9199 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
9200 DECL_CONTEXT (t) = child_fn;
9202 /* Populate the function. */
9203 push_gimplify_context ();
9204 push_cfun (child_cfun);
9206 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
9207 TREE_SIDE_EFFECTS (bind) = 1;
9209 DECL_SAVED_TREE (child_fn) = bind;
9210 DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
9212 /* Remap src and dst argument types if needed. */
9213 record_type = ctx->record_type;
9214 srecord_type = ctx->srecord_type;
9215 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
9216 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
9218 record_needs_remap = true;
9221 for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
9222 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
9224 srecord_needs_remap = true;
9228 if (record_needs_remap || srecord_needs_remap)
9230 memset (&tcctx, '\0', sizeof (tcctx));
9231 tcctx.cb.src_fn = ctx->cb.src_fn;
9232 tcctx.cb.dst_fn = child_fn;
9233 tcctx.cb.src_node = cgraph_get_node (tcctx.cb.src_fn);
9234 gcc_checking_assert (tcctx.cb.src_node);
9235 tcctx.cb.dst_node = tcctx.cb.src_node;
9236 tcctx.cb.src_cfun = ctx->cb.src_cfun;
9237 tcctx.cb.copy_decl = task_copyfn_copy_decl;
9238 tcctx.cb.eh_lp_nr = 0;
9239 tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
9240 tcctx.cb.decl_map = pointer_map_create ();
9243 if (record_needs_remap)
9244 record_type = task_copyfn_remap_type (&tcctx, record_type);
9245 if (srecord_needs_remap)
9246 srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
9249 tcctx.cb.decl_map = NULL;
9251 arg = DECL_ARGUMENTS (child_fn);
9252 TREE_TYPE (arg) = build_pointer_type (record_type);
9253 sarg = DECL_CHAIN (arg);
9254 TREE_TYPE (sarg) = build_pointer_type (srecord_type);
9256 /* First pass: initialize temporaries used in record_type and srecord_type
9257 sizes and field offsets. */
9258 if (tcctx.cb.decl_map)
9259 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
9260 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
9264 decl = OMP_CLAUSE_DECL (c);
9265 p = (tree *) pointer_map_contains (tcctx.cb.decl_map, decl);
9268 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9269 sf = (tree) n->value;
9270 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9271 src = build_simple_mem_ref_loc (loc, sarg);
9272 src = omp_build_component_ref (src, sf);
9273 t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
9274 append_to_statement_list (t, &list);
9277 /* Second pass: copy shared var pointers and copy construct non-VLA
9278 firstprivate vars. */
9279 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
9280 switch (OMP_CLAUSE_CODE (c))
9282 case OMP_CLAUSE_SHARED:
9283 decl = OMP_CLAUSE_DECL (c);
9284 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9287 f = (tree) n->value;
9288 if (tcctx.cb.decl_map)
9289 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9290 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9291 sf = (tree) n->value;
9292 if (tcctx.cb.decl_map)
9293 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9294 src = build_simple_mem_ref_loc (loc, sarg);
9295 src = omp_build_component_ref (src, sf);
9296 dst = build_simple_mem_ref_loc (loc, arg);
9297 dst = omp_build_component_ref (dst, f);
9298 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
9299 append_to_statement_list (t, &list);
9301 case OMP_CLAUSE_FIRSTPRIVATE:
9302 decl = OMP_CLAUSE_DECL (c);
9303 if (is_variable_sized (decl))
9305 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9308 f = (tree) n->value;
9309 if (tcctx.cb.decl_map)
9310 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9311 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9314 sf = (tree) n->value;
9315 if (tcctx.cb.decl_map)
9316 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9317 src = build_simple_mem_ref_loc (loc, sarg);
9318 src = omp_build_component_ref (src, sf);
9319 if (use_pointer_for_field (decl, NULL) || is_reference (decl))
9320 src = build_simple_mem_ref_loc (loc, src);
9324 dst = build_simple_mem_ref_loc (loc, arg);
9325 dst = omp_build_component_ref (dst, f);
9326 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
9327 append_to_statement_list (t, &list);
9329 case OMP_CLAUSE_PRIVATE:
9330 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
9332 decl = OMP_CLAUSE_DECL (c);
9333 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9334 f = (tree) n->value;
9335 if (tcctx.cb.decl_map)
9336 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9337 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9340 sf = (tree) n->value;
9341 if (tcctx.cb.decl_map)
9342 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9343 src = build_simple_mem_ref_loc (loc, sarg);
9344 src = omp_build_component_ref (src, sf);
9345 if (use_pointer_for_field (decl, NULL))
9346 src = build_simple_mem_ref_loc (loc, src);
9350 dst = build_simple_mem_ref_loc (loc, arg);
9351 dst = omp_build_component_ref (dst, f);
9352 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
9353 append_to_statement_list (t, &list);
9359 /* Last pass: handle VLA firstprivates. */
9360 if (tcctx.cb.decl_map)
9361 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
9362 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
9366 decl = OMP_CLAUSE_DECL (c);
9367 if (!is_variable_sized (decl))
9369 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9372 f = (tree) n->value;
9373 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9374 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
9375 ind = DECL_VALUE_EXPR (decl);
9376 gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
9377 gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
9378 n = splay_tree_lookup (ctx->sfield_map,
9379 (splay_tree_key) TREE_OPERAND (ind, 0));
9380 sf = (tree) n->value;
9381 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9382 src = build_simple_mem_ref_loc (loc, sarg);
9383 src = omp_build_component_ref (src, sf);
9384 src = build_simple_mem_ref_loc (loc, src);
9385 dst = build_simple_mem_ref_loc (loc, arg);
9386 dst = omp_build_component_ref (dst, f);
9387 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
9388 append_to_statement_list (t, &list);
9389 n = splay_tree_lookup (ctx->field_map,
9390 (splay_tree_key) TREE_OPERAND (ind, 0));
9391 df = (tree) n->value;
9392 df = *(tree *) pointer_map_contains (tcctx.cb.decl_map, df);
9393 ptr = build_simple_mem_ref_loc (loc, arg);
9394 ptr = omp_build_component_ref (ptr, df);
9395 t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
9396 build_fold_addr_expr_loc (loc, dst));
9397 append_to_statement_list (t, &list);
9400 t = build1 (RETURN_EXPR, void_type_node, NULL);
9401 append_to_statement_list (t, &list);
9403 if (tcctx.cb.decl_map)
9404 pointer_map_destroy (tcctx.cb.decl_map);
9405 pop_gimplify_context (NULL);
9406 BIND_EXPR_BODY (bind) = list;
9411 lower_depend_clauses (gimple stmt, gimple_seq *iseq, gimple_seq *oseq)
9415 size_t n_in = 0, n_out = 0, idx = 2, i;
9417 clauses = find_omp_clause (gimple_omp_task_clauses (stmt),
9419 gcc_assert (clauses);
9420 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9421 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
9422 switch (OMP_CLAUSE_DEPEND_KIND (c))
9424 case OMP_CLAUSE_DEPEND_IN:
9427 case OMP_CLAUSE_DEPEND_OUT:
9428 case OMP_CLAUSE_DEPEND_INOUT:
9434 tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2);
9435 tree array = create_tmp_var (type, NULL);
9436 tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
9438 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out));
9439 gimple_seq_add_stmt (iseq, g);
9440 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
9442 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out));
9443 gimple_seq_add_stmt (iseq, g);
9444 for (i = 0; i < 2; i++)
9446 if ((i ? n_in : n_out) == 0)
9448 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9449 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
9450 && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i))
9452 tree t = OMP_CLAUSE_DECL (c);
9453 t = fold_convert (ptr_type_node, t);
9454 gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
9455 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
9456 NULL_TREE, NULL_TREE);
9457 g = gimple_build_assign (r, t);
9458 gimple_seq_add_stmt (iseq, g);
9461 tree *p = gimple_omp_task_clauses_ptr (stmt);
9462 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
9463 OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
9464 OMP_CLAUSE_CHAIN (c) = *p;
9466 tree clobber = build_constructor (type, NULL);
9467 TREE_THIS_VOLATILE (clobber) = 1;
9468 g = gimple_build_assign (array, clobber);
9469 gimple_seq_add_stmt (oseq, g);
9472 /* Lower the OpenMP parallel or task directive in the current statement
9473 in GSI_P. CTX holds context information for the directive. */
9476 lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9480 gimple stmt = gsi_stmt (*gsi_p);
9481 gimple par_bind, bind, dep_bind = NULL;
9482 gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body;
9483 location_t loc = gimple_location (stmt);
9485 clauses = gimple_omp_taskreg_clauses (stmt);
9486 par_bind = gimple_seq_first_stmt (gimple_omp_body (stmt));
9487 par_body = gimple_bind_body (par_bind);
9488 child_fn = ctx->cb.dst_fn;
9489 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
9490 && !gimple_omp_parallel_combined_p (stmt))
9492 struct walk_stmt_info wi;
9495 memset (&wi, 0, sizeof (wi));
9498 walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
9500 gimple_omp_parallel_set_combined_p (stmt, true);
9502 gimple_seq dep_ilist = NULL;
9503 gimple_seq dep_olist = NULL;
9504 if (gimple_code (stmt) == GIMPLE_OMP_TASK
9505 && find_omp_clause (clauses, OMP_CLAUSE_DEPEND))
9507 push_gimplify_context ();
9508 dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
9509 lower_depend_clauses (stmt, &dep_ilist, &dep_olist);
9512 if (ctx->srecord_type)
9513 create_task_copyfn (stmt, ctx);
9515 push_gimplify_context ();
9520 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
9521 lower_omp (&par_body, ctx);
9522 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL)
9523 lower_reduction_clauses (clauses, &par_rlist, ctx);
9525 /* Declare all the variables created by mapping and the variables
9526 declared in the scope of the parallel body. */
9527 record_vars_into (ctx->block_vars, child_fn);
9528 record_vars_into (gimple_bind_vars (par_bind), child_fn);
9530 if (ctx->record_type)
9533 = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
9534 : ctx->record_type, ".omp_data_o");
9535 DECL_NAMELESS (ctx->sender_decl) = 1;
9536 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
9537 gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
9542 lower_send_clauses (clauses, &ilist, &olist, ctx);
9543 lower_send_shared_vars (&ilist, &olist, ctx);
9545 if (ctx->record_type)
9547 tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL);
9548 TREE_THIS_VOLATILE (clobber) = 1;
9549 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
9553 /* Once all the expansions are done, sequence all the different
9554 fragments inside gimple_omp_body. */
9558 if (ctx->record_type)
9560 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
9561 /* fixup_child_record_type might have changed receiver_decl's type. */
9562 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
9563 gimple_seq_add_stmt (&new_body,
9564 gimple_build_assign (ctx->receiver_decl, t));
9567 gimple_seq_add_seq (&new_body, par_ilist);
9568 gimple_seq_add_seq (&new_body, par_body);
9569 gimple_seq_add_seq (&new_body, par_rlist);
9570 if (ctx->cancellable)
9571 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
9572 gimple_seq_add_seq (&new_body, par_olist);
9573 new_body = maybe_catch_exception (new_body);
9574 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
9575 gimple_omp_set_body (stmt, new_body);
9577 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
9578 gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
9579 gimple_bind_add_seq (bind, ilist);
9580 gimple_bind_add_stmt (bind, stmt);
9581 gimple_bind_add_seq (bind, olist);
9583 pop_gimplify_context (NULL);
9587 gimple_bind_add_seq (dep_bind, dep_ilist);
9588 gimple_bind_add_stmt (dep_bind, bind);
9589 gimple_bind_add_seq (dep_bind, dep_olist);
9590 pop_gimplify_context (dep_bind);
9594 /* Lower the OpenMP target directive in the current statement
9595 in GSI_P. CTX holds context information for the directive. */
9598 lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9601 tree child_fn, t, c;
9602 gimple stmt = gsi_stmt (*gsi_p);
9603 gimple tgt_bind = NULL, bind;
9604 gimple_seq tgt_body = NULL, olist, ilist, new_body;
9605 location_t loc = gimple_location (stmt);
9606 int kind = gimple_omp_target_kind (stmt);
9607 unsigned int map_cnt = 0;
9609 clauses = gimple_omp_target_clauses (stmt);
9610 if (kind == GF_OMP_TARGET_KIND_REGION)
9612 tgt_bind = gimple_seq_first_stmt (gimple_omp_body (stmt));
9613 tgt_body = gimple_bind_body (tgt_bind);
9615 else if (kind == GF_OMP_TARGET_KIND_DATA)
9616 tgt_body = gimple_omp_body (stmt);
9617 child_fn = ctx->cb.dst_fn;
9619 push_gimplify_context ();
9621 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
9622 switch (OMP_CLAUSE_CODE (c))
9628 case OMP_CLAUSE_MAP:
9630 case OMP_CLAUSE_FROM:
9631 var = OMP_CLAUSE_DECL (c);
9634 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
9635 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
9641 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
9643 tree var2 = DECL_VALUE_EXPR (var);
9644 gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
9645 var2 = TREE_OPERAND (var2, 0);
9646 gcc_assert (DECL_P (var2));
9650 if (!maybe_lookup_field (var, ctx))
9653 if (kind == GF_OMP_TARGET_KIND_REGION)
9655 x = build_receiver_ref (var, true, ctx);
9656 tree new_var = lookup_decl (var, ctx);
9657 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
9658 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
9659 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
9660 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
9661 x = build_simple_mem_ref (x);
9662 SET_DECL_VALUE_EXPR (new_var, x);
9663 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
9668 if (kind == GF_OMP_TARGET_KIND_REGION)
9670 target_nesting_level++;
9671 lower_omp (&tgt_body, ctx);
9672 target_nesting_level--;
9674 else if (kind == GF_OMP_TARGET_KIND_DATA)
9675 lower_omp (&tgt_body, ctx);
9677 if (kind == GF_OMP_TARGET_KIND_REGION)
9679 /* Declare all the variables created by mapping and the variables
9680 declared in the scope of the target body. */
9681 record_vars_into (ctx->block_vars, child_fn);
9682 record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
9687 if (ctx->record_type)
9690 = create_tmp_var (ctx->record_type, ".omp_data_arr");
9691 DECL_NAMELESS (ctx->sender_decl) = 1;
9692 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
9693 t = make_tree_vec (3);
9694 TREE_VEC_ELT (t, 0) = ctx->sender_decl;
9696 = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt),
9698 DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
9699 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
9700 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
9702 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node,
9705 DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
9706 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
9707 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
9708 gimple_omp_target_set_data_arg (stmt, t);
9710 vec<constructor_elt, va_gc> *vsize;
9711 vec<constructor_elt, va_gc> *vkind;
9712 vec_alloc (vsize, map_cnt);
9713 vec_alloc (vkind, map_cnt);
9714 unsigned int map_idx = 0;
9716 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
9717 switch (OMP_CLAUSE_CODE (c))
9723 case OMP_CLAUSE_MAP:
9725 case OMP_CLAUSE_FROM:
9727 ovar = OMP_CLAUSE_DECL (c);
9730 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
9731 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
9733 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c))
9734 == get_base_address (ovar));
9735 nc = OMP_CLAUSE_CHAIN (c);
9736 ovar = OMP_CLAUSE_DECL (nc);
9740 tree x = build_sender_ref (ovar, ctx);
9742 = build_fold_addr_expr_with_type (ovar, ptr_type_node);
9743 gimplify_assign (x, v, &ilist);
9749 if (DECL_SIZE (ovar)
9750 && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST)
9752 tree ovar2 = DECL_VALUE_EXPR (ovar);
9753 gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
9754 ovar2 = TREE_OPERAND (ovar2, 0);
9755 gcc_assert (DECL_P (ovar2));
9758 if (!maybe_lookup_field (ovar, ctx))
9764 tree var = lookup_decl_in_outer_ctx (ovar, ctx);
9765 tree x = build_sender_ref (ovar, ctx);
9766 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
9767 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
9768 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
9769 && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
9771 gcc_assert (kind == GF_OMP_TARGET_KIND_REGION);
9773 = create_tmp_var (TREE_TYPE (TREE_TYPE (x)), NULL);
9774 mark_addressable (avar);
9775 gimplify_assign (avar, build_fold_addr_expr (var), &ilist);
9776 avar = build_fold_addr_expr (avar);
9777 gimplify_assign (x, avar, &ilist);
9779 else if (is_gimple_reg (var))
9781 gcc_assert (kind == GF_OMP_TARGET_KIND_REGION);
9782 tree avar = create_tmp_var (TREE_TYPE (var), NULL);
9783 mark_addressable (avar);
9784 if (OMP_CLAUSE_MAP_KIND (c) != OMP_CLAUSE_MAP_ALLOC
9785 && OMP_CLAUSE_MAP_KIND (c) != OMP_CLAUSE_MAP_FROM)
9786 gimplify_assign (avar, var, &ilist);
9787 avar = build_fold_addr_expr (avar);
9788 gimplify_assign (x, avar, &ilist);
9789 if ((OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_FROM
9790 || OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_TOFROM)
9791 && !TYPE_READONLY (TREE_TYPE (var)))
9793 x = build_sender_ref (ovar, ctx);
9794 x = build_simple_mem_ref (x);
9795 gimplify_assign (var, x, &olist);
9800 var = build_fold_addr_expr (var);
9801 gimplify_assign (x, var, &ilist);
9804 tree s = OMP_CLAUSE_SIZE (c);
9806 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
9807 s = fold_convert (size_type_node, s);
9808 tree purpose = size_int (map_idx++);
9809 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
9810 if (TREE_CODE (s) != INTEGER_CST)
9811 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
9813 unsigned char tkind = 0;
9814 switch (OMP_CLAUSE_CODE (c))
9816 case OMP_CLAUSE_MAP:
9817 tkind = OMP_CLAUSE_MAP_KIND (c);
9820 tkind = OMP_CLAUSE_MAP_TO;
9822 case OMP_CLAUSE_FROM:
9823 tkind = OMP_CLAUSE_MAP_FROM;
9828 unsigned int talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
9829 if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
9830 talign = DECL_ALIGN_UNIT (ovar);
9831 talign = ceil_log2 (talign);
9832 tkind |= talign << 3;
9833 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
9834 build_int_cst (unsigned_char_type_node,
9840 gcc_assert (map_idx == map_cnt);
9842 DECL_INITIAL (TREE_VEC_ELT (t, 1))
9843 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
9844 DECL_INITIAL (TREE_VEC_ELT (t, 2))
9845 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
9846 if (!TREE_STATIC (TREE_VEC_ELT (t, 1)))
9848 gimple_seq initlist = NULL;
9849 force_gimple_operand (build1 (DECL_EXPR, void_type_node,
9850 TREE_VEC_ELT (t, 1)),
9851 &initlist, true, NULL_TREE);
9852 gimple_seq_add_seq (&ilist, initlist);
9854 tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)),
9856 TREE_THIS_VOLATILE (clobber) = 1;
9857 gimple_seq_add_stmt (&olist,
9858 gimple_build_assign (TREE_VEC_ELT (t, 1),
9862 tree clobber = build_constructor (ctx->record_type, NULL);
9863 TREE_THIS_VOLATILE (clobber) = 1;
9864 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
9868 /* Once all the expansions are done, sequence all the different
9869 fragments inside gimple_omp_body. */
9873 if (ctx->record_type && kind == GF_OMP_TARGET_KIND_REGION)
9875 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
9876 /* fixup_child_record_type might have changed receiver_decl's type. */
9877 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
9878 gimple_seq_add_stmt (&new_body,
9879 gimple_build_assign (ctx->receiver_decl, t));
9882 if (kind == GF_OMP_TARGET_KIND_REGION)
9884 gimple_seq_add_seq (&new_body, tgt_body);
9885 new_body = maybe_catch_exception (new_body);
9887 else if (kind == GF_OMP_TARGET_KIND_DATA)
9888 new_body = tgt_body;
9889 if (kind != GF_OMP_TARGET_KIND_UPDATE)
9891 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
9892 gimple_omp_set_body (stmt, new_body);
9895 bind = gimple_build_bind (NULL, NULL,
9896 tgt_bind ? gimple_bind_block (tgt_bind)
9898 gsi_replace (gsi_p, bind, true);
9899 gimple_bind_add_seq (bind, ilist);
9900 gimple_bind_add_stmt (bind, stmt);
9901 gimple_bind_add_seq (bind, olist);
9903 pop_gimplify_context (NULL);
9906 /* Expand code for an OpenMP teams directive. */
9909 lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9911 gimple teams_stmt = gsi_stmt (*gsi_p);
9912 push_gimplify_context ();
9914 tree block = make_node (BLOCK);
9915 gimple bind = gimple_build_bind (NULL, NULL, block);
9916 gsi_replace (gsi_p, bind, true);
9917 gimple_seq bind_body = NULL;
9918 gimple_seq dlist = NULL;
9919 gimple_seq olist = NULL;
9921 tree num_teams = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
9922 OMP_CLAUSE_NUM_TEAMS);
9923 if (num_teams == NULL_TREE)
9924 num_teams = build_int_cst (unsigned_type_node, 0);
9927 num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams);
9928 num_teams = fold_convert (unsigned_type_node, num_teams);
9929 gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
9931 tree thread_limit = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
9932 OMP_CLAUSE_THREAD_LIMIT);
9933 if (thread_limit == NULL_TREE)
9934 thread_limit = build_int_cst (unsigned_type_node, 0);
9937 thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
9938 thread_limit = fold_convert (unsigned_type_node, thread_limit);
9939 gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
9943 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
9944 &bind_body, &dlist, ctx, NULL);
9945 lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
9946 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx);
9947 gimple_seq_add_stmt (&bind_body, teams_stmt);
9949 location_t loc = gimple_location (teams_stmt);
9950 tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS);
9951 gimple call = gimple_build_call (decl, 2, num_teams, thread_limit);
9952 gimple_set_location (call, loc);
9953 gimple_seq_add_stmt (&bind_body, call);
9955 gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
9956 gimple_omp_set_body (teams_stmt, NULL);
9957 gimple_seq_add_seq (&bind_body, olist);
9958 gimple_seq_add_seq (&bind_body, dlist);
9959 gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
9960 gimple_bind_set_body (bind, bind_body);
9962 pop_gimplify_context (bind);
9964 gimple_bind_append_vars (bind, ctx->block_vars);
9965 BLOCK_VARS (block) = ctx->block_vars;
9966 if (BLOCK_VARS (block))
9967 TREE_USED (block) = 1;
9971 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
9972 regimplified. If DATA is non-NULL, lower_omp_1 is outside
9973 of OpenMP context, but with task_shared_vars set. */
9976 lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
9981 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
9982 if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t))
9985 if (task_shared_vars
9987 && bitmap_bit_p (task_shared_vars, DECL_UID (t)))
9990 /* If a global variable has been privatized, TREE_CONSTANT on
9991 ADDR_EXPR might be wrong. */
9992 if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
9993 recompute_tree_invariant_for_addr_expr (t);
9995 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
10000 lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10002 gimple stmt = gsi_stmt (*gsi_p);
10003 struct walk_stmt_info wi;
10005 if (gimple_has_location (stmt))
10006 input_location = gimple_location (stmt);
10008 if (task_shared_vars)
10009 memset (&wi, '\0', sizeof (wi));
10011 /* If we have issued syntax errors, avoid doing any heavy lifting.
10012 Just replace the OpenMP directives with a NOP to avoid
10013 confusing RTL expansion. */
10014 if (seen_error () && is_gimple_omp (stmt))
10016 gsi_replace (gsi_p, gimple_build_nop (), true);
10020 switch (gimple_code (stmt))
10023 if ((ctx || task_shared_vars)
10024 && (walk_tree (gimple_cond_lhs_ptr (stmt), lower_omp_regimplify_p,
10025 ctx ? NULL : &wi, NULL)
10026 || walk_tree (gimple_cond_rhs_ptr (stmt), lower_omp_regimplify_p,
10027 ctx ? NULL : &wi, NULL)))
10028 gimple_regimplify_operands (stmt, gsi_p);
10031 lower_omp (gimple_catch_handler_ptr (stmt), ctx);
10033 case GIMPLE_EH_FILTER:
10034 lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
10037 lower_omp (gimple_try_eval_ptr (stmt), ctx);
10038 lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
10040 case GIMPLE_TRANSACTION:
10041 lower_omp (gimple_transaction_body_ptr (stmt), ctx);
10044 lower_omp (gimple_bind_body_ptr (stmt), ctx);
10046 case GIMPLE_OMP_PARALLEL:
10047 case GIMPLE_OMP_TASK:
10048 ctx = maybe_lookup_ctx (stmt);
10050 if (ctx->cancellable)
10051 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
10052 lower_omp_taskreg (gsi_p, ctx);
10054 case GIMPLE_OMP_FOR:
10055 ctx = maybe_lookup_ctx (stmt);
10057 if (ctx->cancellable)
10058 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
10059 lower_omp_for (gsi_p, ctx);
10061 case GIMPLE_OMP_SECTIONS:
10062 ctx = maybe_lookup_ctx (stmt);
10064 if (ctx->cancellable)
10065 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
10066 lower_omp_sections (gsi_p, ctx);
10068 case GIMPLE_OMP_SINGLE:
10069 ctx = maybe_lookup_ctx (stmt);
10071 lower_omp_single (gsi_p, ctx);
10073 case GIMPLE_OMP_MASTER:
10074 ctx = maybe_lookup_ctx (stmt);
10076 lower_omp_master (gsi_p, ctx);
10078 case GIMPLE_OMP_TASKGROUP:
10079 ctx = maybe_lookup_ctx (stmt);
10081 lower_omp_taskgroup (gsi_p, ctx);
10083 case GIMPLE_OMP_ORDERED:
10084 ctx = maybe_lookup_ctx (stmt);
10086 lower_omp_ordered (gsi_p, ctx);
10088 case GIMPLE_OMP_CRITICAL:
10089 ctx = maybe_lookup_ctx (stmt);
10091 lower_omp_critical (gsi_p, ctx);
10093 case GIMPLE_OMP_ATOMIC_LOAD:
10094 if ((ctx || task_shared_vars)
10095 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt),
10096 lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
10097 gimple_regimplify_operands (stmt, gsi_p);
10099 case GIMPLE_OMP_TARGET:
10100 ctx = maybe_lookup_ctx (stmt);
10102 lower_omp_target (gsi_p, ctx);
10104 case GIMPLE_OMP_TEAMS:
10105 ctx = maybe_lookup_ctx (stmt);
10107 lower_omp_teams (gsi_p, ctx);
10111 fndecl = gimple_call_fndecl (stmt);
10113 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
10114 switch (DECL_FUNCTION_CODE (fndecl))
10116 case BUILT_IN_GOMP_BARRIER:
10120 case BUILT_IN_GOMP_CANCEL:
10121 case BUILT_IN_GOMP_CANCELLATION_POINT:
10124 if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
10125 cctx = cctx->outer;
10126 gcc_assert (gimple_call_lhs (stmt) == NULL_TREE);
10127 if (!cctx->cancellable)
10129 if (DECL_FUNCTION_CODE (fndecl)
10130 == BUILT_IN_GOMP_CANCELLATION_POINT)
10132 stmt = gimple_build_nop ();
10133 gsi_replace (gsi_p, stmt, false);
10137 if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
10139 fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
10140 gimple_call_set_fndecl (stmt, fndecl);
10141 gimple_call_set_fntype (stmt, TREE_TYPE (fndecl));
10144 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)), NULL);
10145 gimple_call_set_lhs (stmt, lhs);
10146 tree fallthru_label;
10147 fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
10149 g = gimple_build_label (fallthru_label);
10150 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
10151 g = gimple_build_cond (NE_EXPR, lhs,
10152 fold_convert (TREE_TYPE (lhs),
10153 boolean_false_node),
10154 cctx->cancel_label, fallthru_label);
10155 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
10162 if ((ctx || task_shared_vars)
10163 && walk_gimple_op (stmt, lower_omp_regimplify_p,
10166 /* Just remove clobbers, this should happen only if we have
10167 "privatized" local addressable variables in SIMD regions,
10168 the clobber isn't needed in that case and gimplifying address
10169 of the ARRAY_REF into a pointer and creating MEM_REF based
10170 clobber would create worse code than we get with the clobber
10172 if (gimple_clobber_p (stmt))
10174 gsi_replace (gsi_p, gimple_build_nop (), true);
10177 gimple_regimplify_operands (stmt, gsi_p);
10184 lower_omp (gimple_seq *body, omp_context *ctx)
10186 location_t saved_location = input_location;
10187 gimple_stmt_iterator gsi;
10188 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
10189 lower_omp_1 (&gsi, ctx);
10190 /* During gimplification, we have not always invoked fold_stmt
10191 (gimplify.c:maybe_fold_stmt); call it now. */
10192 if (target_nesting_level)
10193 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
10195 input_location = saved_location;
10198 /* Main entry point. */
10200 static unsigned int
10201 execute_lower_omp (void)
10205 /* This pass always runs, to provide PROP_gimple_lomp.
10206 But there is nothing to do unless -fopenmp is given. */
10207 if (flag_openmp == 0 && flag_openmp_simd == 0 && flag_cilkplus == 0)
10210 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
10211 delete_omp_context);
10213 body = gimple_body (current_function_decl);
10214 scan_omp (&body, NULL);
10215 gcc_assert (taskreg_nesting_level == 0);
10217 if (all_contexts->root)
10219 if (task_shared_vars)
10220 push_gimplify_context ();
10221 lower_omp (&body, NULL);
10222 if (task_shared_vars)
10223 pop_gimplify_context (NULL);
10228 splay_tree_delete (all_contexts);
10229 all_contexts = NULL;
10231 BITMAP_FREE (task_shared_vars);
10237 const pass_data pass_data_lower_omp =
10239 GIMPLE_PASS, /* type */
10240 "omplower", /* name */
10241 OPTGROUP_NONE, /* optinfo_flags */
10242 true, /* has_execute */
10243 TV_NONE, /* tv_id */
10244 PROP_gimple_any, /* properties_required */
10245 PROP_gimple_lomp, /* properties_provided */
10246 0, /* properties_destroyed */
10247 0, /* todo_flags_start */
10248 0, /* todo_flags_finish */
10251 class pass_lower_omp : public gimple_opt_pass
10254 pass_lower_omp (gcc::context *ctxt)
10255 : gimple_opt_pass (pass_data_lower_omp, ctxt)
10258 /* opt_pass methods: */
10259 virtual unsigned int execute (function *) { return execute_lower_omp (); }
10261 }; // class pass_lower_omp
10263 } // anon namespace
10266 make_pass_lower_omp (gcc::context *ctxt)
10268 return new pass_lower_omp (ctxt);
10271 /* The following is a utility to diagnose OpenMP structured block violations.
10272 It is not part of the "omplower" pass, as that's invoked too late. It
10273 should be invoked by the respective front ends after gimplification. */
10275 static splay_tree all_labels;
10277 /* Check for mismatched contexts and generate an error if needed. Return
10278 true if an error is detected. */
10281 diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
10282 gimple branch_ctx, gimple label_ctx)
10284 if (label_ctx == branch_ctx)
10289 Previously we kept track of the label's entire context in diagnose_sb_[12]
10290 so we could traverse it and issue a correct "exit" or "enter" error
10291 message upon a structured block violation.
10293 We built the context by building a list with tree_cons'ing, but there is
10294 no easy counterpart in gimple tuples. It seems like far too much work
10295 for issuing exit/enter error messages. If someone really misses the
10296 distinct error message... patches welcome.
10300 /* Try to avoid confusing the user by producing and error message
10301 with correct "exit" or "enter" verbiage. We prefer "exit"
10302 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10303 if (branch_ctx == NULL)
10309 if (TREE_VALUE (label_ctx) == branch_ctx)
10314 label_ctx = TREE_CHAIN (label_ctx);
10319 error ("invalid exit from OpenMP structured block");
10321 error ("invalid entry to OpenMP structured block");
10324 bool cilkplus_block = false;
10328 && gimple_code (branch_ctx) == GIMPLE_OMP_FOR
10329 && gimple_omp_for_kind (branch_ctx) == GF_OMP_FOR_KIND_CILKSIMD)
10331 && gimple_code (label_ctx) == GIMPLE_OMP_FOR
10332 && gimple_omp_for_kind (label_ctx) == GF_OMP_FOR_KIND_CILKSIMD))
10333 cilkplus_block = true;
10336 /* If it's obvious we have an invalid entry, be specific about the error. */
10337 if (branch_ctx == NULL)
10339 if (cilkplus_block)
10340 error ("invalid entry to Cilk Plus structured block");
10342 error ("invalid entry to OpenMP structured block");
10346 /* Otherwise, be vague and lazy, but efficient. */
10347 if (cilkplus_block)
10348 error ("invalid branch to/from a Cilk Plus structured block");
10350 error ("invalid branch to/from an OpenMP structured block");
10353 gsi_replace (gsi_p, gimple_build_nop (), false);
10357 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10358 where each label is found. */
10361 diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10362 struct walk_stmt_info *wi)
10364 gimple context = (gimple) wi->info;
10365 gimple inner_context;
10366 gimple stmt = gsi_stmt (*gsi_p);
10368 *handled_ops_p = true;
10370 switch (gimple_code (stmt))
10374 case GIMPLE_OMP_PARALLEL:
10375 case GIMPLE_OMP_TASK:
10376 case GIMPLE_OMP_SECTIONS:
10377 case GIMPLE_OMP_SINGLE:
10378 case GIMPLE_OMP_SECTION:
10379 case GIMPLE_OMP_MASTER:
10380 case GIMPLE_OMP_ORDERED:
10381 case GIMPLE_OMP_CRITICAL:
10382 case GIMPLE_OMP_TARGET:
10383 case GIMPLE_OMP_TEAMS:
10384 case GIMPLE_OMP_TASKGROUP:
10385 /* The minimal context here is just the current OMP construct. */
10386 inner_context = stmt;
10387 wi->info = inner_context;
10388 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
10389 wi->info = context;
10392 case GIMPLE_OMP_FOR:
10393 inner_context = stmt;
10394 wi->info = inner_context;
10395 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10397 walk_gimple_seq (gimple_omp_for_pre_body (stmt),
10398 diagnose_sb_1, NULL, wi);
10399 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
10400 wi->info = context;
10404 splay_tree_insert (all_labels, (splay_tree_key) gimple_label_label (stmt),
10405 (splay_tree_value) context);
10415 /* Pass 2: Check each branch and see if its context differs from that of
10416 the destination label's context. */
10419 diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10420 struct walk_stmt_info *wi)
10422 gimple context = (gimple) wi->info;
10424 gimple stmt = gsi_stmt (*gsi_p);
10426 *handled_ops_p = true;
10428 switch (gimple_code (stmt))
10432 case GIMPLE_OMP_PARALLEL:
10433 case GIMPLE_OMP_TASK:
10434 case GIMPLE_OMP_SECTIONS:
10435 case GIMPLE_OMP_SINGLE:
10436 case GIMPLE_OMP_SECTION:
10437 case GIMPLE_OMP_MASTER:
10438 case GIMPLE_OMP_ORDERED:
10439 case GIMPLE_OMP_CRITICAL:
10440 case GIMPLE_OMP_TARGET:
10441 case GIMPLE_OMP_TEAMS:
10442 case GIMPLE_OMP_TASKGROUP:
10444 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
10445 wi->info = context;
10448 case GIMPLE_OMP_FOR:
10450 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10452 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
10453 diagnose_sb_2, NULL, wi);
10454 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
10455 wi->info = context;
10460 tree lab = gimple_cond_true_label (stmt);
10463 n = splay_tree_lookup (all_labels,
10464 (splay_tree_key) lab);
10465 diagnose_sb_0 (gsi_p, context,
10466 n ? (gimple) n->value : NULL);
10468 lab = gimple_cond_false_label (stmt);
10471 n = splay_tree_lookup (all_labels,
10472 (splay_tree_key) lab);
10473 diagnose_sb_0 (gsi_p, context,
10474 n ? (gimple) n->value : NULL);
10481 tree lab = gimple_goto_dest (stmt);
10482 if (TREE_CODE (lab) != LABEL_DECL)
10485 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
10486 diagnose_sb_0 (gsi_p, context, n ? (gimple) n->value : NULL);
10490 case GIMPLE_SWITCH:
10493 for (i = 0; i < gimple_switch_num_labels (stmt); ++i)
10495 tree lab = CASE_LABEL (gimple_switch_label (stmt, i));
10496 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
10497 if (n && diagnose_sb_0 (gsi_p, context, (gimple) n->value))
10503 case GIMPLE_RETURN:
10504 diagnose_sb_0 (gsi_p, context, NULL);
10514 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10517 make_gimple_omp_edges (basic_block bb, struct omp_region **region,
10520 gimple last = last_stmt (bb);
10521 enum gimple_code code = gimple_code (last);
10522 struct omp_region *cur_region = *region;
10523 bool fallthru = false;
10527 case GIMPLE_OMP_PARALLEL:
10528 case GIMPLE_OMP_TASK:
10529 case GIMPLE_OMP_FOR:
10530 case GIMPLE_OMP_SINGLE:
10531 case GIMPLE_OMP_TEAMS:
10532 case GIMPLE_OMP_MASTER:
10533 case GIMPLE_OMP_TASKGROUP:
10534 case GIMPLE_OMP_ORDERED:
10535 case GIMPLE_OMP_CRITICAL:
10536 case GIMPLE_OMP_SECTION:
10537 cur_region = new_omp_region (bb, code, cur_region);
10541 case GIMPLE_OMP_TARGET:
10542 cur_region = new_omp_region (bb, code, cur_region);
10544 if (gimple_omp_target_kind (last) == GF_OMP_TARGET_KIND_UPDATE)
10545 cur_region = cur_region->outer;
10548 case GIMPLE_OMP_SECTIONS:
10549 cur_region = new_omp_region (bb, code, cur_region);
10553 case GIMPLE_OMP_SECTIONS_SWITCH:
10557 case GIMPLE_OMP_ATOMIC_LOAD:
10558 case GIMPLE_OMP_ATOMIC_STORE:
10562 case GIMPLE_OMP_RETURN:
10563 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10564 somewhere other than the next block. This will be
10566 cur_region->exit = bb;
10567 fallthru = cur_region->type != GIMPLE_OMP_SECTION;
10568 cur_region = cur_region->outer;
10571 case GIMPLE_OMP_CONTINUE:
10572 cur_region->cont = bb;
10573 switch (cur_region->type)
10575 case GIMPLE_OMP_FOR:
10576 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10577 succs edges as abnormal to prevent splitting
10579 single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL;
10580 /* Make the loopback edge. */
10581 make_edge (bb, single_succ (cur_region->entry),
10584 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10585 corresponds to the case that the body of the loop
10586 is not executed at all. */
10587 make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL);
10588 make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL);
10592 case GIMPLE_OMP_SECTIONS:
10593 /* Wire up the edges into and out of the nested sections. */
10595 basic_block switch_bb = single_succ (cur_region->entry);
10597 struct omp_region *i;
10598 for (i = cur_region->inner; i ; i = i->next)
10600 gcc_assert (i->type == GIMPLE_OMP_SECTION);
10601 make_edge (switch_bb, i->entry, 0);
10602 make_edge (i->exit, bb, EDGE_FALLTHRU);
10605 /* Make the loopback edge to the block with
10606 GIMPLE_OMP_SECTIONS_SWITCH. */
10607 make_edge (bb, switch_bb, 0);
10609 /* Make the edge from the switch to exit. */
10610 make_edge (switch_bb, bb->next_bb, 0);
10616 gcc_unreachable ();
10621 gcc_unreachable ();
10624 if (*region != cur_region)
10626 *region = cur_region;
10628 *region_idx = cur_region->entry->index;
10636 static unsigned int
10637 diagnose_omp_structured_block_errors (void)
10639 struct walk_stmt_info wi;
10640 gimple_seq body = gimple_body (current_function_decl);
10642 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
10644 memset (&wi, 0, sizeof (wi));
10645 walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
10647 memset (&wi, 0, sizeof (wi));
10648 wi.want_locations = true;
10649 walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
10651 gimple_set_body (current_function_decl, body);
10653 splay_tree_delete (all_labels);
10661 const pass_data pass_data_diagnose_omp_blocks =
10663 GIMPLE_PASS, /* type */
10664 "*diagnose_omp_blocks", /* name */
10665 OPTGROUP_NONE, /* optinfo_flags */
10666 true, /* has_execute */
10667 TV_NONE, /* tv_id */
10668 PROP_gimple_any, /* properties_required */
10669 0, /* properties_provided */
10670 0, /* properties_destroyed */
10671 0, /* todo_flags_start */
10672 0, /* todo_flags_finish */
10675 class pass_diagnose_omp_blocks : public gimple_opt_pass
10678 pass_diagnose_omp_blocks (gcc::context *ctxt)
10679 : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
10682 /* opt_pass methods: */
10683 virtual bool gate (function *) { return flag_openmp || flag_cilkplus; }
10684 virtual unsigned int execute (function *)
10686 return diagnose_omp_structured_block_errors ();
10689 }; // class pass_diagnose_omp_blocks
10691 } // anon namespace
10694 make_pass_diagnose_omp_blocks (gcc::context *ctxt)
10696 return new pass_diagnose_omp_blocks (ctxt);
10699 /* SIMD clone supporting code. */
10701 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
10702 of arguments to reserve space for. */
10704 static struct cgraph_simd_clone *
10705 simd_clone_struct_alloc (int nargs)
10707 struct cgraph_simd_clone *clone_info;
10708 size_t len = (sizeof (struct cgraph_simd_clone)
10709 + nargs * sizeof (struct cgraph_simd_clone_arg));
10710 clone_info = (struct cgraph_simd_clone *)
10711 ggc_internal_cleared_alloc (len);
10715 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
10718 simd_clone_struct_copy (struct cgraph_simd_clone *to,
10719 struct cgraph_simd_clone *from)
10721 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
10722 + ((from->nargs - from->inbranch)
10723 * sizeof (struct cgraph_simd_clone_arg))));
10726 /* Return vector of parameter types of function FNDECL. This uses
10727 TYPE_ARG_TYPES if available, otherwise falls back to types of
10728 DECL_ARGUMENTS types. */
10731 simd_clone_vector_of_formal_parm_types (tree fndecl)
10733 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
10734 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
10735 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
10738 FOR_EACH_VEC_ELT (args, i, arg)
10739 args[i] = TREE_TYPE (args[i]);
10743 /* Given a simd function in NODE, extract the simd specific
10744 information from the OMP clauses passed in CLAUSES, and return
10745 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
10746 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
10747 otherwise set to FALSE. */
10749 static struct cgraph_simd_clone *
10750 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
10751 bool *inbranch_specified)
10753 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
10756 *inbranch_specified = false;
10758 n = args.length ();
10759 if (n > 0 && args.last () == void_type_node)
10762 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
10763 be cloned have a distinctive artificial label in addition to "omp
10767 && lookup_attribute ("cilk simd function",
10768 DECL_ATTRIBUTES (node->decl)));
10770 /* Allocate one more than needed just in case this is an in-branch
10771 clone which will require a mask argument. */
10772 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
10773 clone_info->nargs = n;
10774 clone_info->cilk_elemental = cilk_clone;
10781 clauses = TREE_VALUE (clauses);
10782 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
10785 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
10787 switch (OMP_CLAUSE_CODE (t))
10789 case OMP_CLAUSE_INBRANCH:
10790 clone_info->inbranch = 1;
10791 *inbranch_specified = true;
10793 case OMP_CLAUSE_NOTINBRANCH:
10794 clone_info->inbranch = 0;
10795 *inbranch_specified = true;
10797 case OMP_CLAUSE_SIMDLEN:
10798 clone_info->simdlen
10799 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
10801 case OMP_CLAUSE_LINEAR:
10803 tree decl = OMP_CLAUSE_DECL (t);
10804 tree step = OMP_CLAUSE_LINEAR_STEP (t);
10805 int argno = TREE_INT_CST_LOW (decl);
10806 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
10808 clone_info->args[argno].arg_type
10809 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
10810 clone_info->args[argno].linear_step = tree_to_shwi (step);
10811 gcc_assert (clone_info->args[argno].linear_step >= 0
10812 && clone_info->args[argno].linear_step < n);
10816 if (POINTER_TYPE_P (args[argno]))
10817 step = fold_convert (ssizetype, step);
10818 if (!tree_fits_shwi_p (step))
10820 warning_at (OMP_CLAUSE_LOCATION (t), 0,
10821 "ignoring large linear step");
10825 else if (integer_zerop (step))
10827 warning_at (OMP_CLAUSE_LOCATION (t), 0,
10828 "ignoring zero linear step");
10834 clone_info->args[argno].arg_type
10835 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
10836 clone_info->args[argno].linear_step = tree_to_shwi (step);
10841 case OMP_CLAUSE_UNIFORM:
10843 tree decl = OMP_CLAUSE_DECL (t);
10844 int argno = tree_to_uhwi (decl);
10845 clone_info->args[argno].arg_type
10846 = SIMD_CLONE_ARG_TYPE_UNIFORM;
10849 case OMP_CLAUSE_ALIGNED:
10851 tree decl = OMP_CLAUSE_DECL (t);
10852 int argno = tree_to_uhwi (decl);
10853 clone_info->args[argno].alignment
10854 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
10865 /* Given a SIMD clone in NODE, calculate the characteristic data
10866 type and return the coresponding type. The characteristic data
10867 type is computed as described in the Intel Vector ABI. */
10870 simd_clone_compute_base_data_type (struct cgraph_node *node,
10871 struct cgraph_simd_clone *clone_info)
10873 tree type = integer_type_node;
10874 tree fndecl = node->decl;
10876 /* a) For non-void function, the characteristic data type is the
10878 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
10879 type = TREE_TYPE (TREE_TYPE (fndecl));
10881 /* b) If the function has any non-uniform, non-linear parameters,
10882 then the characteristic data type is the type of the first
10886 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
10887 for (unsigned int i = 0; i < clone_info->nargs; ++i)
10888 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
10896 /* c) If the characteristic data type determined by a) or b) above
10897 is struct, union, or class type which is pass-by-value (except
10898 for the type that maps to the built-in complex data type), the
10899 characteristic data type is int. */
10900 if (RECORD_OR_UNION_TYPE_P (type)
10901 && !aggregate_value_p (type, NULL)
10902 && TREE_CODE (type) != COMPLEX_TYPE)
10903 return integer_type_node;
10905 /* d) If none of the above three classes is applicable, the
10906 characteristic data type is int. */
10910 /* e) For Intel Xeon Phi native and offload compilation, if the
10911 resulting characteristic data type is 8-bit or 16-bit integer
10912 data type, the characteristic data type is int. */
10913 /* Well, we don't handle Xeon Phi yet. */
10917 simd_clone_mangle (struct cgraph_node *node,
10918 struct cgraph_simd_clone *clone_info)
10920 char vecsize_mangle = clone_info->vecsize_mangle;
10921 char mask = clone_info->inbranch ? 'M' : 'N';
10922 unsigned int simdlen = clone_info->simdlen;
10926 gcc_assert (vecsize_mangle && simdlen);
10928 pp_string (&pp, "_ZGV");
10929 pp_character (&pp, vecsize_mangle);
10930 pp_character (&pp, mask);
10931 pp_decimal_int (&pp, simdlen);
10933 for (n = 0; n < clone_info->nargs; ++n)
10935 struct cgraph_simd_clone_arg arg = clone_info->args[n];
10937 if (arg.arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM)
10938 pp_character (&pp, 'u');
10939 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
10941 gcc_assert (arg.linear_step != 0);
10942 pp_character (&pp, 'l');
10943 if (arg.linear_step > 1)
10944 pp_unsigned_wide_integer (&pp, arg.linear_step);
10945 else if (arg.linear_step < 0)
10947 pp_character (&pp, 'n');
10948 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
10952 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
10954 pp_character (&pp, 's');
10955 pp_unsigned_wide_integer (&pp, arg.linear_step);
10958 pp_character (&pp, 'v');
10961 pp_character (&pp, 'a');
10962 pp_decimal_int (&pp, arg.alignment);
10966 pp_underscore (&pp);
10968 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)));
10969 const char *str = pp_formatted_text (&pp);
10971 /* If there already is a SIMD clone with the same mangled name, don't
10972 add another one. This can happen e.g. for
10973 #pragma omp declare simd
10974 #pragma omp declare simd simdlen(8)
10975 int foo (int, int);
10976 if the simdlen is assumed to be 8 for the first one, etc. */
10977 for (struct cgraph_node *clone = node->simd_clones; clone;
10978 clone = clone->simdclone->next_clone)
10979 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)),
10983 return get_identifier (str);
10986 /* Create a simd clone of OLD_NODE and return it. */
10988 static struct cgraph_node *
10989 simd_clone_create (struct cgraph_node *old_node)
10991 struct cgraph_node *new_node;
10992 if (old_node->definition)
10994 if (!cgraph_function_with_gimple_body_p (old_node))
10996 cgraph_get_body (old_node);
10997 new_node = cgraph_function_versioning (old_node, vNULL, NULL, NULL,
10998 false, NULL, NULL, "simdclone");
11002 tree old_decl = old_node->decl;
11003 tree new_decl = copy_node (old_node->decl);
11004 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
11005 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
11006 SET_DECL_RTL (new_decl, NULL);
11007 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
11008 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
11010 = cgraph_copy_node_for_versioning (old_node, new_decl, vNULL, NULL);
11011 cgraph_call_function_insertion_hooks (new_node);
11013 if (new_node == NULL)
11016 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
11018 /* The function cgraph_function_versioning () will force the new
11019 symbol local. Undo this, and inherit external visability from
11021 new_node->local.local = old_node->local.local;
11022 new_node->externally_visible = old_node->externally_visible;
11027 /* Adjust the return type of the given function to its appropriate
11028 vector counterpart. Returns a simd array to be used throughout the
11029 function as a return value. */
11032 simd_clone_adjust_return_type (struct cgraph_node *node)
11034 tree fndecl = node->decl;
11035 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
11036 unsigned int veclen;
11039 /* Adjust the function return type. */
11040 if (orig_rettype == void_type_node)
11042 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
11043 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl)))
11044 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl))))
11045 veclen = node->simdclone->vecsize_int;
11047 veclen = node->simdclone->vecsize_float;
11048 veclen /= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl))));
11049 if (veclen > node->simdclone->simdlen)
11050 veclen = node->simdclone->simdlen;
11051 if (veclen == node->simdclone->simdlen)
11052 TREE_TYPE (TREE_TYPE (fndecl))
11053 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl)),
11054 node->simdclone->simdlen);
11057 t = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl)), veclen);
11058 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
11059 TREE_TYPE (TREE_TYPE (fndecl)) = t;
11061 if (!node->definition)
11064 t = DECL_RESULT (fndecl);
11065 /* Adjust the DECL_RESULT. */
11066 gcc_assert (TREE_TYPE (t) != void_type_node);
11067 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
11070 tree atype = build_array_type_nelts (orig_rettype,
11071 node->simdclone->simdlen);
11072 if (veclen != node->simdclone->simdlen)
11073 return build1 (VIEW_CONVERT_EXPR, atype, t);
11075 /* Set up a SIMD array to use as the return value. */
11076 tree retval = create_tmp_var_raw (atype, "retval");
11077 gimple_add_tmp_var (retval);
11081 /* Each vector argument has a corresponding array to be used locally
11082 as part of the eventual loop. Create such temporary array and
11085 PREFIX is the prefix to be used for the temporary.
11087 TYPE is the inner element type.
11089 SIMDLEN is the number of elements. */
11092 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
11094 tree atype = build_array_type_nelts (type, simdlen);
11095 tree avar = create_tmp_var_raw (atype, prefix);
11096 gimple_add_tmp_var (avar);
11100 /* Modify the function argument types to their corresponding vector
11101 counterparts if appropriate. Also, create one array for each simd
11102 argument to be used locally when using the function arguments as
11105 NODE is the function whose arguments are to be adjusted.
11107 Returns an adjustment vector that will be filled describing how the
11108 argument types will be adjusted. */
11110 static ipa_parm_adjustment_vec
11111 simd_clone_adjust_argument_types (struct cgraph_node *node)
11114 ipa_parm_adjustment_vec adjustments;
11116 if (node->definition)
11117 args = ipa_get_vector_of_formal_parms (node->decl);
11119 args = simd_clone_vector_of_formal_parm_types (node->decl);
11120 adjustments.create (args.length ());
11121 unsigned i, j, veclen;
11122 struct ipa_parm_adjustment adj;
11123 for (i = 0; i < node->simdclone->nargs; ++i)
11125 memset (&adj, 0, sizeof (adj));
11126 tree parm = args[i];
11127 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
11128 adj.base_index = i;
11131 node->simdclone->args[i].orig_arg = node->definition ? parm : NULL_TREE;
11132 node->simdclone->args[i].orig_type = parm_type;
11134 if (node->simdclone->args[i].arg_type != SIMD_CLONE_ARG_TYPE_VECTOR)
11136 /* No adjustment necessary for scalar arguments. */
11137 adj.op = IPA_PARM_OP_COPY;
11141 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
11142 veclen = node->simdclone->vecsize_int;
11144 veclen = node->simdclone->vecsize_float;
11145 veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type));
11146 if (veclen > node->simdclone->simdlen)
11147 veclen = node->simdclone->simdlen;
11148 adj.arg_prefix = "simd";
11149 adj.type = build_vector_type (parm_type, veclen);
11150 node->simdclone->args[i].vector_type = adj.type;
11151 for (j = veclen; j < node->simdclone->simdlen; j += veclen)
11153 adjustments.safe_push (adj);
11156 memset (&adj, 0, sizeof (adj));
11157 adj.op = IPA_PARM_OP_NEW;
11158 adj.arg_prefix = "simd";
11159 adj.base_index = i;
11160 adj.type = node->simdclone->args[i].vector_type;
11164 if (node->definition)
11165 node->simdclone->args[i].simd_array
11166 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
11167 parm_type, node->simdclone->simdlen);
11169 adjustments.safe_push (adj);
11172 if (node->simdclone->inbranch)
11175 = simd_clone_compute_base_data_type (node->simdclone->origin,
11178 memset (&adj, 0, sizeof (adj));
11179 adj.op = IPA_PARM_OP_NEW;
11180 adj.arg_prefix = "mask";
11182 adj.base_index = i;
11183 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
11184 veclen = node->simdclone->vecsize_int;
11186 veclen = node->simdclone->vecsize_float;
11187 veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
11188 if (veclen > node->simdclone->simdlen)
11189 veclen = node->simdclone->simdlen;
11190 adj.type = build_vector_type (base_type, veclen);
11191 adjustments.safe_push (adj);
11193 for (j = veclen; j < node->simdclone->simdlen; j += veclen)
11194 adjustments.safe_push (adj);
11196 /* We have previously allocated one extra entry for the mask. Use
11198 struct cgraph_simd_clone *sc = node->simdclone;
11200 if (node->definition)
11202 sc->args[i].orig_arg
11203 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
11204 sc->args[i].simd_array
11205 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
11207 sc->args[i].orig_type = base_type;
11208 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
11211 if (node->definition)
11212 ipa_modify_formal_parameters (node->decl, adjustments);
11215 tree new_arg_types = NULL_TREE, new_reversed;
11216 bool last_parm_void = false;
11217 if (args.length () > 0 && args.last () == void_type_node)
11218 last_parm_void = true;
11220 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
11221 j = adjustments.length ();
11222 for (i = 0; i < j; i++)
11224 struct ipa_parm_adjustment *adj = &adjustments[i];
11226 if (adj->op == IPA_PARM_OP_COPY)
11227 ptype = args[adj->base_index];
11230 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
11232 new_reversed = nreverse (new_arg_types);
11233 if (last_parm_void)
11236 TREE_CHAIN (new_arg_types) = void_list_node;
11238 new_reversed = void_list_node;
11241 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
11242 TYPE_ARG_TYPES (new_type) = new_reversed;
11243 TREE_TYPE (node->decl) = new_type;
11245 adjustments.release ();
11248 return adjustments;
11251 /* Initialize and copy the function arguments in NODE to their
11252 corresponding local simd arrays. Returns a fresh gimple_seq with
11253 the instruction sequence generated. */
11256 simd_clone_init_simd_arrays (struct cgraph_node *node,
11257 ipa_parm_adjustment_vec adjustments)
11259 gimple_seq seq = NULL;
11260 unsigned i = 0, j = 0, k;
11262 for (tree arg = DECL_ARGUMENTS (node->decl);
11264 arg = DECL_CHAIN (arg), i++, j++)
11266 if (adjustments[j].op == IPA_PARM_OP_COPY)
11269 node->simdclone->args[i].vector_arg = arg;
11271 tree array = node->simdclone->args[i].simd_array;
11272 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
11274 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
11275 tree ptr = build_fold_addr_expr (array);
11276 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
11277 build_int_cst (ptype, 0));
11278 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
11279 gimplify_and_add (t, &seq);
11283 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
11284 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
11285 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
11287 tree ptr = build_fold_addr_expr (array);
11291 arg = DECL_CHAIN (arg);
11295 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg))));
11296 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
11297 build_int_cst (ptype, k * elemsize));
11298 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
11299 gimplify_and_add (t, &seq);
11306 /* Callback info for ipa_simd_modify_stmt_ops below. */
11308 struct modify_stmt_info {
11309 ipa_parm_adjustment_vec adjustments;
11311 /* True if the parent statement was modified by
11312 ipa_simd_modify_stmt_ops. */
11316 /* Callback for walk_gimple_op.
11318 Adjust operands from a given statement as specified in the
11319 adjustments vector in the callback data. */
11322 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
11324 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
11325 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
11326 tree *orig_tp = tp;
11327 if (TREE_CODE (*tp) == ADDR_EXPR)
11328 tp = &TREE_OPERAND (*tp, 0);
11329 struct ipa_parm_adjustment *cand = NULL;
11330 if (TREE_CODE (*tp) == PARM_DECL)
11331 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
11335 *walk_subtrees = 0;
11338 tree repl = NULL_TREE;
11340 repl = unshare_expr (cand->new_decl);
11345 *walk_subtrees = 0;
11346 bool modified = info->modified;
11347 info->modified = false;
11348 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
11349 if (!info->modified)
11351 info->modified = modified;
11354 info->modified = modified;
11363 repl = build_fold_addr_expr (repl);
11365 = gimple_build_assign (make_ssa_name (TREE_TYPE (repl), NULL), repl);
11366 repl = gimple_assign_lhs (stmt);
11367 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
11368 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
11371 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
11373 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
11379 info->modified = true;
11383 /* Traverse the function body and perform all modifications as
11384 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11385 modified such that the replacement/reduction value will now be an
11386 offset into the corresponding simd_array.
11388 This function will replace all function argument uses with their
11389 corresponding simd array elements, and ajust the return values
11393 ipa_simd_modify_function_body (struct cgraph_node *node,
11394 ipa_parm_adjustment_vec adjustments,
11395 tree retval_array, tree iter)
11398 unsigned int i, j, l;
11400 /* Re-use the adjustments array, but this time use it to replace
11401 every function argument use to an offset into the corresponding
11403 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
11405 if (!node->simdclone->args[i].vector_arg)
11408 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
11409 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
11410 adjustments[j].new_decl
11411 = build4 (ARRAY_REF,
11413 node->simdclone->args[i].simd_array,
11415 NULL_TREE, NULL_TREE);
11416 if (adjustments[j].op == IPA_PARM_OP_NONE
11417 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
11418 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
11421 l = adjustments.length ();
11422 for (i = 1; i < num_ssa_names; i++)
11424 tree name = ssa_name (i);
11426 && SSA_NAME_VAR (name)
11427 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
11429 for (j = 0; j < l; j++)
11430 if (SSA_NAME_VAR (name) == adjustments[j].base
11431 && adjustments[j].new_decl)
11434 if (adjustments[j].new_ssa_base == NULL_TREE)
11437 = copy_var_decl (adjustments[j].base,
11438 DECL_NAME (adjustments[j].base),
11439 TREE_TYPE (adjustments[j].base));
11440 adjustments[j].new_ssa_base = base_var;
11443 base_var = adjustments[j].new_ssa_base;
11444 if (SSA_NAME_IS_DEFAULT_DEF (name))
11446 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
11447 gimple_stmt_iterator gsi = gsi_after_labels (bb);
11448 tree new_decl = unshare_expr (adjustments[j].new_decl);
11449 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
11450 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
11451 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
11452 gimple stmt = gimple_build_assign (name, new_decl);
11453 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
11456 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
11461 struct modify_stmt_info info;
11462 info.adjustments = adjustments;
11464 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
11466 gimple_stmt_iterator gsi;
11468 gsi = gsi_start_bb (bb);
11469 while (!gsi_end_p (gsi))
11471 gimple stmt = gsi_stmt (gsi);
11473 struct walk_stmt_info wi;
11475 memset (&wi, 0, sizeof (wi));
11476 info.modified = false;
11478 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
11480 if (gimple_code (stmt) == GIMPLE_RETURN)
11482 tree retval = gimple_return_retval (stmt);
11485 gsi_remove (&gsi, true);
11489 /* Replace `return foo' with `retval_array[iter] = foo'. */
11490 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
11491 retval_array, iter, NULL, NULL);
11492 stmt = gimple_build_assign (ref, retval);
11493 gsi_replace (&gsi, stmt, true);
11494 info.modified = true;
11499 update_stmt (stmt);
11500 if (maybe_clean_eh_stmt (stmt))
11501 gimple_purge_dead_eh_edges (gimple_bb (stmt));
11508 /* Adjust the argument types in NODE to their appropriate vector
11512 simd_clone_adjust (struct cgraph_node *node)
11514 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
11516 targetm.simd_clone.adjust (node);
11518 tree retval = simd_clone_adjust_return_type (node);
11519 ipa_parm_adjustment_vec adjustments
11520 = simd_clone_adjust_argument_types (node);
11522 push_gimplify_context ();
11524 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
11526 /* Adjust all uses of vector arguments accordingly. Adjust all
11527 return values accordingly. */
11528 tree iter = create_tmp_var (unsigned_type_node, "iter");
11529 tree iter1 = make_ssa_name (iter, NULL);
11530 tree iter2 = make_ssa_name (iter, NULL);
11531 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
11533 /* Initialize the iteration variable. */
11534 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
11535 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
11536 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
11537 /* Insert the SIMD array and iv initialization at function
11539 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
11541 pop_gimplify_context (NULL);
11543 /* Create a new BB right before the original exit BB, to hold the
11544 iteration increment and the condition/branch. */
11545 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
11546 basic_block incr_bb = create_empty_bb (orig_exit);
11547 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
11548 flag. Set it now to be a FALLTHRU_EDGE. */
11549 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
11550 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
11551 for (unsigned i = 0;
11552 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
11554 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
11555 redirect_edge_succ (e, incr_bb);
11557 edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
11558 e->probability = REG_BR_PROB_BASE;
11559 gsi = gsi_last_bb (incr_bb);
11560 gimple g = gimple_build_assign_with_ops (PLUS_EXPR, iter2, iter1,
11561 build_int_cst (unsigned_type_node,
11563 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11565 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
11566 struct loop *loop = alloc_loop ();
11567 cfun->has_force_vectorize_loops = true;
11568 loop->safelen = node->simdclone->simdlen;
11569 loop->force_vectorize = true;
11570 loop->header = body_bb;
11571 add_bb_to_loop (incr_bb, loop);
11573 /* Branch around the body if the mask applies. */
11574 if (node->simdclone->inbranch)
11576 gimple_stmt_iterator gsi = gsi_last_bb (loop->header);
11578 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
11579 tree mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)), NULL);
11580 tree aref = build4 (ARRAY_REF,
11581 TREE_TYPE (TREE_TYPE (mask_array)),
11584 g = gimple_build_assign (mask, aref);
11585 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11586 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref)));
11587 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
11589 aref = build1 (VIEW_CONVERT_EXPR,
11590 build_nonstandard_integer_type (bitsize, 0), mask);
11591 mask = make_ssa_name (TREE_TYPE (aref), NULL);
11592 g = gimple_build_assign (mask, aref);
11593 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11596 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
11598 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11599 make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
11600 FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE;
11603 /* Generate the condition. */
11604 g = gimple_build_cond (LT_EXPR,
11606 build_int_cst (unsigned_type_node,
11607 node->simdclone->simdlen),
11609 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11610 e = split_block (incr_bb, gsi_stmt (gsi));
11611 basic_block latch_bb = e->dest;
11612 basic_block new_exit_bb = e->dest;
11613 new_exit_bb = split_block (latch_bb, NULL)->dest;
11614 loop->latch = latch_bb;
11616 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
11618 make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
11619 /* The successor of incr_bb is already pointing to latch_bb; just
11621 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
11622 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
11624 gimple phi = create_phi_node (iter1, body_bb);
11625 edge preheader_edge = find_edge (entry_bb, body_bb);
11626 edge latch_edge = single_succ_edge (latch_bb);
11627 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
11629 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
11631 /* Generate the new return. */
11632 gsi = gsi_last_bb (new_exit_bb);
11634 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
11635 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
11636 retval = TREE_OPERAND (retval, 0);
11639 retval = build1 (VIEW_CONVERT_EXPR,
11640 TREE_TYPE (TREE_TYPE (node->decl)),
11642 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
11643 false, GSI_CONTINUE_LINKING);
11645 g = gimple_build_return (retval);
11646 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11648 /* Handle aligned clauses by replacing default defs of the aligned
11649 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
11650 lhs. Handle linear by adding PHIs. */
11651 for (unsigned i = 0; i < node->simdclone->nargs; i++)
11652 if (node->simdclone->args[i].alignment
11653 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
11654 && (node->simdclone->args[i].alignment
11655 & (node->simdclone->args[i].alignment - 1)) == 0
11656 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
11659 unsigned int alignment = node->simdclone->args[i].alignment;
11660 tree orig_arg = node->simdclone->args[i].orig_arg;
11661 tree def = ssa_default_def (cfun, orig_arg);
11662 if (def && !has_zero_uses (def))
11664 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
11665 gimple_seq seq = NULL;
11666 bool need_cvt = false;
11668 = gimple_build_call (fn, 2, def, size_int (alignment));
11670 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
11673 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg, NULL);
11674 gimple_call_set_lhs (g, t);
11675 gimple_seq_add_stmt_without_update (&seq, g);
11678 t = make_ssa_name (orig_arg, NULL);
11679 g = gimple_build_assign_with_ops (NOP_EXPR, t,
11680 gimple_call_lhs (g),
11682 gimple_seq_add_stmt_without_update (&seq, g);
11684 gsi_insert_seq_on_edge_immediate
11685 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
11687 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
11688 int freq = compute_call_stmt_bb_frequency (current_function_decl,
11690 cgraph_create_edge (node, cgraph_get_create_node (fn),
11691 call, entry_bb->count, freq);
11693 imm_use_iterator iter;
11694 use_operand_p use_p;
11696 tree repl = gimple_get_lhs (g);
11697 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
11698 if (is_gimple_debug (use_stmt) || use_stmt == call)
11701 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
11702 SET_USE (use_p, repl);
11705 else if (node->simdclone->args[i].arg_type
11706 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
11708 tree orig_arg = node->simdclone->args[i].orig_arg;
11709 tree def = ssa_default_def (cfun, orig_arg);
11710 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
11711 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
11712 if (def && !has_zero_uses (def))
11714 iter1 = make_ssa_name (orig_arg, NULL);
11715 iter2 = make_ssa_name (orig_arg, NULL);
11716 phi = create_phi_node (iter1, body_bb);
11717 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
11718 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
11719 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
11720 ? PLUS_EXPR : POINTER_PLUS_EXPR;
11721 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
11722 ? TREE_TYPE (orig_arg) : sizetype;
11724 = build_int_cst (addtype, node->simdclone->args[i].linear_step);
11725 g = gimple_build_assign_with_ops (code, iter2, iter1, addcst);
11726 gsi = gsi_last_bb (incr_bb);
11727 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
11729 imm_use_iterator iter;
11730 use_operand_p use_p;
11732 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
11733 if (use_stmt == phi)
11736 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
11737 SET_USE (use_p, iter1);
11741 calculate_dominance_info (CDI_DOMINATORS);
11742 add_loop (loop, loop->header->loop_father);
11743 update_ssa (TODO_update_ssa);
11748 /* If the function in NODE is tagged as an elemental SIMD function,
11749 create the appropriate SIMD clones. */
11752 expand_simd_clones (struct cgraph_node *node)
11754 tree attr = lookup_attribute ("omp declare simd",
11755 DECL_ATTRIBUTES (node->decl));
11756 if (attr == NULL_TREE
11757 || node->global.inlined_to
11758 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
11762 #pragma omp declare simd
11764 in C, there we don't know the argument types at all. */
11765 if (!node->definition
11766 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
11771 /* Start with parsing the "omp declare simd" attribute(s). */
11772 bool inbranch_clause_specified;
11773 struct cgraph_simd_clone *clone_info
11774 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
11775 &inbranch_clause_specified);
11776 if (clone_info == NULL)
11779 int orig_simdlen = clone_info->simdlen;
11780 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
11781 /* The target can return 0 (no simd clones should be created),
11782 1 (just one ISA of simd clones should be created) or higher
11783 count of ISA variants. In that case, clone_info is initialized
11784 for the first ISA variant. */
11786 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
11791 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
11792 also create one inbranch and one !inbranch clone of it. */
11793 for (int i = 0; i < count * 2; i++)
11795 struct cgraph_simd_clone *clone = clone_info;
11796 if (inbranch_clause_specified && (i & 1) != 0)
11801 clone = simd_clone_struct_alloc (clone_info->nargs
11803 simd_clone_struct_copy (clone, clone_info);
11804 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
11805 and simd_clone_adjust_argument_types did to the first
11807 clone->nargs -= clone_info->inbranch;
11808 clone->simdlen = orig_simdlen;
11809 /* And call the target hook again to get the right ISA. */
11810 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
11814 clone->inbranch = 1;
11817 /* simd_clone_mangle might fail if such a clone has been created
11819 tree id = simd_clone_mangle (node, clone);
11820 if (id == NULL_TREE)
11823 /* Only when we are sure we want to create the clone actually
11824 clone the function (or definitions) or create another
11825 extern FUNCTION_DECL (for prototypes without definitions). */
11826 struct cgraph_node *n = simd_clone_create (node);
11830 n->simdclone = clone;
11831 clone->origin = node;
11832 clone->next_clone = NULL;
11833 if (node->simd_clones == NULL)
11835 clone->prev_clone = n;
11836 node->simd_clones = n;
11840 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
11841 clone->prev_clone->simdclone->next_clone = n;
11842 node->simd_clones->simdclone->prev_clone = n;
11844 change_decl_assembler_name (n->decl, id);
11845 /* And finally adjust the return type, parameters and for
11846 definitions also function body. */
11847 if (node->definition)
11848 simd_clone_adjust (n);
11851 simd_clone_adjust_return_type (n);
11852 simd_clone_adjust_argument_types (n);
11856 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
11859 /* Entry point for IPA simd clone creation pass. */
11861 static unsigned int
11862 ipa_omp_simd_clone (void)
11864 struct cgraph_node *node;
11865 FOR_EACH_FUNCTION (node)
11866 expand_simd_clones (node);
11872 const pass_data pass_data_omp_simd_clone =
11874 SIMPLE_IPA_PASS, /* type */
11875 "simdclone", /* name */
11876 OPTGROUP_NONE, /* optinfo_flags */
11877 true, /* has_execute */
11878 TV_NONE, /* tv_id */
11879 ( PROP_ssa | PROP_cfg ), /* properties_required */
11880 0, /* properties_provided */
11881 0, /* properties_destroyed */
11882 0, /* todo_flags_start */
11883 0, /* todo_flags_finish */
11886 class pass_omp_simd_clone : public simple_ipa_opt_pass
11889 pass_omp_simd_clone(gcc::context *ctxt)
11890 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
11893 /* opt_pass methods: */
11894 virtual bool gate (function *);
11895 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
11899 pass_omp_simd_clone::gate (function *)
11901 return ((flag_openmp || flag_openmp_simd
11903 || (in_lto_p && !flag_wpa))
11904 && (targetm.simd_clone.compute_vecsize_and_simdlen != NULL));
11907 } // anon namespace
11909 simple_ipa_opt_pass *
11910 make_pass_omp_simd_clone (gcc::context *ctxt)
11912 return new pass_omp_simd_clone (ctxt);
11915 #include "gt-omp-low.h"