1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2013 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
31 #include "tree-iterator.h"
32 #include "tree-inline.h"
33 #include "langhooks.h"
34 #include "diagnostic-core.h"
35 #include "tree-flow.h"
39 #include "tree-pass.h"
42 #include "splay-tree.h"
47 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
48 phases. The first phase scans the function looking for OMP statements
49 and then for variables that must be replaced to satisfy data sharing
50 clauses. The second phase expands code for the constructs, as well as
51 re-gimplifying things when variables have been replaced with complex
54 Final code generation is done by pass_expand_omp. The flowgraph is
55 scanned for parallel regions which are then moved to a new
56 function, to be invoked by the thread library. */
58 /* Context structure. Used to store information about each parallel
59 directive in the code. */
61 typedef struct omp_context
63 /* This field must be at the beginning, as we do "inheritance": Some
64 callback functions for tree-inline.c (e.g., omp_copy_decl)
65 receive a copy_body_data pointer that is up-casted to an
66 omp_context pointer. */
69 /* The tree of contexts corresponding to the encountered constructs. */
70 struct omp_context *outer;
73 /* Map variables to fields in a structure that allows communication
74 between sending and receiving threads. */
80 /* These are used just by task contexts, if task firstprivate fn is
81 needed. srecord_type is used to communicate from the thread
82 that encountered the task construct to task firstprivate fn,
83 record_type is allocated by GOMP_task, initialized by task firstprivate
84 fn and passed to the task body fn. */
85 splay_tree sfield_map;
88 /* A chain of variables to add to the top-level block surrounding the
89 construct. In the case of a parallel, this is in the child function. */
92 /* What to do with variables with implicitly determined sharing
94 enum omp_clause_default_kind default_kind;
96 /* Nesting depth of this context. Used to beautify error messages re
97 invalid gotos. The outermost ctx is depth 1, with depth 0 being
98 reserved for the main body of the function. */
101 /* True if this parallel directive is nested within another. */
106 struct omp_for_data_loop
108 tree v, n1, n2, step;
109 enum tree_code cond_code;
112 /* A structure describing the main elements of a parallel loop. */
116 struct omp_for_data_loop loop;
121 bool have_nowait, have_ordered;
122 enum omp_clause_schedule_kind sched_kind;
123 struct omp_for_data_loop *loops;
127 static splay_tree all_contexts;
128 static int taskreg_nesting_level;
129 struct omp_region *root_omp_region;
130 static bitmap task_shared_vars;
132 static void scan_omp (gimple_seq *, omp_context *);
133 static tree scan_omp_1_op (tree *, int *, void *);
135 #define WALK_SUBSTMTS \
139 case GIMPLE_EH_FILTER: \
140 case GIMPLE_TRANSACTION: \
141 /* The sub-statements for these should be walked. */ \
142 *handled_ops_p = false; \
145 /* Convenience function for calling scan_omp_1_op on tree operands. */
148 scan_omp_op (tree *tp, omp_context *ctx)
150 struct walk_stmt_info wi;
152 memset (&wi, 0, sizeof (wi));
154 wi.want_locations = true;
156 return walk_tree (tp, scan_omp_1_op, &wi, NULL);
159 static void lower_omp (gimple_seq *, omp_context *);
160 static tree lookup_decl_in_outer_ctx (tree, omp_context *);
161 static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
163 /* Find an OpenMP clause of type KIND within CLAUSES. */
166 find_omp_clause (tree clauses, enum omp_clause_code kind)
168 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
169 if (OMP_CLAUSE_CODE (clauses) == kind)
175 /* Return true if CTX is for an omp parallel. */
178 is_parallel_ctx (omp_context *ctx)
180 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
184 /* Return true if CTX is for an omp task. */
187 is_task_ctx (omp_context *ctx)
189 return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
193 /* Return true if CTX is for an omp parallel or omp task. */
196 is_taskreg_ctx (omp_context *ctx)
198 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL
199 || gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
203 /* Return true if REGION is a combined parallel+workshare region. */
206 is_combined_parallel (struct omp_region *region)
208 return region->is_combined_parallel;
212 /* Extract the header elements of parallel loop FOR_STMT and store
216 extract_omp_for_data (gimple for_stmt, struct omp_for_data *fd,
217 struct omp_for_data_loop *loops)
219 tree t, var, *collapse_iter, *collapse_count;
220 tree count = NULL_TREE, iter_type = long_integer_type_node;
221 struct omp_for_data_loop *loop;
223 struct omp_for_data_loop dummy_loop;
224 location_t loc = gimple_location (for_stmt);
226 fd->for_stmt = for_stmt;
228 fd->collapse = gimple_omp_for_collapse (for_stmt);
229 if (fd->collapse > 1)
232 fd->loops = &fd->loop;
234 fd->have_nowait = fd->have_ordered = false;
235 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
236 fd->chunk_size = NULL_TREE;
237 collapse_iter = NULL;
238 collapse_count = NULL;
240 for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t))
241 switch (OMP_CLAUSE_CODE (t))
243 case OMP_CLAUSE_NOWAIT:
244 fd->have_nowait = true;
246 case OMP_CLAUSE_ORDERED:
247 fd->have_ordered = true;
249 case OMP_CLAUSE_SCHEDULE:
250 fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t);
251 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t);
253 case OMP_CLAUSE_COLLAPSE:
254 if (fd->collapse > 1)
256 collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t);
257 collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t);
263 /* FIXME: for now map schedule(auto) to schedule(static).
264 There should be analysis to determine whether all iterations
265 are approximately the same amount of work (then schedule(static)
266 is best) or if it varies (then schedule(dynamic,N) is better). */
267 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO)
269 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
270 gcc_assert (fd->chunk_size == NULL);
272 gcc_assert (fd->collapse == 1 || collapse_iter != NULL);
273 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
274 gcc_assert (fd->chunk_size == NULL);
275 else if (fd->chunk_size == NULL)
277 /* We only need to compute a default chunk size for ordered
278 static loops and dynamic loops. */
279 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
282 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
283 ? integer_zero_node : integer_one_node;
286 for (i = 0; i < fd->collapse; i++)
288 if (fd->collapse == 1)
290 else if (loops != NULL)
296 loop->v = gimple_omp_for_index (for_stmt, i);
297 gcc_assert (SSA_VAR_P (loop->v));
298 gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
299 || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE);
300 var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v;
301 loop->n1 = gimple_omp_for_initial (for_stmt, i);
303 loop->cond_code = gimple_omp_for_cond (for_stmt, i);
304 loop->n2 = gimple_omp_for_final (for_stmt, i);
305 switch (loop->cond_code)
311 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
312 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, 1);
314 loop->n2 = fold_build2_loc (loc,
315 PLUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
316 build_int_cst (TREE_TYPE (loop->n2), 1));
317 loop->cond_code = LT_EXPR;
320 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
321 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, -1);
323 loop->n2 = fold_build2_loc (loc,
324 MINUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
325 build_int_cst (TREE_TYPE (loop->n2), 1));
326 loop->cond_code = GT_EXPR;
332 t = gimple_omp_for_incr (for_stmt, i);
333 gcc_assert (TREE_OPERAND (t, 0) == var);
334 switch (TREE_CODE (t))
337 loop->step = TREE_OPERAND (t, 1);
339 case POINTER_PLUS_EXPR:
340 loop->step = fold_convert (ssizetype, TREE_OPERAND (t, 1));
343 loop->step = TREE_OPERAND (t, 1);
344 loop->step = fold_build1_loc (loc,
345 NEGATE_EXPR, TREE_TYPE (loop->step),
352 if (iter_type != long_long_unsigned_type_node)
354 if (POINTER_TYPE_P (TREE_TYPE (loop->v)))
355 iter_type = long_long_unsigned_type_node;
356 else if (TYPE_UNSIGNED (TREE_TYPE (loop->v))
357 && TYPE_PRECISION (TREE_TYPE (loop->v))
358 >= TYPE_PRECISION (iter_type))
362 if (loop->cond_code == LT_EXPR)
363 n = fold_build2_loc (loc,
364 PLUS_EXPR, TREE_TYPE (loop->v),
365 loop->n2, loop->step);
368 if (TREE_CODE (n) != INTEGER_CST
369 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n))
370 iter_type = long_long_unsigned_type_node;
372 else if (TYPE_PRECISION (TREE_TYPE (loop->v))
373 > TYPE_PRECISION (iter_type))
377 if (loop->cond_code == LT_EXPR)
380 n2 = fold_build2_loc (loc,
381 PLUS_EXPR, TREE_TYPE (loop->v),
382 loop->n2, loop->step);
386 n1 = fold_build2_loc (loc,
387 MINUS_EXPR, TREE_TYPE (loop->v),
388 loop->n2, loop->step);
391 if (TREE_CODE (n1) != INTEGER_CST
392 || TREE_CODE (n2) != INTEGER_CST
393 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1)
394 || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type)))
395 iter_type = long_long_unsigned_type_node;
399 if (collapse_count && *collapse_count == NULL)
401 t = fold_binary (loop->cond_code, boolean_type_node,
402 fold_convert (TREE_TYPE (loop->v), loop->n1),
403 fold_convert (TREE_TYPE (loop->v), loop->n2));
404 if (t && integer_zerop (t))
405 count = build_zero_cst (long_long_unsigned_type_node);
406 else if ((i == 0 || count != NULL_TREE)
407 && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
408 && TREE_CONSTANT (loop->n1)
409 && TREE_CONSTANT (loop->n2)
410 && TREE_CODE (loop->step) == INTEGER_CST)
412 tree itype = TREE_TYPE (loop->v);
414 if (POINTER_TYPE_P (itype))
415 itype = signed_type_for (itype);
416 t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1));
417 t = fold_build2_loc (loc,
419 fold_convert_loc (loc, itype, loop->step), t);
420 t = fold_build2_loc (loc, PLUS_EXPR, itype, t,
421 fold_convert_loc (loc, itype, loop->n2));
422 t = fold_build2_loc (loc, MINUS_EXPR, itype, t,
423 fold_convert_loc (loc, itype, loop->n1));
424 if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR)
425 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype,
426 fold_build1_loc (loc, NEGATE_EXPR, itype, t),
427 fold_build1_loc (loc, NEGATE_EXPR, itype,
428 fold_convert_loc (loc, itype,
431 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t,
432 fold_convert_loc (loc, itype, loop->step));
433 t = fold_convert_loc (loc, long_long_unsigned_type_node, t);
434 if (count != NULL_TREE)
435 count = fold_build2_loc (loc,
436 MULT_EXPR, long_long_unsigned_type_node,
440 if (TREE_CODE (count) != INTEGER_CST)
443 else if (count && !integer_zerop (count))
450 if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node)))
451 iter_type = long_long_unsigned_type_node;
453 iter_type = long_integer_type_node;
455 else if (collapse_iter && *collapse_iter != NULL)
456 iter_type = TREE_TYPE (*collapse_iter);
457 fd->iter_type = iter_type;
458 if (collapse_iter && *collapse_iter == NULL)
459 *collapse_iter = create_tmp_var (iter_type, ".iter");
460 if (collapse_count && *collapse_count == NULL)
463 *collapse_count = fold_convert_loc (loc, iter_type, count);
465 *collapse_count = create_tmp_var (iter_type, ".count");
468 if (fd->collapse > 1)
470 fd->loop.v = *collapse_iter;
471 fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0);
472 fd->loop.n2 = *collapse_count;
473 fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1);
474 fd->loop.cond_code = LT_EXPR;
479 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
480 is the immediate dominator of PAR_ENTRY_BB, return true if there
481 are no data dependencies that would prevent expanding the parallel
482 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
484 When expanding a combined parallel+workshare region, the call to
485 the child function may need additional arguments in the case of
486 GIMPLE_OMP_FOR regions. In some cases, these arguments are
487 computed out of variables passed in from the parent to the child
488 via 'struct .omp_data_s'. For instance:
490 #pragma omp parallel for schedule (guided, i * 4)
495 # BLOCK 2 (PAR_ENTRY_BB)
497 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
499 # BLOCK 3 (WS_ENTRY_BB)
500 .omp_data_i = &.omp_data_o;
501 D.1667 = .omp_data_i->i;
503 #pragma omp for schedule (guided, D.1598)
505 When we outline the parallel region, the call to the child function
506 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
507 that value is computed *after* the call site. So, in principle we
508 cannot do the transformation.
510 To see whether the code in WS_ENTRY_BB blocks the combined
511 parallel+workshare call, we collect all the variables used in the
512 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
513 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
516 FIXME. If we had the SSA form built at this point, we could merely
517 hoist the code in block 3 into block 2 and be done with it. But at
518 this point we don't have dataflow information and though we could
519 hack something up here, it is really not worth the aggravation. */
522 workshare_safe_to_combine_p (basic_block ws_entry_bb)
524 struct omp_for_data fd;
525 gimple ws_stmt = last_stmt (ws_entry_bb);
527 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
530 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
532 extract_omp_for_data (ws_stmt, &fd, NULL);
534 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
536 if (fd.iter_type != long_integer_type_node)
539 /* FIXME. We give up too easily here. If any of these arguments
540 are not constants, they will likely involve variables that have
541 been mapped into fields of .omp_data_s for sharing with the child
542 function. With appropriate data flow, it would be possible to
544 if (!is_gimple_min_invariant (fd.loop.n1)
545 || !is_gimple_min_invariant (fd.loop.n2)
546 || !is_gimple_min_invariant (fd.loop.step)
547 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
554 /* Collect additional arguments needed to emit a combined
555 parallel+workshare call. WS_STMT is the workshare directive being
558 static vec<tree, va_gc> *
559 get_ws_args_for (gimple ws_stmt)
562 location_t loc = gimple_location (ws_stmt);
563 vec<tree, va_gc> *ws_args;
565 if (gimple_code (ws_stmt) == GIMPLE_OMP_FOR)
567 struct omp_for_data fd;
569 extract_omp_for_data (ws_stmt, &fd, NULL);
571 vec_alloc (ws_args, 3 + (fd.chunk_size != 0));
573 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.n1);
574 ws_args->quick_push (t);
576 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.n2);
577 ws_args->quick_push (t);
579 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step);
580 ws_args->quick_push (t);
584 t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size);
585 ws_args->quick_push (t);
590 else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
592 /* Number of sections is equal to the number of edges from the
593 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
594 the exit of the sections region. */
595 basic_block bb = single_succ (gimple_bb (ws_stmt));
596 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
597 vec_alloc (ws_args, 1);
598 ws_args->quick_push (t);
606 /* Discover whether REGION is a combined parallel+workshare region. */
609 determine_parallel_type (struct omp_region *region)
611 basic_block par_entry_bb, par_exit_bb;
612 basic_block ws_entry_bb, ws_exit_bb;
614 if (region == NULL || region->inner == NULL
615 || region->exit == NULL || region->inner->exit == NULL
616 || region->inner->cont == NULL)
619 /* We only support parallel+for and parallel+sections. */
620 if (region->type != GIMPLE_OMP_PARALLEL
621 || (region->inner->type != GIMPLE_OMP_FOR
622 && region->inner->type != GIMPLE_OMP_SECTIONS))
625 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
626 WS_EXIT_BB -> PAR_EXIT_BB. */
627 par_entry_bb = region->entry;
628 par_exit_bb = region->exit;
629 ws_entry_bb = region->inner->entry;
630 ws_exit_bb = region->inner->exit;
632 if (single_succ (par_entry_bb) == ws_entry_bb
633 && single_succ (ws_exit_bb) == par_exit_bb
634 && workshare_safe_to_combine_p (ws_entry_bb)
635 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb))
636 || (last_and_only_stmt (ws_entry_bb)
637 && last_and_only_stmt (par_exit_bb))))
639 gimple ws_stmt = last_stmt (ws_entry_bb);
641 if (region->inner->type == GIMPLE_OMP_FOR)
643 /* If this is a combined parallel loop, we need to determine
644 whether or not to use the combined library calls. There
645 are two cases where we do not apply the transformation:
646 static loops and any kind of ordered loop. In the first
647 case, we already open code the loop so there is no need
648 to do anything else. In the latter case, the combined
649 parallel loop call would still need extra synchronization
650 to implement ordered semantics, so there would not be any
651 gain in using the combined call. */
652 tree clauses = gimple_omp_for_clauses (ws_stmt);
653 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE);
655 || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC
656 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED))
658 region->is_combined_parallel = false;
659 region->inner->is_combined_parallel = false;
664 region->is_combined_parallel = true;
665 region->inner->is_combined_parallel = true;
666 region->ws_args = get_ws_args_for (ws_stmt);
671 /* Return true if EXPR is variable sized. */
674 is_variable_sized (const_tree expr)
676 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
679 /* Return true if DECL is a reference type. */
682 is_reference (tree decl)
684 return lang_hooks.decls.omp_privatize_by_reference (decl);
687 /* Lookup variables in the decl or field splay trees. The "maybe" form
688 allows for the variable form to not have been entered, otherwise we
689 assert that the variable must have been entered. */
692 lookup_decl (tree var, omp_context *ctx)
695 n = (tree *) pointer_map_contains (ctx->cb.decl_map, var);
700 maybe_lookup_decl (const_tree var, omp_context *ctx)
703 n = (tree *) pointer_map_contains (ctx->cb.decl_map, var);
704 return n ? *n : NULL_TREE;
708 lookup_field (tree var, omp_context *ctx)
711 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
712 return (tree) n->value;
716 lookup_sfield (tree var, omp_context *ctx)
719 n = splay_tree_lookup (ctx->sfield_map
720 ? ctx->sfield_map : ctx->field_map,
721 (splay_tree_key) var);
722 return (tree) n->value;
726 maybe_lookup_field (tree var, omp_context *ctx)
729 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
730 return n ? (tree) n->value : NULL_TREE;
733 /* Return true if DECL should be copied by pointer. SHARED_CTX is
734 the parallel context if DECL is to be shared. */
737 use_pointer_for_field (tree decl, omp_context *shared_ctx)
739 if (AGGREGATE_TYPE_P (TREE_TYPE (decl)))
742 /* We can only use copy-in/copy-out semantics for shared variables
743 when we know the value is not accessible from an outer scope. */
746 /* ??? Trivially accessible from anywhere. But why would we even
747 be passing an address in this case? Should we simply assert
748 this to be false, or should we have a cleanup pass that removes
749 these from the list of mappings? */
750 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
753 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
754 without analyzing the expression whether or not its location
755 is accessible to anyone else. In the case of nested parallel
756 regions it certainly may be. */
757 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
760 /* Do not use copy-in/copy-out for variables that have their
762 if (TREE_ADDRESSABLE (decl))
765 /* lower_send_shared_vars only uses copy-in, but not copy-out
767 if (TREE_READONLY (decl)
768 || ((TREE_CODE (decl) == RESULT_DECL
769 || TREE_CODE (decl) == PARM_DECL)
770 && DECL_BY_REFERENCE (decl)))
773 /* Disallow copy-in/out in nested parallel if
774 decl is shared in outer parallel, otherwise
775 each thread could store the shared variable
776 in its own copy-in location, making the
777 variable no longer really shared. */
778 if (shared_ctx->is_nested)
782 for (up = shared_ctx->outer; up; up = up->outer)
783 if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up))
790 for (c = gimple_omp_taskreg_clauses (up->stmt);
791 c; c = OMP_CLAUSE_CHAIN (c))
792 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
793 && OMP_CLAUSE_DECL (c) == decl)
797 goto maybe_mark_addressable_and_ret;
801 /* For tasks avoid using copy-in/out. As tasks can be
802 deferred or executed in different thread, when GOMP_task
803 returns, the task hasn't necessarily terminated. */
804 if (is_task_ctx (shared_ctx))
807 maybe_mark_addressable_and_ret:
808 outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
809 if (is_gimple_reg (outer))
811 /* Taking address of OUTER in lower_send_shared_vars
812 might need regimplification of everything that uses the
814 if (!task_shared_vars)
815 task_shared_vars = BITMAP_ALLOC (NULL);
816 bitmap_set_bit (task_shared_vars, DECL_UID (outer));
817 TREE_ADDRESSABLE (outer) = 1;
826 /* Create a new VAR_DECL and copy information from VAR to it. */
829 copy_var_decl (tree var, tree name, tree type)
831 tree copy = build_decl (DECL_SOURCE_LOCATION (var), VAR_DECL, name, type);
833 TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (var);
834 TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (var);
835 DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (var);
836 DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (var);
837 DECL_IGNORED_P (copy) = DECL_IGNORED_P (var);
838 DECL_CONTEXT (copy) = DECL_CONTEXT (var);
839 TREE_USED (copy) = 1;
840 DECL_SEEN_IN_BIND_EXPR_P (copy) = 1;
845 /* Construct a new automatic decl similar to VAR. */
848 omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
850 tree copy = copy_var_decl (var, name, type);
852 DECL_CONTEXT (copy) = current_function_decl;
853 DECL_CHAIN (copy) = ctx->block_vars;
854 ctx->block_vars = copy;
860 omp_copy_decl_1 (tree var, omp_context *ctx)
862 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
865 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
868 omp_build_component_ref (tree obj, tree field)
870 tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL);
871 if (TREE_THIS_VOLATILE (field))
872 TREE_THIS_VOLATILE (ret) |= 1;
873 if (TREE_READONLY (field))
874 TREE_READONLY (ret) |= 1;
878 /* Build tree nodes to access the field for VAR on the receiver side. */
881 build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
883 tree x, field = lookup_field (var, ctx);
885 /* If the receiver record type was remapped in the child function,
886 remap the field into the new record type. */
887 x = maybe_lookup_field (field, ctx);
891 x = build_simple_mem_ref (ctx->receiver_decl);
892 x = omp_build_component_ref (x, field);
894 x = build_simple_mem_ref (x);
899 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
900 of a parallel, this is a component reference; for workshare constructs
901 this is some variable. */
904 build_outer_var_ref (tree var, omp_context *ctx)
908 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
910 else if (is_variable_sized (var))
912 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
913 x = build_outer_var_ref (x, ctx);
914 x = build_simple_mem_ref (x);
916 else if (is_taskreg_ctx (ctx))
918 bool by_ref = use_pointer_for_field (var, NULL);
919 x = build_receiver_ref (var, by_ref, ctx);
922 x = lookup_decl (var, ctx->outer);
923 else if (is_reference (var))
924 /* This can happen with orphaned constructs. If var is reference, it is
925 possible it is shared and as such valid. */
930 if (is_reference (var))
931 x = build_simple_mem_ref (x);
936 /* Build tree nodes to access the field for VAR on the sender side. */
939 build_sender_ref (tree var, omp_context *ctx)
941 tree field = lookup_sfield (var, ctx);
942 return omp_build_component_ref (ctx->sender_decl, field);
945 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
948 install_var_field (tree var, bool by_ref, int mask, omp_context *ctx)
950 tree field, type, sfield = NULL_TREE;
952 gcc_assert ((mask & 1) == 0
953 || !splay_tree_lookup (ctx->field_map, (splay_tree_key) var));
954 gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
955 || !splay_tree_lookup (ctx->sfield_map, (splay_tree_key) var));
957 type = TREE_TYPE (var);
959 type = build_pointer_type (type);
960 else if ((mask & 3) == 1 && is_reference (var))
961 type = TREE_TYPE (type);
963 field = build_decl (DECL_SOURCE_LOCATION (var),
964 FIELD_DECL, DECL_NAME (var), type);
966 /* Remember what variable this field was created for. This does have a
967 side effect of making dwarf2out ignore this member, so for helpful
968 debugging we clear it later in delete_omp_context. */
969 DECL_ABSTRACT_ORIGIN (field) = var;
970 if (type == TREE_TYPE (var))
972 DECL_ALIGN (field) = DECL_ALIGN (var);
973 DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
974 TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
977 DECL_ALIGN (field) = TYPE_ALIGN (type);
981 insert_field_into_struct (ctx->record_type, field);
982 if (ctx->srecord_type)
984 sfield = build_decl (DECL_SOURCE_LOCATION (var),
985 FIELD_DECL, DECL_NAME (var), type);
986 DECL_ABSTRACT_ORIGIN (sfield) = var;
987 DECL_ALIGN (sfield) = DECL_ALIGN (field);
988 DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
989 TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
990 insert_field_into_struct (ctx->srecord_type, sfield);
995 if (ctx->srecord_type == NULL_TREE)
999 ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
1000 ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1001 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
1003 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1004 FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
1005 DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
1006 insert_field_into_struct (ctx->srecord_type, sfield);
1007 splay_tree_insert (ctx->sfield_map,
1008 (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
1009 (splay_tree_value) sfield);
1013 insert_field_into_struct ((mask & 1) ? ctx->record_type
1014 : ctx->srecord_type, field);
1018 splay_tree_insert (ctx->field_map, (splay_tree_key) var,
1019 (splay_tree_value) field);
1020 if ((mask & 2) && ctx->sfield_map)
1021 splay_tree_insert (ctx->sfield_map, (splay_tree_key) var,
1022 (splay_tree_value) sfield);
1026 install_var_local (tree var, omp_context *ctx)
1028 tree new_var = omp_copy_decl_1 (var, ctx);
1029 insert_decl_map (&ctx->cb, var, new_var);
1033 /* Adjust the replacement for DECL in CTX for the new context. This means
1034 copying the DECL_VALUE_EXPR, and fixing up the type. */
1037 fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
1039 tree new_decl, size;
1041 new_decl = lookup_decl (decl, ctx);
1043 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
1045 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
1046 && DECL_HAS_VALUE_EXPR_P (decl))
1048 tree ve = DECL_VALUE_EXPR (decl);
1049 walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
1050 SET_DECL_VALUE_EXPR (new_decl, ve);
1051 DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
1054 if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
1056 size = remap_decl (DECL_SIZE (decl), &ctx->cb);
1057 if (size == error_mark_node)
1058 size = TYPE_SIZE (TREE_TYPE (new_decl));
1059 DECL_SIZE (new_decl) = size;
1061 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
1062 if (size == error_mark_node)
1063 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
1064 DECL_SIZE_UNIT (new_decl) = size;
1068 /* The callback for remap_decl. Search all containing contexts for a
1069 mapping of the variable; this avoids having to duplicate the splay
1070 tree ahead of time. We know a mapping doesn't already exist in the
1071 given context. Create new mappings to implement default semantics. */
1074 omp_copy_decl (tree var, copy_body_data *cb)
1076 omp_context *ctx = (omp_context *) cb;
1079 if (TREE_CODE (var) == LABEL_DECL)
1081 new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
1082 DECL_CONTEXT (new_var) = current_function_decl;
1083 insert_decl_map (&ctx->cb, var, new_var);
1087 while (!is_taskreg_ctx (ctx))
1092 new_var = maybe_lookup_decl (var, ctx);
1097 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
1100 return error_mark_node;
1104 /* Return the parallel region associated with STMT. */
1106 /* Debugging dumps for parallel regions. */
1107 void dump_omp_region (FILE *, struct omp_region *, int);
1108 void debug_omp_region (struct omp_region *);
1109 void debug_all_omp_regions (void);
1111 /* Dump the parallel region tree rooted at REGION. */
1114 dump_omp_region (FILE *file, struct omp_region *region, int indent)
1116 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
1117 gimple_code_name[region->type]);
1120 dump_omp_region (file, region->inner, indent + 4);
1124 fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "",
1125 region->cont->index);
1129 fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "",
1130 region->exit->index);
1132 fprintf (file, "%*s[no exit marker]\n", indent, "");
1135 dump_omp_region (file, region->next, indent);
1139 debug_omp_region (struct omp_region *region)
1141 dump_omp_region (stderr, region, 0);
1145 debug_all_omp_regions (void)
1147 dump_omp_region (stderr, root_omp_region, 0);
1151 /* Create a new parallel region starting at STMT inside region PARENT. */
1154 new_omp_region (basic_block bb, enum gimple_code type,
1155 struct omp_region *parent)
1157 struct omp_region *region = XCNEW (struct omp_region);
1159 region->outer = parent;
1161 region->type = type;
1165 /* This is a nested region. Add it to the list of inner
1166 regions in PARENT. */
1167 region->next = parent->inner;
1168 parent->inner = region;
1172 /* This is a toplevel region. Add it to the list of toplevel
1173 regions in ROOT_OMP_REGION. */
1174 region->next = root_omp_region;
1175 root_omp_region = region;
1181 /* Release the memory associated with the region tree rooted at REGION. */
1184 free_omp_region_1 (struct omp_region *region)
1186 struct omp_region *i, *n;
1188 for (i = region->inner; i ; i = n)
1191 free_omp_region_1 (i);
1197 /* Release the memory for the entire omp region tree. */
1200 free_omp_regions (void)
1202 struct omp_region *r, *n;
1203 for (r = root_omp_region; r ; r = n)
1206 free_omp_region_1 (r);
1208 root_omp_region = NULL;
1212 /* Create a new context, with OUTER_CTX being the surrounding context. */
1214 static omp_context *
1215 new_omp_context (gimple stmt, omp_context *outer_ctx)
1217 omp_context *ctx = XCNEW (omp_context);
1219 splay_tree_insert (all_contexts, (splay_tree_key) stmt,
1220 (splay_tree_value) ctx);
1225 ctx->outer = outer_ctx;
1226 ctx->cb = outer_ctx->cb;
1227 ctx->cb.block = NULL;
1228 ctx->depth = outer_ctx->depth + 1;
1232 ctx->cb.src_fn = current_function_decl;
1233 ctx->cb.dst_fn = current_function_decl;
1234 ctx->cb.src_node = cgraph_get_node (current_function_decl);
1235 gcc_checking_assert (ctx->cb.src_node);
1236 ctx->cb.dst_node = ctx->cb.src_node;
1237 ctx->cb.src_cfun = cfun;
1238 ctx->cb.copy_decl = omp_copy_decl;
1239 ctx->cb.eh_lp_nr = 0;
1240 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1244 ctx->cb.decl_map = pointer_map_create ();
1249 static gimple_seq maybe_catch_exception (gimple_seq);
1251 /* Finalize task copyfn. */
1254 finalize_task_copyfn (gimple task_stmt)
1256 struct function *child_cfun;
1258 gimple_seq seq = NULL, new_seq;
1261 child_fn = gimple_omp_task_copy_fn (task_stmt);
1262 if (child_fn == NULL_TREE)
1265 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1267 /* Inform the callgraph about the new function. */
1268 DECL_STRUCT_FUNCTION (child_fn)->curr_properties
1269 = cfun->curr_properties & ~PROP_loops;
1271 push_cfun (child_cfun);
1272 bind = gimplify_body (child_fn, false);
1273 gimple_seq_add_stmt (&seq, bind);
1274 new_seq = maybe_catch_exception (seq);
1277 bind = gimple_build_bind (NULL, new_seq, NULL);
1279 gimple_seq_add_stmt (&seq, bind);
1281 gimple_set_body (child_fn, seq);
1284 cgraph_add_new_function (child_fn, false);
1287 /* Destroy a omp_context data structures. Called through the splay tree
1288 value delete callback. */
1291 delete_omp_context (splay_tree_value value)
1293 omp_context *ctx = (omp_context *) value;
1295 pointer_map_destroy (ctx->cb.decl_map);
1298 splay_tree_delete (ctx->field_map);
1299 if (ctx->sfield_map)
1300 splay_tree_delete (ctx->sfield_map);
1302 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1303 it produces corrupt debug information. */
1304 if (ctx->record_type)
1307 for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1308 DECL_ABSTRACT_ORIGIN (t) = NULL;
1310 if (ctx->srecord_type)
1313 for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1314 DECL_ABSTRACT_ORIGIN (t) = NULL;
1317 if (is_task_ctx (ctx))
1318 finalize_task_copyfn (ctx->stmt);
1323 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1327 fixup_child_record_type (omp_context *ctx)
1329 tree f, type = ctx->record_type;
1331 /* ??? It isn't sufficient to just call remap_type here, because
1332 variably_modified_type_p doesn't work the way we expect for
1333 record types. Testing each field for whether it needs remapping
1334 and creating a new record by hand works, however. */
1335 for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1336 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1340 tree name, new_fields = NULL;
1342 type = lang_hooks.types.make_type (RECORD_TYPE);
1343 name = DECL_NAME (TYPE_NAME (ctx->record_type));
1344 name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1345 TYPE_DECL, name, type);
1346 TYPE_NAME (type) = name;
1348 for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1350 tree new_f = copy_node (f);
1351 DECL_CONTEXT (new_f) = type;
1352 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1353 DECL_CHAIN (new_f) = new_fields;
1354 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1355 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1357 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1361 /* Arrange to be able to look up the receiver field
1362 given the sender field. */
1363 splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1364 (splay_tree_value) new_f);
1366 TYPE_FIELDS (type) = nreverse (new_fields);
1370 TREE_TYPE (ctx->receiver_decl) = build_pointer_type (type);
1373 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1374 specified by CLAUSES. */
1377 scan_sharing_clauses (tree clauses, omp_context *ctx)
1380 bool scan_array_reductions = false;
1382 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1386 switch (OMP_CLAUSE_CODE (c))
1388 case OMP_CLAUSE_PRIVATE:
1389 decl = OMP_CLAUSE_DECL (c);
1390 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1392 else if (!is_variable_sized (decl))
1393 install_var_local (decl, ctx);
1396 case OMP_CLAUSE_SHARED:
1397 gcc_assert (is_taskreg_ctx (ctx));
1398 decl = OMP_CLAUSE_DECL (c);
1399 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1400 || !is_variable_sized (decl));
1401 /* Global variables don't need to be copied,
1402 the receiver side will use them directly. */
1403 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1405 by_ref = use_pointer_for_field (decl, ctx);
1406 if (! TREE_READONLY (decl)
1407 || TREE_ADDRESSABLE (decl)
1409 || is_reference (decl))
1411 install_var_field (decl, by_ref, 3, ctx);
1412 install_var_local (decl, ctx);
1415 /* We don't need to copy const scalar vars back. */
1416 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1419 case OMP_CLAUSE_LASTPRIVATE:
1420 /* Let the corresponding firstprivate clause create
1422 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1426 case OMP_CLAUSE_FIRSTPRIVATE:
1427 case OMP_CLAUSE_REDUCTION:
1428 decl = OMP_CLAUSE_DECL (c);
1430 if (is_variable_sized (decl))
1432 if (is_task_ctx (ctx))
1433 install_var_field (decl, false, 1, ctx);
1436 else if (is_taskreg_ctx (ctx))
1439 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1440 by_ref = use_pointer_for_field (decl, NULL);
1442 if (is_task_ctx (ctx)
1443 && (global || by_ref || is_reference (decl)))
1445 install_var_field (decl, false, 1, ctx);
1447 install_var_field (decl, by_ref, 2, ctx);
1450 install_var_field (decl, by_ref, 3, ctx);
1452 install_var_local (decl, ctx);
1455 case OMP_CLAUSE_COPYPRIVATE:
1456 case OMP_CLAUSE_COPYIN:
1457 decl = OMP_CLAUSE_DECL (c);
1458 by_ref = use_pointer_for_field (decl, NULL);
1459 install_var_field (decl, by_ref, 3, ctx);
1462 case OMP_CLAUSE_DEFAULT:
1463 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c);
1466 case OMP_CLAUSE_FINAL:
1468 case OMP_CLAUSE_NUM_THREADS:
1469 case OMP_CLAUSE_SCHEDULE:
1471 scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1474 case OMP_CLAUSE_NOWAIT:
1475 case OMP_CLAUSE_ORDERED:
1476 case OMP_CLAUSE_COLLAPSE:
1477 case OMP_CLAUSE_UNTIED:
1478 case OMP_CLAUSE_MERGEABLE:
1486 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1488 switch (OMP_CLAUSE_CODE (c))
1490 case OMP_CLAUSE_LASTPRIVATE:
1491 /* Let the corresponding firstprivate clause create
1493 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1494 scan_array_reductions = true;
1495 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1499 case OMP_CLAUSE_PRIVATE:
1500 case OMP_CLAUSE_FIRSTPRIVATE:
1501 case OMP_CLAUSE_REDUCTION:
1502 decl = OMP_CLAUSE_DECL (c);
1503 if (is_variable_sized (decl))
1504 install_var_local (decl, ctx);
1505 fixup_remapped_decl (decl, ctx,
1506 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1507 && OMP_CLAUSE_PRIVATE_DEBUG (c));
1508 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1509 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1510 scan_array_reductions = true;
1513 case OMP_CLAUSE_SHARED:
1514 decl = OMP_CLAUSE_DECL (c);
1515 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1516 fixup_remapped_decl (decl, ctx, false);
1519 case OMP_CLAUSE_COPYPRIVATE:
1520 case OMP_CLAUSE_COPYIN:
1521 case OMP_CLAUSE_DEFAULT:
1523 case OMP_CLAUSE_NUM_THREADS:
1524 case OMP_CLAUSE_SCHEDULE:
1525 case OMP_CLAUSE_NOWAIT:
1526 case OMP_CLAUSE_ORDERED:
1527 case OMP_CLAUSE_COLLAPSE:
1528 case OMP_CLAUSE_UNTIED:
1529 case OMP_CLAUSE_FINAL:
1530 case OMP_CLAUSE_MERGEABLE:
1538 if (scan_array_reductions)
1539 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1540 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1541 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1543 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx);
1544 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
1546 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
1547 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1548 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
1551 /* Create a new name for omp child function. Returns an identifier. */
1553 static GTY(()) unsigned int tmp_ompfn_id_num;
1556 create_omp_child_function_name (bool task_copy)
1558 return (clone_function_name (current_function_decl,
1559 task_copy ? "_omp_cpyfn" : "_omp_fn"));
1562 /* Build a decl for the omp child function. It'll not contain a body
1563 yet, just the bare decl. */
1566 create_omp_child_function (omp_context *ctx, bool task_copy)
1568 tree decl, type, name, t;
1570 name = create_omp_child_function_name (task_copy);
1572 type = build_function_type_list (void_type_node, ptr_type_node,
1573 ptr_type_node, NULL_TREE);
1575 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
1577 decl = build_decl (gimple_location (ctx->stmt),
1578 FUNCTION_DECL, name, type);
1581 ctx->cb.dst_fn = decl;
1583 gimple_omp_task_set_copy_fn (ctx->stmt, decl);
1585 TREE_STATIC (decl) = 1;
1586 TREE_USED (decl) = 1;
1587 DECL_ARTIFICIAL (decl) = 1;
1588 DECL_NAMELESS (decl) = 1;
1589 DECL_IGNORED_P (decl) = 0;
1590 TREE_PUBLIC (decl) = 0;
1591 DECL_UNINLINABLE (decl) = 1;
1592 DECL_EXTERNAL (decl) = 0;
1593 DECL_CONTEXT (decl) = NULL_TREE;
1594 DECL_INITIAL (decl) = make_node (BLOCK);
1596 t = build_decl (DECL_SOURCE_LOCATION (decl),
1597 RESULT_DECL, NULL_TREE, void_type_node);
1598 DECL_ARTIFICIAL (t) = 1;
1599 DECL_IGNORED_P (t) = 1;
1600 DECL_CONTEXT (t) = decl;
1601 DECL_RESULT (decl) = t;
1603 t = build_decl (DECL_SOURCE_LOCATION (decl),
1604 PARM_DECL, get_identifier (".omp_data_i"), ptr_type_node);
1605 DECL_ARTIFICIAL (t) = 1;
1606 DECL_NAMELESS (t) = 1;
1607 DECL_ARG_TYPE (t) = ptr_type_node;
1608 DECL_CONTEXT (t) = current_function_decl;
1610 DECL_ARGUMENTS (decl) = t;
1612 ctx->receiver_decl = t;
1615 t = build_decl (DECL_SOURCE_LOCATION (decl),
1616 PARM_DECL, get_identifier (".omp_data_o"),
1618 DECL_ARTIFICIAL (t) = 1;
1619 DECL_NAMELESS (t) = 1;
1620 DECL_ARG_TYPE (t) = ptr_type_node;
1621 DECL_CONTEXT (t) = current_function_decl;
1623 TREE_ADDRESSABLE (t) = 1;
1624 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
1625 DECL_ARGUMENTS (decl) = t;
1628 /* Allocate memory for the function structure. The call to
1629 allocate_struct_function clobbers CFUN, so we need to restore
1631 push_struct_function (decl);
1632 cfun->function_end_locus = gimple_location (ctx->stmt);
1637 /* Scan an OpenMP parallel directive. */
1640 scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
1644 gimple stmt = gsi_stmt (*gsi);
1646 /* Ignore parallel directives with empty bodies, unless there
1647 are copyin clauses. */
1649 && empty_body_p (gimple_omp_body (stmt))
1650 && find_omp_clause (gimple_omp_parallel_clauses (stmt),
1651 OMP_CLAUSE_COPYIN) == NULL)
1653 gsi_replace (gsi, gimple_build_nop (), false);
1657 ctx = new_omp_context (stmt, outer_ctx);
1658 if (taskreg_nesting_level > 1)
1659 ctx->is_nested = true;
1660 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1661 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
1662 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
1663 name = create_tmp_var_name (".omp_data_s");
1664 name = build_decl (gimple_location (stmt),
1665 TYPE_DECL, name, ctx->record_type);
1666 DECL_ARTIFICIAL (name) = 1;
1667 DECL_NAMELESS (name) = 1;
1668 TYPE_NAME (ctx->record_type) = name;
1669 create_omp_child_function (ctx, false);
1670 gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
1672 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
1673 scan_omp (gimple_omp_body_ptr (stmt), ctx);
1675 if (TYPE_FIELDS (ctx->record_type) == NULL)
1676 ctx->record_type = ctx->receiver_decl = NULL;
1679 layout_type (ctx->record_type);
1680 fixup_child_record_type (ctx);
1684 /* Scan an OpenMP task directive. */
1687 scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
1691 gimple stmt = gsi_stmt (*gsi);
1692 location_t loc = gimple_location (stmt);
1694 /* Ignore task directives with empty bodies. */
1696 && empty_body_p (gimple_omp_body (stmt)))
1698 gsi_replace (gsi, gimple_build_nop (), false);
1702 ctx = new_omp_context (stmt, outer_ctx);
1703 if (taskreg_nesting_level > 1)
1704 ctx->is_nested = true;
1705 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1706 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
1707 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
1708 name = create_tmp_var_name (".omp_data_s");
1709 name = build_decl (gimple_location (stmt),
1710 TYPE_DECL, name, ctx->record_type);
1711 DECL_ARTIFICIAL (name) = 1;
1712 DECL_NAMELESS (name) = 1;
1713 TYPE_NAME (ctx->record_type) = name;
1714 create_omp_child_function (ctx, false);
1715 gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
1717 scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
1719 if (ctx->srecord_type)
1721 name = create_tmp_var_name (".omp_data_a");
1722 name = build_decl (gimple_location (stmt),
1723 TYPE_DECL, name, ctx->srecord_type);
1724 DECL_ARTIFICIAL (name) = 1;
1725 DECL_NAMELESS (name) = 1;
1726 TYPE_NAME (ctx->srecord_type) = name;
1727 create_omp_child_function (ctx, true);
1730 scan_omp (gimple_omp_body_ptr (stmt), ctx);
1732 if (TYPE_FIELDS (ctx->record_type) == NULL)
1734 ctx->record_type = ctx->receiver_decl = NULL;
1735 t = build_int_cst (long_integer_type_node, 0);
1736 gimple_omp_task_set_arg_size (stmt, t);
1737 t = build_int_cst (long_integer_type_node, 1);
1738 gimple_omp_task_set_arg_align (stmt, t);
1742 tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
1743 /* Move VLA fields to the end. */
1744 p = &TYPE_FIELDS (ctx->record_type);
1746 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
1747 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
1750 *p = TREE_CHAIN (*p);
1751 TREE_CHAIN (*q) = NULL_TREE;
1752 q = &TREE_CHAIN (*q);
1755 p = &DECL_CHAIN (*p);
1757 layout_type (ctx->record_type);
1758 fixup_child_record_type (ctx);
1759 if (ctx->srecord_type)
1760 layout_type (ctx->srecord_type);
1761 t = fold_convert_loc (loc, long_integer_type_node,
1762 TYPE_SIZE_UNIT (ctx->record_type));
1763 gimple_omp_task_set_arg_size (stmt, t);
1764 t = build_int_cst (long_integer_type_node,
1765 TYPE_ALIGN_UNIT (ctx->record_type));
1766 gimple_omp_task_set_arg_align (stmt, t);
1771 /* Scan an OpenMP loop directive. */
1774 scan_omp_for (gimple stmt, omp_context *outer_ctx)
1779 ctx = new_omp_context (stmt, outer_ctx);
1781 scan_sharing_clauses (gimple_omp_for_clauses (stmt), ctx);
1783 scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
1784 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
1786 scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
1787 scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
1788 scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
1789 scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
1791 scan_omp (gimple_omp_body_ptr (stmt), ctx);
1794 /* Scan an OpenMP sections directive. */
1797 scan_omp_sections (gimple stmt, omp_context *outer_ctx)
1801 ctx = new_omp_context (stmt, outer_ctx);
1802 scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
1803 scan_omp (gimple_omp_body_ptr (stmt), ctx);
1806 /* Scan an OpenMP single directive. */
1809 scan_omp_single (gimple stmt, omp_context *outer_ctx)
1814 ctx = new_omp_context (stmt, outer_ctx);
1815 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1816 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
1817 name = create_tmp_var_name (".omp_copy_s");
1818 name = build_decl (gimple_location (stmt),
1819 TYPE_DECL, name, ctx->record_type);
1820 TYPE_NAME (ctx->record_type) = name;
1822 scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
1823 scan_omp (gimple_omp_body_ptr (stmt), ctx);
1825 if (TYPE_FIELDS (ctx->record_type) == NULL)
1826 ctx->record_type = NULL;
1828 layout_type (ctx->record_type);
1832 /* Check OpenMP nesting restrictions. */
1834 check_omp_nesting_restrictions (gimple stmt, omp_context *ctx)
1836 switch (gimple_code (stmt))
1838 case GIMPLE_OMP_FOR:
1839 case GIMPLE_OMP_SECTIONS:
1840 case GIMPLE_OMP_SINGLE:
1842 for (; ctx != NULL; ctx = ctx->outer)
1843 switch (gimple_code (ctx->stmt))
1845 case GIMPLE_OMP_FOR:
1846 case GIMPLE_OMP_SECTIONS:
1847 case GIMPLE_OMP_SINGLE:
1848 case GIMPLE_OMP_ORDERED:
1849 case GIMPLE_OMP_MASTER:
1850 case GIMPLE_OMP_TASK:
1851 if (is_gimple_call (stmt))
1853 error_at (gimple_location (stmt),
1854 "barrier region may not be closely nested inside "
1855 "of work-sharing, critical, ordered, master or "
1856 "explicit task region");
1859 error_at (gimple_location (stmt),
1860 "work-sharing region may not be closely nested inside "
1861 "of work-sharing, critical, ordered, master or explicit "
1864 case GIMPLE_OMP_PARALLEL:
1870 case GIMPLE_OMP_MASTER:
1871 for (; ctx != NULL; ctx = ctx->outer)
1872 switch (gimple_code (ctx->stmt))
1874 case GIMPLE_OMP_FOR:
1875 case GIMPLE_OMP_SECTIONS:
1876 case GIMPLE_OMP_SINGLE:
1877 case GIMPLE_OMP_TASK:
1878 error_at (gimple_location (stmt),
1879 "master region may not be closely nested inside "
1880 "of work-sharing or explicit task region");
1882 case GIMPLE_OMP_PARALLEL:
1888 case GIMPLE_OMP_ORDERED:
1889 for (; ctx != NULL; ctx = ctx->outer)
1890 switch (gimple_code (ctx->stmt))
1892 case GIMPLE_OMP_CRITICAL:
1893 case GIMPLE_OMP_TASK:
1894 error_at (gimple_location (stmt),
1895 "ordered region may not be closely nested inside "
1896 "of critical or explicit task region");
1898 case GIMPLE_OMP_FOR:
1899 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
1900 OMP_CLAUSE_ORDERED) == NULL)
1902 error_at (gimple_location (stmt),
1903 "ordered region must be closely nested inside "
1904 "a loop region with an ordered clause");
1908 case GIMPLE_OMP_PARALLEL:
1914 case GIMPLE_OMP_CRITICAL:
1915 for (; ctx != NULL; ctx = ctx->outer)
1916 if (gimple_code (ctx->stmt) == GIMPLE_OMP_CRITICAL
1917 && (gimple_omp_critical_name (stmt)
1918 == gimple_omp_critical_name (ctx->stmt)))
1920 error_at (gimple_location (stmt),
1921 "critical region may not be nested inside a critical "
1922 "region with the same name");
1933 /* Helper function scan_omp.
1935 Callback for walk_tree or operators in walk_gimple_stmt used to
1936 scan for OpenMP directives in TP. */
1939 scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
1941 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
1942 omp_context *ctx = (omp_context *) wi->info;
1945 switch (TREE_CODE (t))
1952 *tp = remap_decl (t, &ctx->cb);
1956 if (ctx && TYPE_P (t))
1957 *tp = remap_type (t, &ctx->cb);
1958 else if (!DECL_P (t))
1963 tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
1964 if (tem != TREE_TYPE (t))
1966 if (TREE_CODE (t) == INTEGER_CST)
1967 *tp = build_int_cst_wide (tem,
1968 TREE_INT_CST_LOW (t),
1969 TREE_INT_CST_HIGH (t));
1971 TREE_TYPE (t) = tem;
1982 /* Helper function for scan_omp.
1984 Callback for walk_gimple_stmt used to scan for OpenMP directives in
1985 the current statement in GSI. */
1988 scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
1989 struct walk_stmt_info *wi)
1991 gimple stmt = gsi_stmt (*gsi);
1992 omp_context *ctx = (omp_context *) wi->info;
1994 if (gimple_has_location (stmt))
1995 input_location = gimple_location (stmt);
1997 /* Check the OpenMP nesting restrictions. */
2000 bool remove = false;
2001 if (is_gimple_omp (stmt))
2002 remove = !check_omp_nesting_restrictions (stmt, ctx);
2003 else if (is_gimple_call (stmt))
2005 tree fndecl = gimple_call_fndecl (stmt);
2006 if (fndecl && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
2007 && DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
2008 remove = !check_omp_nesting_restrictions (stmt, ctx);
2012 stmt = gimple_build_nop ();
2013 gsi_replace (gsi, stmt, false);
2017 *handled_ops_p = true;
2019 switch (gimple_code (stmt))
2021 case GIMPLE_OMP_PARALLEL:
2022 taskreg_nesting_level++;
2023 scan_omp_parallel (gsi, ctx);
2024 taskreg_nesting_level--;
2027 case GIMPLE_OMP_TASK:
2028 taskreg_nesting_level++;
2029 scan_omp_task (gsi, ctx);
2030 taskreg_nesting_level--;
2033 case GIMPLE_OMP_FOR:
2034 scan_omp_for (stmt, ctx);
2037 case GIMPLE_OMP_SECTIONS:
2038 scan_omp_sections (stmt, ctx);
2041 case GIMPLE_OMP_SINGLE:
2042 scan_omp_single (stmt, ctx);
2045 case GIMPLE_OMP_SECTION:
2046 case GIMPLE_OMP_MASTER:
2047 case GIMPLE_OMP_ORDERED:
2048 case GIMPLE_OMP_CRITICAL:
2049 ctx = new_omp_context (stmt, ctx);
2050 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2057 *handled_ops_p = false;
2059 for (var = gimple_bind_vars (stmt); var ; var = DECL_CHAIN (var))
2060 insert_decl_map (&ctx->cb, var, var);
2064 *handled_ops_p = false;
2072 /* Scan all the statements starting at the current statement. CTX
2073 contains context information about the OpenMP directives and
2074 clauses found during the scan. */
2077 scan_omp (gimple_seq *body_p, omp_context *ctx)
2079 location_t saved_location;
2080 struct walk_stmt_info wi;
2082 memset (&wi, 0, sizeof (wi));
2084 wi.want_locations = true;
2086 saved_location = input_location;
2087 walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
2088 input_location = saved_location;
2091 /* Re-gimplification and code generation routines. */
2093 /* Build a call to GOMP_barrier. */
2096 build_omp_barrier (void)
2098 return build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_BARRIER), 0);
2101 /* If a context was created for STMT when it was scanned, return it. */
2103 static omp_context *
2104 maybe_lookup_ctx (gimple stmt)
2107 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
2108 return n ? (omp_context *) n->value : NULL;
2112 /* Find the mapping for DECL in CTX or the immediately enclosing
2113 context that has a mapping for DECL.
2115 If CTX is a nested parallel directive, we may have to use the decl
2116 mappings created in CTX's parent context. Suppose that we have the
2117 following parallel nesting (variable UIDs showed for clarity):
2120 #omp parallel shared(iD.1562) -> outer parallel
2121 iD.1562 = iD.1562 + 1;
2123 #omp parallel shared (iD.1562) -> inner parallel
2124 iD.1562 = iD.1562 - 1;
2126 Each parallel structure will create a distinct .omp_data_s structure
2127 for copying iD.1562 in/out of the directive:
2129 outer parallel .omp_data_s.1.i -> iD.1562
2130 inner parallel .omp_data_s.2.i -> iD.1562
2132 A shared variable mapping will produce a copy-out operation before
2133 the parallel directive and a copy-in operation after it. So, in
2134 this case we would have:
2137 .omp_data_o.1.i = iD.1562;
2138 #omp parallel shared(iD.1562) -> outer parallel
2139 .omp_data_i.1 = &.omp_data_o.1
2140 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2142 .omp_data_o.2.i = iD.1562; -> **
2143 #omp parallel shared(iD.1562) -> inner parallel
2144 .omp_data_i.2 = &.omp_data_o.2
2145 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2148 ** This is a problem. The symbol iD.1562 cannot be referenced
2149 inside the body of the outer parallel region. But since we are
2150 emitting this copy operation while expanding the inner parallel
2151 directive, we need to access the CTX structure of the outer
2152 parallel directive to get the correct mapping:
2154 .omp_data_o.2.i = .omp_data_i.1->i
2156 Since there may be other workshare or parallel directives enclosing
2157 the parallel directive, it may be necessary to walk up the context
2158 parent chain. This is not a problem in general because nested
2159 parallelism happens only rarely. */
2162 lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
2167 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
2168 t = maybe_lookup_decl (decl, up);
2170 gcc_assert (!ctx->is_nested || t || is_global_var (decl));
2172 return t ? t : decl;
2176 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2177 in outer contexts. */
2180 maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
2185 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
2186 t = maybe_lookup_decl (decl, up);
2188 return t ? t : decl;
2192 /* Construct the initialization value for reduction CLAUSE. */
2195 omp_reduction_init (tree clause, tree type)
2197 location_t loc = OMP_CLAUSE_LOCATION (clause);
2198 switch (OMP_CLAUSE_REDUCTION_CODE (clause))
2205 case TRUTH_ORIF_EXPR:
2206 case TRUTH_XOR_EXPR:
2208 return build_zero_cst (type);
2211 case TRUTH_AND_EXPR:
2212 case TRUTH_ANDIF_EXPR:
2214 return fold_convert_loc (loc, type, integer_one_node);
2217 return fold_convert_loc (loc, type, integer_minus_one_node);
2220 if (SCALAR_FLOAT_TYPE_P (type))
2222 REAL_VALUE_TYPE max, min;
2223 if (HONOR_INFINITIES (TYPE_MODE (type)))
2226 real_arithmetic (&min, NEGATE_EXPR, &max, NULL);
2229 real_maxval (&min, 1, TYPE_MODE (type));
2230 return build_real (type, min);
2234 gcc_assert (INTEGRAL_TYPE_P (type));
2235 return TYPE_MIN_VALUE (type);
2239 if (SCALAR_FLOAT_TYPE_P (type))
2241 REAL_VALUE_TYPE max;
2242 if (HONOR_INFINITIES (TYPE_MODE (type)))
2245 real_maxval (&max, 0, TYPE_MODE (type));
2246 return build_real (type, max);
2250 gcc_assert (INTEGRAL_TYPE_P (type));
2251 return TYPE_MAX_VALUE (type);
2259 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
2260 from the receiver (aka child) side and initializers for REFERENCE_TYPE
2261 private variables. Initialization statements go in ILIST, while calls
2262 to destructors go in DLIST. */
2265 lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
2268 tree c, dtor, copyin_seq, x, ptr;
2269 bool copyin_by_ref = false;
2270 bool lastprivate_firstprivate = false;
2275 /* Do all the fixed sized types in the first pass, and the variable sized
2276 types in the second pass. This makes sure that the scalar arguments to
2277 the variable sized types are processed before we use them in the
2278 variable sized operations. */
2279 for (pass = 0; pass < 2; ++pass)
2281 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
2283 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
2286 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
2290 case OMP_CLAUSE_PRIVATE:
2291 if (OMP_CLAUSE_PRIVATE_DEBUG (c))
2294 case OMP_CLAUSE_SHARED:
2295 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
2297 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c)));
2300 case OMP_CLAUSE_FIRSTPRIVATE:
2301 case OMP_CLAUSE_COPYIN:
2302 case OMP_CLAUSE_REDUCTION:
2304 case OMP_CLAUSE_LASTPRIVATE:
2305 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
2307 lastprivate_firstprivate = true;
2316 new_var = var = OMP_CLAUSE_DECL (c);
2317 if (c_kind != OMP_CLAUSE_COPYIN)
2318 new_var = lookup_decl (var, ctx);
2320 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
2325 else if (is_variable_sized (var))
2327 /* For variable sized types, we need to allocate the
2328 actual storage here. Call alloca and store the
2329 result in the pointer decl that we created elsewhere. */
2333 if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
2338 ptr = DECL_VALUE_EXPR (new_var);
2339 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
2340 ptr = TREE_OPERAND (ptr, 0);
2341 gcc_assert (DECL_P (ptr));
2342 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
2344 /* void *tmp = __builtin_alloca */
2345 atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
2346 stmt = gimple_build_call (atmp, 1, x);
2347 tmp = create_tmp_var_raw (ptr_type_node, NULL);
2348 gimple_add_tmp_var (tmp);
2349 gimple_call_set_lhs (stmt, tmp);
2351 gimple_seq_add_stmt (ilist, stmt);
2353 x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
2354 gimplify_assign (ptr, x, ilist);
2357 else if (is_reference (var))
2359 /* For references that are being privatized for Fortran,
2360 allocate new backing storage for the new pointer
2361 variable. This allows us to avoid changing all the
2362 code that expects a pointer to something that expects
2363 a direct variable. Note that this doesn't apply to
2364 C++, since reference types are disallowed in data
2365 sharing clauses there, except for NRV optimized
2370 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
2371 if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
2373 x = build_receiver_ref (var, false, ctx);
2374 x = build_fold_addr_expr_loc (clause_loc, x);
2376 else if (TREE_CONSTANT (x))
2378 const char *name = NULL;
2379 if (DECL_NAME (var))
2380 name = IDENTIFIER_POINTER (DECL_NAME (new_var));
2382 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
2384 gimple_add_tmp_var (x);
2385 TREE_ADDRESSABLE (x) = 1;
2386 x = build_fold_addr_expr_loc (clause_loc, x);
2390 tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
2391 x = build_call_expr_loc (clause_loc, atmp, 1, x);
2394 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
2395 gimplify_assign (new_var, x, ilist);
2397 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
2399 else if (c_kind == OMP_CLAUSE_REDUCTION
2400 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2408 switch (OMP_CLAUSE_CODE (c))
2410 case OMP_CLAUSE_SHARED:
2411 /* Shared global vars are just accessed directly. */
2412 if (is_global_var (new_var))
2414 /* Set up the DECL_VALUE_EXPR for shared variables now. This
2415 needs to be delayed until after fixup_child_record_type so
2416 that we get the correct type during the dereference. */
2417 by_ref = use_pointer_for_field (var, ctx);
2418 x = build_receiver_ref (var, by_ref, ctx);
2419 SET_DECL_VALUE_EXPR (new_var, x);
2420 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
2422 /* ??? If VAR is not passed by reference, and the variable
2423 hasn't been initialized yet, then we'll get a warning for
2424 the store into the omp_data_s structure. Ideally, we'd be
2425 able to notice this and not store anything at all, but
2426 we're generating code too early. Suppress the warning. */
2428 TREE_NO_WARNING (var) = 1;
2431 case OMP_CLAUSE_LASTPRIVATE:
2432 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
2436 case OMP_CLAUSE_PRIVATE:
2437 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
2438 x = build_outer_var_ref (var, ctx);
2439 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
2441 if (is_task_ctx (ctx))
2442 x = build_receiver_ref (var, false, ctx);
2444 x = build_outer_var_ref (var, ctx);
2448 x = lang_hooks.decls.omp_clause_default_ctor (c, new_var, x);
2450 gimplify_and_add (x, ilist);
2454 x = lang_hooks.decls.omp_clause_dtor (c, new_var);
2457 gimple_seq tseq = NULL;
2460 gimplify_stmt (&dtor, &tseq);
2461 gimple_seq_add_seq (dlist, tseq);
2465 case OMP_CLAUSE_FIRSTPRIVATE:
2466 if (is_task_ctx (ctx))
2468 if (is_reference (var) || is_variable_sized (var))
2470 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
2472 || use_pointer_for_field (var, NULL))
2474 x = build_receiver_ref (var, false, ctx);
2475 SET_DECL_VALUE_EXPR (new_var, x);
2476 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
2480 x = build_outer_var_ref (var, ctx);
2481 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
2482 gimplify_and_add (x, ilist);
2486 case OMP_CLAUSE_COPYIN:
2487 by_ref = use_pointer_for_field (var, NULL);
2488 x = build_receiver_ref (var, by_ref, ctx);
2489 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
2490 append_to_statement_list (x, ©in_seq);
2491 copyin_by_ref |= by_ref;
2494 case OMP_CLAUSE_REDUCTION:
2495 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2497 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
2498 x = build_outer_var_ref (var, ctx);
2500 if (is_reference (var))
2501 x = build_fold_addr_expr_loc (clause_loc, x);
2502 SET_DECL_VALUE_EXPR (placeholder, x);
2503 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
2504 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx);
2505 gimple_seq_add_seq (ilist,
2506 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c));
2507 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
2508 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
2512 x = omp_reduction_init (c, TREE_TYPE (new_var));
2513 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
2514 gimplify_assign (new_var, x, ilist);
2524 /* The copyin sequence is not to be executed by the main thread, since
2525 that would result in self-copies. Perhaps not visible to scalars,
2526 but it certainly is to C++ operator=. */
2529 x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
2531 x = build2 (NE_EXPR, boolean_type_node, x,
2532 build_int_cst (TREE_TYPE (x), 0));
2533 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
2534 gimplify_and_add (x, ilist);
2537 /* If any copyin variable is passed by reference, we must ensure the
2538 master thread doesn't modify it before it is copied over in all
2539 threads. Similarly for variables in both firstprivate and
2540 lastprivate clauses we need to ensure the lastprivate copying
2541 happens after firstprivate copying in all threads. */
2542 if (copyin_by_ref || lastprivate_firstprivate)
2543 gimplify_and_add (build_omp_barrier (), ilist);
2547 /* Generate code to implement the LASTPRIVATE clauses. This is used for
2548 both parallel and workshare constructs. PREDICATE may be NULL if it's
2552 lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list,
2555 tree x, c, label = NULL;
2556 bool par_clauses = false;
2558 /* Early exit if there are no lastprivate clauses. */
2559 clauses = find_omp_clause (clauses, OMP_CLAUSE_LASTPRIVATE);
2560 if (clauses == NULL)
2562 /* If this was a workshare clause, see if it had been combined
2563 with its parallel. In that case, look for the clauses on the
2564 parallel statement itself. */
2565 if (is_parallel_ctx (ctx))
2569 if (ctx == NULL || !is_parallel_ctx (ctx))
2572 clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
2573 OMP_CLAUSE_LASTPRIVATE);
2574 if (clauses == NULL)
2582 tree label_true, arm1, arm2;
2584 label = create_artificial_label (UNKNOWN_LOCATION);
2585 label_true = create_artificial_label (UNKNOWN_LOCATION);
2586 arm1 = TREE_OPERAND (predicate, 0);
2587 arm2 = TREE_OPERAND (predicate, 1);
2588 gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
2589 gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
2590 stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2,
2592 gimple_seq_add_stmt (stmt_list, stmt);
2593 gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
2596 for (c = clauses; c ;)
2599 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
2601 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
2603 var = OMP_CLAUSE_DECL (c);
2604 new_var = lookup_decl (var, ctx);
2606 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2608 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
2609 gimple_seq_add_seq (stmt_list,
2610 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
2612 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
2614 x = build_outer_var_ref (var, ctx);
2615 if (is_reference (var))
2616 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
2617 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
2618 gimplify_and_add (x, stmt_list);
2620 c = OMP_CLAUSE_CHAIN (c);
2621 if (c == NULL && !par_clauses)
2623 /* If this was a workshare clause, see if it had been combined
2624 with its parallel. In that case, continue looking for the
2625 clauses also on the parallel statement itself. */
2626 if (is_parallel_ctx (ctx))
2630 if (ctx == NULL || !is_parallel_ctx (ctx))
2633 c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
2634 OMP_CLAUSE_LASTPRIVATE);
2640 gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
2644 /* Generate code to implement the REDUCTION clauses. */
2647 lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx)
2649 gimple_seq sub_seq = NULL;
2654 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
2655 update in that case, otherwise use a lock. */
2656 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
2657 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
2659 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2661 /* Never use OMP_ATOMIC for array reductions. */
2671 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
2673 tree var, ref, new_var;
2674 enum tree_code code;
2675 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
2677 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
2680 var = OMP_CLAUSE_DECL (c);
2681 new_var = lookup_decl (var, ctx);
2682 if (is_reference (var))
2683 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
2684 ref = build_outer_var_ref (var, ctx);
2685 code = OMP_CLAUSE_REDUCTION_CODE (c);
2687 /* reduction(-:var) sums up the partial results, so it acts
2688 identically to reduction(+:var). */
2689 if (code == MINUS_EXPR)
2694 tree addr = build_fold_addr_expr_loc (clause_loc, ref);
2696 addr = save_expr (addr);
2697 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
2698 x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var);
2699 x = build2 (OMP_ATOMIC, void_type_node, addr, x);
2700 gimplify_and_add (x, stmt_seqp);
2704 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2706 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
2708 if (is_reference (var))
2709 ref = build_fold_addr_expr_loc (clause_loc, ref);
2710 SET_DECL_VALUE_EXPR (placeholder, ref);
2711 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
2712 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
2713 gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
2714 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
2715 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
2719 x = build2 (code, TREE_TYPE (ref), ref, new_var);
2720 ref = build_outer_var_ref (var, ctx);
2721 gimplify_assign (ref, x, &sub_seq);
2725 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START),
2727 gimple_seq_add_stmt (stmt_seqp, stmt);
2729 gimple_seq_add_seq (stmt_seqp, sub_seq);
2731 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
2733 gimple_seq_add_stmt (stmt_seqp, stmt);
2737 /* Generate code to implement the COPYPRIVATE clauses. */
2740 lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
2745 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
2747 tree var, new_var, ref, x;
2749 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
2751 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
2754 var = OMP_CLAUSE_DECL (c);
2755 by_ref = use_pointer_for_field (var, NULL);
2757 ref = build_sender_ref (var, ctx);
2758 x = new_var = lookup_decl_in_outer_ctx (var, ctx);
2761 x = build_fold_addr_expr_loc (clause_loc, new_var);
2762 x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
2764 gimplify_assign (ref, x, slist);
2766 ref = build_receiver_ref (var, false, ctx);
2769 ref = fold_convert_loc (clause_loc,
2770 build_pointer_type (TREE_TYPE (new_var)),
2772 ref = build_fold_indirect_ref_loc (clause_loc, ref);
2774 if (is_reference (var))
2776 ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
2777 ref = build_simple_mem_ref_loc (clause_loc, ref);
2778 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
2780 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
2781 gimplify_and_add (x, rlist);
2786 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
2787 and REDUCTION from the sender (aka parent) side. */
2790 lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
2795 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
2797 tree val, ref, x, var;
2798 bool by_ref, do_in = false, do_out = false;
2799 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
2801 switch (OMP_CLAUSE_CODE (c))
2803 case OMP_CLAUSE_PRIVATE:
2804 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
2807 case OMP_CLAUSE_FIRSTPRIVATE:
2808 case OMP_CLAUSE_COPYIN:
2809 case OMP_CLAUSE_LASTPRIVATE:
2810 case OMP_CLAUSE_REDUCTION:
2816 val = OMP_CLAUSE_DECL (c);
2817 var = lookup_decl_in_outer_ctx (val, ctx);
2819 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
2820 && is_global_var (var))
2822 if (is_variable_sized (val))
2824 by_ref = use_pointer_for_field (val, NULL);
2826 switch (OMP_CLAUSE_CODE (c))
2828 case OMP_CLAUSE_PRIVATE:
2829 case OMP_CLAUSE_FIRSTPRIVATE:
2830 case OMP_CLAUSE_COPYIN:
2834 case OMP_CLAUSE_LASTPRIVATE:
2835 if (by_ref || is_reference (val))
2837 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
2844 if (lang_hooks.decls.omp_private_outer_ref (val))
2849 case OMP_CLAUSE_REDUCTION:
2851 do_out = !(by_ref || is_reference (val));
2860 ref = build_sender_ref (val, ctx);
2861 x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
2862 gimplify_assign (ref, x, ilist);
2863 if (is_task_ctx (ctx))
2864 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
2869 ref = build_sender_ref (val, ctx);
2870 gimplify_assign (var, ref, olist);
2875 /* Generate code to implement SHARED from the sender (aka parent)
2876 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
2877 list things that got automatically shared. */
2880 lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
2882 tree var, ovar, nvar, f, x, record_type;
2884 if (ctx->record_type == NULL)
2887 record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
2888 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
2890 ovar = DECL_ABSTRACT_ORIGIN (f);
2891 nvar = maybe_lookup_decl (ovar, ctx);
2892 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar))
2895 /* If CTX is a nested parallel directive. Find the immediately
2896 enclosing parallel or workshare construct that contains a
2897 mapping for OVAR. */
2898 var = lookup_decl_in_outer_ctx (ovar, ctx);
2900 if (use_pointer_for_field (ovar, ctx))
2902 x = build_sender_ref (ovar, ctx);
2903 var = build_fold_addr_expr (var);
2904 gimplify_assign (x, var, ilist);
2908 x = build_sender_ref (ovar, ctx);
2909 gimplify_assign (x, var, ilist);
2911 if (!TREE_READONLY (var)
2912 /* We don't need to receive a new reference to a result
2913 or parm decl. In fact we may not store to it as we will
2914 invalidate any pending RSO and generate wrong gimple
2916 && !((TREE_CODE (var) == RESULT_DECL
2917 || TREE_CODE (var) == PARM_DECL)
2918 && DECL_BY_REFERENCE (var)))
2920 x = build_sender_ref (ovar, ctx);
2921 gimplify_assign (var, x, olist);
2928 /* A convenience function to build an empty GIMPLE_COND with just the
2932 gimple_build_cond_empty (tree cond)
2934 enum tree_code pred_code;
2937 gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs);
2938 return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE);
2942 /* Build the function calls to GOMP_parallel_start etc to actually
2943 generate the parallel operation. REGION is the parallel region
2944 being expanded. BB is the block where to insert the code. WS_ARGS
2945 will be set if this is a call to a combined parallel+workshare
2946 construct, it contains the list of additional arguments needed by
2947 the workshare construct. */
2950 expand_parallel_call (struct omp_region *region, basic_block bb,
2951 gimple entry_stmt, vec<tree, va_gc> *ws_args)
2953 tree t, t1, t2, val, cond, c, clauses;
2954 gimple_stmt_iterator gsi;
2956 enum built_in_function start_ix;
2958 location_t clause_loc;
2959 vec<tree, va_gc> *args;
2961 clauses = gimple_omp_parallel_clauses (entry_stmt);
2963 /* Determine what flavor of GOMP_parallel_start we will be
2965 start_ix = BUILT_IN_GOMP_PARALLEL_START;
2966 if (is_combined_parallel (region))
2968 switch (region->inner->type)
2970 case GIMPLE_OMP_FOR:
2971 gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
2972 start_ix2 = ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC_START
2973 + (region->inner->sched_kind
2974 == OMP_CLAUSE_SCHEDULE_RUNTIME
2975 ? 3 : region->inner->sched_kind));
2976 start_ix = (enum built_in_function)start_ix2;
2978 case GIMPLE_OMP_SECTIONS:
2979 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS_START;
2986 /* By default, the value of NUM_THREADS is zero (selected at run time)
2987 and there is no conditional. */
2989 val = build_int_cst (unsigned_type_node, 0);
2991 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
2993 cond = OMP_CLAUSE_IF_EXPR (c);
2995 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS);
2998 val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
2999 clause_loc = OMP_CLAUSE_LOCATION (c);
3002 clause_loc = gimple_location (entry_stmt);
3004 /* Ensure 'val' is of the correct type. */
3005 val = fold_convert_loc (clause_loc, unsigned_type_node, val);
3007 /* If we found the clause 'if (cond)', build either
3008 (cond != 0) or (cond ? val : 1u). */
3011 gimple_stmt_iterator gsi;
3013 cond = gimple_boolify (cond);
3015 if (integer_zerop (val))
3016 val = fold_build2_loc (clause_loc,
3017 EQ_EXPR, unsigned_type_node, cond,
3018 build_int_cst (TREE_TYPE (cond), 0));
3021 basic_block cond_bb, then_bb, else_bb;
3022 edge e, e_then, e_else;
3023 tree tmp_then, tmp_else, tmp_join, tmp_var;
3025 tmp_var = create_tmp_var (TREE_TYPE (val), NULL);
3026 if (gimple_in_ssa_p (cfun))
3028 tmp_then = make_ssa_name (tmp_var, NULL);
3029 tmp_else = make_ssa_name (tmp_var, NULL);
3030 tmp_join = make_ssa_name (tmp_var, NULL);
3039 e = split_block (bb, NULL);
3044 then_bb = create_empty_bb (cond_bb);
3045 else_bb = create_empty_bb (then_bb);
3046 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
3047 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
3049 stmt = gimple_build_cond_empty (cond);
3050 gsi = gsi_start_bb (cond_bb);
3051 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3053 gsi = gsi_start_bb (then_bb);
3054 stmt = gimple_build_assign (tmp_then, val);
3055 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3057 gsi = gsi_start_bb (else_bb);
3058 stmt = gimple_build_assign
3059 (tmp_else, build_int_cst (unsigned_type_node, 1));
3060 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3062 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
3063 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
3064 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
3065 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
3067 if (gimple_in_ssa_p (cfun))
3069 gimple phi = create_phi_node (tmp_join, bb);
3070 add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION);
3071 add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION);
3077 gsi = gsi_start_bb (bb);
3078 val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE,
3079 false, GSI_CONTINUE_LINKING);
3082 gsi = gsi_last_bb (bb);
3083 t = gimple_omp_parallel_data_arg (entry_stmt);
3085 t1 = null_pointer_node;
3087 t1 = build_fold_addr_expr (t);
3088 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
3090 vec_alloc (args, 3 + vec_safe_length (ws_args));
3091 args->quick_push (t2);
3092 args->quick_push (t1);
3093 args->quick_push (val);
3095 args->splice (*ws_args);
3097 t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
3098 builtin_decl_explicit (start_ix), args);
3100 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
3101 false, GSI_CONTINUE_LINKING);
3103 t = gimple_omp_parallel_data_arg (entry_stmt);
3105 t = null_pointer_node;
3107 t = build_fold_addr_expr (t);
3108 t = build_call_expr_loc (gimple_location (entry_stmt),
3109 gimple_omp_parallel_child_fn (entry_stmt), 1, t);
3110 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
3111 false, GSI_CONTINUE_LINKING);
3113 t = build_call_expr_loc (gimple_location (entry_stmt),
3114 builtin_decl_explicit (BUILT_IN_GOMP_PARALLEL_END),
3116 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
3117 false, GSI_CONTINUE_LINKING);
3121 /* Build the function call to GOMP_task to actually
3122 generate the task operation. BB is the block where to insert the code. */
3125 expand_task_call (basic_block bb, gimple entry_stmt)
3127 tree t, t1, t2, t3, flags, cond, c, c2, clauses;
3128 gimple_stmt_iterator gsi;
3129 location_t loc = gimple_location (entry_stmt);
3131 clauses = gimple_omp_task_clauses (entry_stmt);
3133 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
3135 cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (c));
3137 cond = boolean_true_node;
3139 c = find_omp_clause (clauses, OMP_CLAUSE_UNTIED);
3140 c2 = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE);
3141 flags = build_int_cst (unsigned_type_node,
3142 (c ? 1 : 0) + (c2 ? 4 : 0));
3144 c = find_omp_clause (clauses, OMP_CLAUSE_FINAL);
3147 c = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c));
3148 c = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, c,
3149 build_int_cst (unsigned_type_node, 2),
3150 build_int_cst (unsigned_type_node, 0));
3151 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, c);
3154 gsi = gsi_last_bb (bb);
3155 t = gimple_omp_task_data_arg (entry_stmt);
3157 t2 = null_pointer_node;
3159 t2 = build_fold_addr_expr_loc (loc, t);
3160 t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt));
3161 t = gimple_omp_task_copy_fn (entry_stmt);
3163 t3 = null_pointer_node;
3165 t3 = build_fold_addr_expr_loc (loc, t);
3167 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
3169 gimple_omp_task_arg_size (entry_stmt),
3170 gimple_omp_task_arg_align (entry_stmt), cond, flags);
3172 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
3173 false, GSI_CONTINUE_LINKING);
3177 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
3178 catch handler and return it. This prevents programs from violating the
3179 structured block semantics with throws. */
3182 maybe_catch_exception (gimple_seq body)
3187 if (!flag_exceptions)
3190 if (lang_hooks.eh_protect_cleanup_actions != NULL)
3191 decl = lang_hooks.eh_protect_cleanup_actions ();
3193 decl = builtin_decl_explicit (BUILT_IN_TRAP);
3195 g = gimple_build_eh_must_not_throw (decl);
3196 g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
3199 return gimple_seq_alloc_with_stmt (g);
3202 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
3205 vec2chain (vec<tree, va_gc> *v)
3207 tree chain = NULL_TREE, t;
3210 FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t)
3212 DECL_CHAIN (t) = chain;
3220 /* Remove barriers in REGION->EXIT's block. Note that this is only
3221 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
3222 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
3223 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
3227 remove_exit_barrier (struct omp_region *region)
3229 gimple_stmt_iterator gsi;
3230 basic_block exit_bb;
3234 int any_addressable_vars = -1;
3236 exit_bb = region->exit;
3238 /* If the parallel region doesn't return, we don't have REGION->EXIT
3243 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
3244 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
3245 statements that can appear in between are extremely limited -- no
3246 memory operations at all. Here, we allow nothing at all, so the
3247 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
3248 gsi = gsi_last_bb (exit_bb);
3249 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
3251 if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
3254 FOR_EACH_EDGE (e, ei, exit_bb->preds)
3256 gsi = gsi_last_bb (e->src);
3257 if (gsi_end_p (gsi))
3259 stmt = gsi_stmt (gsi);
3260 if (gimple_code (stmt) == GIMPLE_OMP_RETURN
3261 && !gimple_omp_return_nowait_p (stmt))
3263 /* OpenMP 3.0 tasks unfortunately prevent this optimization
3264 in many cases. If there could be tasks queued, the barrier
3265 might be needed to let the tasks run before some local
3266 variable of the parallel that the task uses as shared
3267 runs out of scope. The task can be spawned either
3268 from within current function (this would be easy to check)
3269 or from some function it calls and gets passed an address
3270 of such a variable. */
3271 if (any_addressable_vars < 0)
3273 gimple parallel_stmt = last_stmt (region->entry);
3274 tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt);
3275 tree local_decls, block, decl;
3278 any_addressable_vars = 0;
3279 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl)
3280 if (TREE_ADDRESSABLE (decl))
3282 any_addressable_vars = 1;
3285 for (block = gimple_block (stmt);
3286 !any_addressable_vars
3288 && TREE_CODE (block) == BLOCK;
3289 block = BLOCK_SUPERCONTEXT (block))
3291 for (local_decls = BLOCK_VARS (block);
3293 local_decls = DECL_CHAIN (local_decls))
3294 if (TREE_ADDRESSABLE (local_decls))
3296 any_addressable_vars = 1;
3299 if (block == gimple_block (parallel_stmt))
3303 if (!any_addressable_vars)
3304 gimple_omp_return_set_nowait (stmt);
3310 remove_exit_barriers (struct omp_region *region)
3312 if (region->type == GIMPLE_OMP_PARALLEL)
3313 remove_exit_barrier (region);
3317 region = region->inner;
3318 remove_exit_barriers (region);
3319 while (region->next)
3321 region = region->next;
3322 remove_exit_barriers (region);
3327 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
3328 calls. These can't be declared as const functions, but
3329 within one parallel body they are constant, so they can be
3330 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
3331 which are declared const. Similarly for task body, except
3332 that in untied task omp_get_thread_num () can change at any task
3333 scheduling point. */
3336 optimize_omp_library_calls (gimple entry_stmt)
3339 gimple_stmt_iterator gsi;
3340 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
3341 tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree);
3342 tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
3343 tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree);
3344 bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
3345 && find_omp_clause (gimple_omp_task_clauses (entry_stmt),
3346 OMP_CLAUSE_UNTIED) != NULL);
3349 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
3351 gimple call = gsi_stmt (gsi);
3354 if (is_gimple_call (call)
3355 && (decl = gimple_call_fndecl (call))
3356 && DECL_EXTERNAL (decl)
3357 && TREE_PUBLIC (decl)
3358 && DECL_INITIAL (decl) == NULL)
3362 if (DECL_NAME (decl) == thr_num_id)
3364 /* In #pragma omp task untied omp_get_thread_num () can change
3365 during the execution of the task region. */
3368 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
3370 else if (DECL_NAME (decl) == num_thr_id)
3371 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
3375 if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in)
3376 || gimple_call_num_args (call) != 0)
3379 if (flag_exceptions && !TREE_NOTHROW (decl))
3382 if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE
3383 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)),
3384 TREE_TYPE (TREE_TYPE (built_in))))
3387 gimple_call_set_fndecl (call, built_in);
3392 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
3396 expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *)
3400 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
3401 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t))
3404 if (TREE_CODE (t) == ADDR_EXPR)
3405 recompute_tree_invariant_for_addr_expr (t);
3407 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
3411 /* Expand the OpenMP parallel or task directive starting at REGION. */
3414 expand_omp_taskreg (struct omp_region *region)
3416 basic_block entry_bb, exit_bb, new_bb;
3417 struct function *child_cfun;
3418 tree child_fn, block, t;
3419 gimple_stmt_iterator gsi;
3420 gimple entry_stmt, stmt;
3422 vec<tree, va_gc> *ws_args;
3424 entry_stmt = last_stmt (region->entry);
3425 child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
3426 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
3428 entry_bb = region->entry;
3429 exit_bb = region->exit;
3431 if (is_combined_parallel (region))
3432 ws_args = region->ws_args;
3436 if (child_cfun->cfg)
3438 /* Due to inlining, it may happen that we have already outlined
3439 the region, in which case all we need to do is make the
3440 sub-graph unreachable and emit the parallel call. */
3441 edge entry_succ_e, exit_succ_e;
3442 gimple_stmt_iterator gsi;
3444 entry_succ_e = single_succ_edge (entry_bb);
3446 gsi = gsi_last_bb (entry_bb);
3447 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
3448 || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
3449 gsi_remove (&gsi, true);
3454 exit_succ_e = single_succ_edge (exit_bb);
3455 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
3457 remove_edge_and_dominated_blocks (entry_succ_e);
3461 unsigned srcidx, dstidx, num;
3463 /* If the parallel region needs data sent from the parent
3464 function, then the very first statement (except possible
3465 tree profile counter updates) of the parallel body
3466 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
3467 &.OMP_DATA_O is passed as an argument to the child function,
3468 we need to replace it with the argument as seen by the child
3471 In most cases, this will end up being the identity assignment
3472 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
3473 a function call that has been inlined, the original PARM_DECL
3474 .OMP_DATA_I may have been converted into a different local
3475 variable. In which case, we need to keep the assignment. */
3476 if (gimple_omp_taskreg_data_arg (entry_stmt))
3478 basic_block entry_succ_bb = single_succ (entry_bb);
3479 gimple_stmt_iterator gsi;
3481 gimple parcopy_stmt = NULL;
3483 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
3487 gcc_assert (!gsi_end_p (gsi));
3488 stmt = gsi_stmt (gsi);
3489 if (gimple_code (stmt) != GIMPLE_ASSIGN)
3492 if (gimple_num_ops (stmt) == 2)
3494 tree arg = gimple_assign_rhs1 (stmt);
3496 /* We're ignore the subcode because we're
3497 effectively doing a STRIP_NOPS. */
3499 if (TREE_CODE (arg) == ADDR_EXPR
3500 && TREE_OPERAND (arg, 0)
3501 == gimple_omp_taskreg_data_arg (entry_stmt))
3503 parcopy_stmt = stmt;
3509 gcc_assert (parcopy_stmt != NULL);
3510 arg = DECL_ARGUMENTS (child_fn);
3512 if (!gimple_in_ssa_p (cfun))
3514 if (gimple_assign_lhs (parcopy_stmt) == arg)
3515 gsi_remove (&gsi, true);
3518 /* ?? Is setting the subcode really necessary ?? */
3519 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg));
3520 gimple_assign_set_rhs1 (parcopy_stmt, arg);
3525 /* If we are in ssa form, we must load the value from the default
3526 definition of the argument. That should not be defined now,
3527 since the argument is not used uninitialized. */
3528 gcc_assert (ssa_default_def (cfun, arg) == NULL);
3529 narg = make_ssa_name (arg, gimple_build_nop ());
3530 set_ssa_default_def (cfun, arg, narg);
3531 /* ?? Is setting the subcode really necessary ?? */
3532 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (narg));
3533 gimple_assign_set_rhs1 (parcopy_stmt, narg);
3534 update_stmt (parcopy_stmt);
3538 /* Declare local variables needed in CHILD_CFUN. */
3539 block = DECL_INITIAL (child_fn);
3540 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
3541 /* The gimplifier could record temporaries in parallel/task block
3542 rather than in containing function's local_decls chain,
3543 which would mean cgraph missed finalizing them. Do it now. */
3544 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
3545 if (TREE_CODE (t) == VAR_DECL
3547 && !DECL_EXTERNAL (t))
3548 varpool_finalize_decl (t);
3549 DECL_SAVED_TREE (child_fn) = NULL;
3550 /* We'll create a CFG for child_fn, so no gimple body is needed. */
3551 gimple_set_body (child_fn, NULL);
3552 TREE_USED (block) = 1;
3554 /* Reset DECL_CONTEXT on function arguments. */
3555 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
3556 DECL_CONTEXT (t) = child_fn;
3558 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
3559 so that it can be moved to the child function. */
3560 gsi = gsi_last_bb (entry_bb);
3561 stmt = gsi_stmt (gsi);
3562 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
3563 || gimple_code (stmt) == GIMPLE_OMP_TASK));
3564 gsi_remove (&gsi, true);
3565 e = split_block (entry_bb, stmt);
3567 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
3569 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
3572 gsi = gsi_last_bb (exit_bb);
3573 gcc_assert (!gsi_end_p (gsi)
3574 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
3575 stmt = gimple_build_return (NULL);
3576 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
3577 gsi_remove (&gsi, true);
3580 /* Move the parallel region into CHILD_CFUN. */
3582 if (gimple_in_ssa_p (cfun))
3584 init_tree_ssa (child_cfun);
3585 init_ssa_operands (child_cfun);
3586 child_cfun->gimple_df->in_ssa_p = true;
3590 block = gimple_block (entry_stmt);
3592 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
3594 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
3596 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
3597 num = vec_safe_length (child_cfun->local_decls);
3598 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
3600 t = (*child_cfun->local_decls)[srcidx];
3601 if (DECL_CONTEXT (t) == cfun->decl)
3603 if (srcidx != dstidx)
3604 (*child_cfun->local_decls)[dstidx] = t;
3608 vec_safe_truncate (child_cfun->local_decls, dstidx);
3610 /* Inform the callgraph about the new function. */
3611 DECL_STRUCT_FUNCTION (child_fn)->curr_properties
3612 = cfun->curr_properties & ~PROP_loops;
3613 cgraph_add_new_function (child_fn, true);
3615 /* Fix the callgraph edges for child_cfun. Those for cfun will be
3616 fixed in a following pass. */
3617 push_cfun (child_cfun);
3619 optimize_omp_library_calls (entry_stmt);
3620 rebuild_cgraph_edges ();
3622 /* Some EH regions might become dead, see PR34608. If
3623 pass_cleanup_cfg isn't the first pass to happen with the
3624 new child, these dead EH edges might cause problems.
3625 Clean them up now. */
3626 if (flag_exceptions)
3629 bool changed = false;
3632 changed |= gimple_purge_dead_eh_edges (bb);
3634 cleanup_tree_cfg ();
3636 if (gimple_in_ssa_p (cfun))
3637 update_ssa (TODO_update_ssa);
3641 /* Emit a library call to launch the children threads. */
3642 if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
3643 expand_parallel_call (region, new_bb, entry_stmt, ws_args);
3645 expand_task_call (new_bb, entry_stmt);
3646 if (gimple_in_ssa_p (cfun))
3647 update_ssa (TODO_update_ssa_only_virtuals);
3651 /* A subroutine of expand_omp_for. Generate code for a parallel
3652 loop with any schedule. Given parameters:
3654 for (V = N1; V cond N2; V += STEP) BODY;
3656 where COND is "<" or ">", we generate pseudocode
3658 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
3659 if (more) goto L0; else goto L3;
3666 if (V cond iend) goto L1; else goto L2;
3668 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
3671 If this is a combined omp parallel loop, instead of the call to
3672 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
3674 For collapsed loops, given parameters:
3676 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
3677 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
3678 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
3681 we generate pseudocode
3683 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
3688 count3 = (adj + N32 - N31) / STEP3;
3689 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
3694 count2 = (adj + N22 - N21) / STEP2;
3695 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
3700 count1 = (adj + N12 - N11) / STEP1;
3701 count = count1 * count2 * count3;
3706 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
3707 if (more) goto L0; else goto L3;
3711 V3 = N31 + (T % count3) * STEP3;
3713 V2 = N21 + (T % count2) * STEP2;
3715 V1 = N11 + T * STEP1;
3720 if (V < iend) goto L10; else goto L2;
3723 if (V3 cond3 N32) goto L1; else goto L11;
3727 if (V2 cond2 N22) goto L1; else goto L12;
3733 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
3739 expand_omp_for_generic (struct omp_region *region,
3740 struct omp_for_data *fd,
3741 enum built_in_function start_fn,
3742 enum built_in_function next_fn)
3744 tree type, istart0, iend0, iend;
3745 tree t, vmain, vback, bias = NULL_TREE;
3746 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
3747 basic_block l2_bb = NULL, l3_bb = NULL;
3748 gimple_stmt_iterator gsi;
3750 bool in_combined_parallel = is_combined_parallel (region);
3751 bool broken_loop = region->cont == NULL;
3753 tree *counts = NULL;
3756 gcc_assert (!broken_loop || !in_combined_parallel);
3757 gcc_assert (fd->iter_type == long_integer_type_node
3758 || !in_combined_parallel);
3760 type = TREE_TYPE (fd->loop.v);
3761 istart0 = create_tmp_var (fd->iter_type, ".istart0");
3762 iend0 = create_tmp_var (fd->iter_type, ".iend0");
3763 TREE_ADDRESSABLE (istart0) = 1;
3764 TREE_ADDRESSABLE (iend0) = 1;
3766 /* See if we need to bias by LLONG_MIN. */
3767 if (fd->iter_type == long_long_unsigned_type_node
3768 && TREE_CODE (type) == INTEGER_TYPE
3769 && !TYPE_UNSIGNED (type))
3773 if (fd->loop.cond_code == LT_EXPR)
3776 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
3780 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
3783 if (TREE_CODE (n1) != INTEGER_CST
3784 || TREE_CODE (n2) != INTEGER_CST
3785 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
3786 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
3789 entry_bb = region->entry;
3790 cont_bb = region->cont;
3792 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
3793 gcc_assert (broken_loop
3794 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
3795 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
3796 l1_bb = single_succ (l0_bb);
3799 l2_bb = create_empty_bb (cont_bb);
3800 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb);
3801 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
3805 l3_bb = BRANCH_EDGE (entry_bb)->dest;
3806 exit_bb = region->exit;
3808 gsi = gsi_last_bb (entry_bb);
3810 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
3811 if (fd->collapse > 1)
3813 basic_block zero_iter_bb = NULL;
3814 int first_zero_iter = -1;
3816 /* collapsed loops need work for expansion in SSA form. */
3817 gcc_assert (!gimple_in_ssa_p (cfun));
3818 counts = (tree *) alloca (fd->collapse * sizeof (tree));
3819 for (i = 0; i < fd->collapse; i++)
3821 tree itype = TREE_TYPE (fd->loops[i].v);
3823 if (SSA_VAR_P (fd->loop.n2)
3824 && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
3825 fold_convert (itype, fd->loops[i].n1),
3826 fold_convert (itype, fd->loops[i].n2)))
3827 == NULL_TREE || !integer_onep (t)))
3830 n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1));
3831 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
3832 true, GSI_SAME_STMT);
3833 n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
3834 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
3835 true, GSI_SAME_STMT);
3836 stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
3837 NULL_TREE, NULL_TREE);
3838 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
3839 if (walk_tree (gimple_cond_lhs_ptr (stmt),
3840 expand_omp_regimplify_p, NULL, NULL)
3841 || walk_tree (gimple_cond_rhs_ptr (stmt),
3842 expand_omp_regimplify_p, NULL, NULL))
3844 gsi = gsi_for_stmt (stmt);
3845 gimple_regimplify_operands (stmt, &gsi);
3847 e = split_block (entry_bb, stmt);
3848 if (zero_iter_bb == NULL)
3850 first_zero_iter = i;
3851 zero_iter_bb = create_empty_bb (entry_bb);
3853 add_bb_to_loop (zero_iter_bb, entry_bb->loop_father);
3854 gsi = gsi_after_labels (zero_iter_bb);
3855 stmt = gimple_build_assign (fd->loop.n2,
3856 build_zero_cst (type));
3857 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
3858 set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb,
3861 ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
3862 ne->probability = REG_BR_PROB_BASE / 2000 - 1;
3863 e->flags = EDGE_TRUE_VALUE;
3864 e->probability = REG_BR_PROB_BASE - ne->probability;
3866 gsi = gsi_last_bb (entry_bb);
3868 if (POINTER_TYPE_P (itype))
3869 itype = signed_type_for (itype);
3870 t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
3872 t = fold_build2 (PLUS_EXPR, itype,
3873 fold_convert (itype, fd->loops[i].step), t);
3874 t = fold_build2 (PLUS_EXPR, itype, t,
3875 fold_convert (itype, fd->loops[i].n2));
3876 t = fold_build2 (MINUS_EXPR, itype, t,
3877 fold_convert (itype, fd->loops[i].n1));
3878 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
3879 t = fold_build2 (TRUNC_DIV_EXPR, itype,
3880 fold_build1 (NEGATE_EXPR, itype, t),
3881 fold_build1 (NEGATE_EXPR, itype,
3882 fold_convert (itype,
3883 fd->loops[i].step)));
3885 t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
3886 fold_convert (itype, fd->loops[i].step));
3887 t = fold_convert (type, t);
3888 if (TREE_CODE (t) == INTEGER_CST)
3892 counts[i] = create_tmp_reg (type, ".count");
3893 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE,
3894 true, GSI_SAME_STMT);
3895 stmt = gimple_build_assign (counts[i], t);
3896 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
3898 if (SSA_VAR_P (fd->loop.n2))
3904 t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]);
3905 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE,
3906 true, GSI_SAME_STMT);
3908 stmt = gimple_build_assign (fd->loop.n2, t);
3909 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
3914 /* Some counts[i] vars might be uninitialized if
3915 some loop has zero iterations. But the body shouldn't
3916 be executed in that case, so just avoid uninit warnings. */
3917 for (i = first_zero_iter; i < fd->collapse; i++)
3918 if (SSA_VAR_P (counts[i]))
3919 TREE_NO_WARNING (counts[i]) = 1;
3921 e = split_block (entry_bb, gsi_stmt (gsi));
3923 make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU);
3924 gsi = gsi_last_bb (entry_bb);
3925 set_immediate_dominator (CDI_DOMINATORS, entry_bb,
3926 get_immediate_dominator (CDI_DOMINATORS,
3930 if (in_combined_parallel)
3932 /* In a combined parallel loop, emit a call to
3933 GOMP_loop_foo_next. */
3934 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
3935 build_fold_addr_expr (istart0),
3936 build_fold_addr_expr (iend0));
3940 tree t0, t1, t2, t3, t4;
3941 /* If this is not a combined parallel loop, emit a call to
3942 GOMP_loop_foo_start in ENTRY_BB. */
3943 t4 = build_fold_addr_expr (iend0);
3944 t3 = build_fold_addr_expr (istart0);
3945 t2 = fold_convert (fd->iter_type, fd->loop.step);
3946 if (POINTER_TYPE_P (type)
3947 && TYPE_PRECISION (type) != TYPE_PRECISION (fd->iter_type))
3949 /* Avoid casting pointers to integer of a different size. */
3950 tree itype = signed_type_for (type);
3951 t1 = fold_convert (fd->iter_type, fold_convert (itype, fd->loop.n2));
3952 t0 = fold_convert (fd->iter_type, fold_convert (itype, fd->loop.n1));
3956 t1 = fold_convert (fd->iter_type, fd->loop.n2);
3957 t0 = fold_convert (fd->iter_type, fd->loop.n1);
3961 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
3962 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
3964 if (fd->iter_type == long_integer_type_node)
3968 t = fold_convert (fd->iter_type, fd->chunk_size);
3969 t = build_call_expr (builtin_decl_explicit (start_fn),
3970 6, t0, t1, t2, t, t3, t4);
3973 t = build_call_expr (builtin_decl_explicit (start_fn),
3974 5, t0, t1, t2, t3, t4);
3982 /* The GOMP_loop_ull_*start functions have additional boolean
3983 argument, true for < loops and false for > loops.
3984 In Fortran, the C bool type can be different from
3985 boolean_type_node. */
3986 bfn_decl = builtin_decl_explicit (start_fn);
3987 c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl));
3988 t5 = build_int_cst (c_bool_type,
3989 fd->loop.cond_code == LT_EXPR ? 1 : 0);
3992 tree bfn_decl = builtin_decl_explicit (start_fn);
3993 t = fold_convert (fd->iter_type, fd->chunk_size);
3994 t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
3997 t = build_call_expr (builtin_decl_explicit (start_fn),
3998 6, t5, t0, t1, t2, t3, t4);
4001 if (TREE_TYPE (t) != boolean_type_node)
4002 t = fold_build2 (NE_EXPR, boolean_type_node,
4003 t, build_int_cst (TREE_TYPE (t), 0));
4004 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4005 true, GSI_SAME_STMT);
4006 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
4008 /* Remove the GIMPLE_OMP_FOR statement. */
4009 gsi_remove (&gsi, true);
4011 /* Iteration setup for sequential loop goes in L0_BB. */
4012 gsi = gsi_start_bb (l0_bb);
4015 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
4016 if (POINTER_TYPE_P (type))
4017 t = fold_convert (signed_type_for (type), t);
4018 t = fold_convert (type, t);
4019 t = force_gimple_operand_gsi (&gsi, t,
4021 && TREE_ADDRESSABLE (fd->loop.v),
4022 NULL_TREE, false, GSI_CONTINUE_LINKING);
4023 stmt = gimple_build_assign (fd->loop.v, t);
4024 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4028 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
4029 if (POINTER_TYPE_P (type))
4030 t = fold_convert (signed_type_for (type), t);
4031 t = fold_convert (type, t);
4032 iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4033 false, GSI_CONTINUE_LINKING);
4034 if (fd->collapse > 1)
4036 tree tem = create_tmp_reg (type, ".tem");
4037 stmt = gimple_build_assign (tem, fd->loop.v);
4038 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4039 for (i = fd->collapse - 1; i >= 0; i--)
4041 tree vtype = TREE_TYPE (fd->loops[i].v), itype;
4043 if (POINTER_TYPE_P (vtype))
4044 itype = signed_type_for (vtype);
4045 t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]);
4046 t = fold_convert (itype, t);
4047 t = fold_build2 (MULT_EXPR, itype, t,
4048 fold_convert (itype, fd->loops[i].step));
4049 if (POINTER_TYPE_P (vtype))
4050 t = fold_build_pointer_plus (fd->loops[i].n1, t);
4052 t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
4053 t = force_gimple_operand_gsi (&gsi, t,
4054 DECL_P (fd->loops[i].v)
4055 && TREE_ADDRESSABLE (fd->loops[i].v),
4057 GSI_CONTINUE_LINKING);
4058 stmt = gimple_build_assign (fd->loops[i].v, t);
4059 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4062 t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]);
4063 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE,
4064 false, GSI_CONTINUE_LINKING);
4065 stmt = gimple_build_assign (tem, t);
4066 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4073 /* Code to control the increment and predicate for the sequential
4074 loop goes in the CONT_BB. */
4075 gsi = gsi_last_bb (cont_bb);
4076 stmt = gsi_stmt (gsi);
4077 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
4078 vmain = gimple_omp_continue_control_use (stmt);
4079 vback = gimple_omp_continue_control_def (stmt);
4081 if (POINTER_TYPE_P (type))
4082 t = fold_build_pointer_plus (vmain, fd->loop.step);
4084 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
4085 t = force_gimple_operand_gsi (&gsi, t,
4086 DECL_P (vback) && TREE_ADDRESSABLE (vback),
4087 NULL_TREE, true, GSI_SAME_STMT);
4088 stmt = gimple_build_assign (vback, t);
4089 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
4091 t = build2 (fd->loop.cond_code, boolean_type_node,
4092 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
4094 stmt = gimple_build_cond_empty (t);
4095 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
4097 /* Remove GIMPLE_OMP_CONTINUE. */
4098 gsi_remove (&gsi, true);
4100 if (fd->collapse > 1)
4102 basic_block last_bb, bb;
4105 for (i = fd->collapse - 1; i >= 0; i--)
4107 tree vtype = TREE_TYPE (fd->loops[i].v);
4109 bb = create_empty_bb (last_bb);
4110 gsi = gsi_start_bb (bb);
4112 if (i < fd->collapse - 1)
4114 e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
4115 e->probability = REG_BR_PROB_BASE / 8;
4117 t = fd->loops[i + 1].n1;
4118 t = force_gimple_operand_gsi (&gsi, t,
4119 DECL_P (fd->loops[i + 1].v)
4121 (fd->loops[i + 1].v),
4123 GSI_CONTINUE_LINKING);
4124 stmt = gimple_build_assign (fd->loops[i + 1].v, t);
4125 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4130 set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
4132 if (POINTER_TYPE_P (vtype))
4133 t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
4135 t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v,
4137 t = force_gimple_operand_gsi (&gsi, t,
4138 DECL_P (fd->loops[i].v)
4139 && TREE_ADDRESSABLE (fd->loops[i].v),
4141 GSI_CONTINUE_LINKING);
4142 stmt = gimple_build_assign (fd->loops[i].v, t);
4143 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4147 t = fd->loops[i].n2;
4148 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4149 false, GSI_CONTINUE_LINKING);
4150 tree v = fd->loops[i].v;
4151 if (DECL_P (v) && TREE_ADDRESSABLE (v))
4152 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
4153 false, GSI_CONTINUE_LINKING);
4154 t = fold_build2 (fd->loops[i].cond_code, boolean_type_node,
4156 stmt = gimple_build_cond_empty (t);
4157 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4158 e = make_edge (bb, l1_bb, EDGE_TRUE_VALUE);
4159 e->probability = REG_BR_PROB_BASE * 7 / 8;
4162 make_edge (bb, l1_bb, EDGE_FALLTHRU);
4167 /* Emit code to get the next parallel iteration in L2_BB. */
4168 gsi = gsi_start_bb (l2_bb);
4170 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
4171 build_fold_addr_expr (istart0),
4172 build_fold_addr_expr (iend0));
4173 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4174 false, GSI_CONTINUE_LINKING);
4175 if (TREE_TYPE (t) != boolean_type_node)
4176 t = fold_build2 (NE_EXPR, boolean_type_node,
4177 t, build_int_cst (TREE_TYPE (t), 0));
4178 stmt = gimple_build_cond_empty (t);
4179 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4182 /* Add the loop cleanup function. */
4183 gsi = gsi_last_bb (exit_bb);
4184 if (gimple_omp_return_nowait_p (gsi_stmt (gsi)))
4185 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
4187 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
4188 stmt = gimple_build_call (t, 0);
4189 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
4190 gsi_remove (&gsi, true);
4192 /* Connect the new blocks. */
4193 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
4194 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
4200 e = find_edge (cont_bb, l3_bb);
4201 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
4203 phis = phi_nodes (l3_bb);
4204 for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi))
4206 gimple phi = gsi_stmt (gsi);
4207 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
4208 PHI_ARG_DEF_FROM_EDGE (phi, e));
4212 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
4213 if (fd->collapse > 1)
4215 e = find_edge (cont_bb, l1_bb);
4217 e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
4221 e = find_edge (cont_bb, l1_bb);
4222 e->flags = EDGE_TRUE_VALUE;
4224 e->probability = REG_BR_PROB_BASE * 7 / 8;
4225 find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8;
4226 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
4228 set_immediate_dominator (CDI_DOMINATORS, l2_bb,
4229 recompute_dominator (CDI_DOMINATORS, l2_bb));
4230 set_immediate_dominator (CDI_DOMINATORS, l3_bb,
4231 recompute_dominator (CDI_DOMINATORS, l3_bb));
4232 set_immediate_dominator (CDI_DOMINATORS, l0_bb,
4233 recompute_dominator (CDI_DOMINATORS, l0_bb));
4234 set_immediate_dominator (CDI_DOMINATORS, l1_bb,
4235 recompute_dominator (CDI_DOMINATORS, l1_bb));
4240 /* A subroutine of expand_omp_for. Generate code for a parallel
4241 loop with static schedule and no specified chunk size. Given
4244 for (V = N1; V cond N2; V += STEP) BODY;
4246 where COND is "<" or ">", we generate pseudocode
4248 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
4253 if ((__typeof (V)) -1 > 0 && cond is >)
4254 n = -(adj + N2 - N1) / -STEP;
4256 n = (adj + N2 - N1) / STEP;
4259 if (threadid < tt) goto L3; else goto L4;
4264 s0 = q * threadid + tt;
4267 if (s0 >= e0) goto L2; else goto L0;
4273 if (V cond e) goto L1;
4278 expand_omp_for_static_nochunk (struct omp_region *region,
4279 struct omp_for_data *fd)
4281 tree n, q, s0, e0, e, t, tt, nthreads, threadid;
4282 tree type, itype, vmain, vback;
4283 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
4284 basic_block body_bb, cont_bb;
4286 gimple_stmt_iterator gsi;
4290 itype = type = TREE_TYPE (fd->loop.v);
4291 if (POINTER_TYPE_P (type))
4292 itype = signed_type_for (type);
4294 entry_bb = region->entry;
4295 cont_bb = region->cont;
4296 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
4297 gcc_assert (BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
4298 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
4299 body_bb = single_succ (seq_start_bb);
4300 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
4301 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
4302 fin_bb = FALLTHRU_EDGE (cont_bb)->dest;
4303 exit_bb = region->exit;
4305 /* Iteration space partitioning goes in ENTRY_BB. */
4306 gsi = gsi_last_bb (entry_bb);
4307 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
4309 t = fold_binary (fd->loop.cond_code, boolean_type_node,
4310 fold_convert (type, fd->loop.n1),
4311 fold_convert (type, fd->loop.n2));
4312 if (TYPE_UNSIGNED (type)
4313 && (t == NULL_TREE || !integer_onep (t)))
4316 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
4317 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
4318 true, GSI_SAME_STMT);
4319 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
4320 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
4321 true, GSI_SAME_STMT);
4322 stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
4323 NULL_TREE, NULL_TREE);
4324 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
4325 if (walk_tree (gimple_cond_lhs_ptr (stmt),
4326 expand_omp_regimplify_p, NULL, NULL)
4327 || walk_tree (gimple_cond_rhs_ptr (stmt),
4328 expand_omp_regimplify_p, NULL, NULL))
4330 gsi = gsi_for_stmt (stmt);
4331 gimple_regimplify_operands (stmt, &gsi);
4333 ep = split_block (entry_bb, stmt);
4334 ep->flags = EDGE_TRUE_VALUE;
4335 entry_bb = ep->dest;
4336 ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
4337 ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
4338 ep->probability = REG_BR_PROB_BASE / 2000 - 1;
4339 if (gimple_in_ssa_p (cfun))
4341 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
4342 for (gsi = gsi_start_phis (fin_bb);
4343 !gsi_end_p (gsi); gsi_next (&gsi))
4345 gimple phi = gsi_stmt (gsi);
4346 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
4347 ep, UNKNOWN_LOCATION);
4350 gsi = gsi_last_bb (entry_bb);
4353 t = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS), 0);
4354 t = fold_convert (itype, t);
4355 nthreads = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4356 true, GSI_SAME_STMT);
4358 t = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM), 0);
4359 t = fold_convert (itype, t);
4360 threadid = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4361 true, GSI_SAME_STMT);
4364 = force_gimple_operand_gsi (&gsi, fold_convert (type, fd->loop.n1),
4365 true, NULL_TREE, true, GSI_SAME_STMT);
4367 = force_gimple_operand_gsi (&gsi, fold_convert (itype, fd->loop.n2),
4368 true, NULL_TREE, true, GSI_SAME_STMT);
4370 = force_gimple_operand_gsi (&gsi, fold_convert (itype, fd->loop.step),
4371 true, NULL_TREE, true, GSI_SAME_STMT);
4373 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
4374 t = fold_build2 (PLUS_EXPR, itype, fd->loop.step, t);
4375 t = fold_build2 (PLUS_EXPR, itype, t, fd->loop.n2);
4376 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, fd->loop.n1));
4377 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
4378 t = fold_build2 (TRUNC_DIV_EXPR, itype,
4379 fold_build1 (NEGATE_EXPR, itype, t),
4380 fold_build1 (NEGATE_EXPR, itype, fd->loop.step));
4382 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, fd->loop.step);
4383 t = fold_convert (itype, t);
4384 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
4386 q = create_tmp_reg (itype, "q");
4387 t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads);
4388 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
4389 gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT);
4391 tt = create_tmp_reg (itype, "tt");
4392 t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads);
4393 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
4394 gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT);
4396 t = build2 (LT_EXPR, boolean_type_node, threadid, tt);
4397 stmt = gimple_build_cond_empty (t);
4398 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
4400 second_bb = split_block (entry_bb, stmt)->dest;
4401 gsi = gsi_last_bb (second_bb);
4402 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
4404 gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)),
4406 stmt = gimple_build_assign_with_ops (PLUS_EXPR, q, q,
4407 build_int_cst (itype, 1));
4408 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
4410 third_bb = split_block (second_bb, stmt)->dest;
4411 gsi = gsi_last_bb (third_bb);
4412 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
4414 t = build2 (MULT_EXPR, itype, q, threadid);
4415 t = build2 (PLUS_EXPR, itype, t, tt);
4416 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
4418 t = fold_build2 (PLUS_EXPR, itype, s0, q);
4419 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
4421 t = build2 (GE_EXPR, boolean_type_node, s0, e0);
4422 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
4424 /* Remove the GIMPLE_OMP_FOR statement. */
4425 gsi_remove (&gsi, true);
4427 /* Setup code for sequential iteration goes in SEQ_START_BB. */
4428 gsi = gsi_start_bb (seq_start_bb);
4430 t = fold_convert (itype, s0);
4431 t = fold_build2 (MULT_EXPR, itype, t, fd->loop.step);
4432 if (POINTER_TYPE_P (type))
4433 t = fold_build_pointer_plus (fd->loop.n1, t);
4435 t = fold_build2 (PLUS_EXPR, type, t, fd->loop.n1);
4436 t = force_gimple_operand_gsi (&gsi, t,
4438 && TREE_ADDRESSABLE (fd->loop.v),
4439 NULL_TREE, false, GSI_CONTINUE_LINKING);
4440 stmt = gimple_build_assign (fd->loop.v, t);
4441 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4443 t = fold_convert (itype, e0);
4444 t = fold_build2 (MULT_EXPR, itype, t, fd->loop.step);
4445 if (POINTER_TYPE_P (type))
4446 t = fold_build_pointer_plus (fd->loop.n1, t);
4448 t = fold_build2 (PLUS_EXPR, type, t, fd->loop.n1);
4449 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4450 false, GSI_CONTINUE_LINKING);
4452 /* The code controlling the sequential loop replaces the
4453 GIMPLE_OMP_CONTINUE. */
4454 gsi = gsi_last_bb (cont_bb);
4455 stmt = gsi_stmt (gsi);
4456 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
4457 vmain = gimple_omp_continue_control_use (stmt);
4458 vback = gimple_omp_continue_control_def (stmt);
4460 if (POINTER_TYPE_P (type))
4461 t = fold_build_pointer_plus (vmain, fd->loop.step);
4463 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
4464 t = force_gimple_operand_gsi (&gsi, t,
4465 DECL_P (vback) && TREE_ADDRESSABLE (vback),
4466 NULL_TREE, true, GSI_SAME_STMT);
4467 stmt = gimple_build_assign (vback, t);
4468 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
4470 t = build2 (fd->loop.cond_code, boolean_type_node,
4471 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback, e);
4472 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
4474 /* Remove the GIMPLE_OMP_CONTINUE statement. */
4475 gsi_remove (&gsi, true);
4477 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
4478 gsi = gsi_last_bb (exit_bb);
4479 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
4480 force_gimple_operand_gsi (&gsi, build_omp_barrier (), false, NULL_TREE,
4481 false, GSI_SAME_STMT);
4482 gsi_remove (&gsi, true);
4484 /* Connect all the blocks. */
4485 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
4486 ep->probability = REG_BR_PROB_BASE / 4 * 3;
4487 ep = find_edge (entry_bb, second_bb);
4488 ep->flags = EDGE_TRUE_VALUE;
4489 ep->probability = REG_BR_PROB_BASE / 4;
4490 find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
4491 find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
4493 find_edge (cont_bb, body_bb)->flags = EDGE_TRUE_VALUE;
4494 find_edge (cont_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
4496 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
4497 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
4498 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb);
4499 set_immediate_dominator (CDI_DOMINATORS, body_bb,
4500 recompute_dominator (CDI_DOMINATORS, body_bb));
4501 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
4502 recompute_dominator (CDI_DOMINATORS, fin_bb));
4506 /* A subroutine of expand_omp_for. Generate code for a parallel
4507 loop with static schedule and a specified chunk size. Given
4510 for (V = N1; V cond N2; V += STEP) BODY;
4512 where COND is "<" or ">", we generate pseudocode
4514 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
4519 if ((__typeof (V)) -1 > 0 && cond is >)
4520 n = -(adj + N2 - N1) / -STEP;
4522 n = (adj + N2 - N1) / STEP;
4524 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
4525 here so that V is defined
4526 if the loop is not entered
4528 s0 = (trip * nthreads + threadid) * CHUNK;
4529 e0 = min(s0 + CHUNK, n);
4530 if (s0 < n) goto L1; else goto L4;
4537 if (V cond e) goto L2; else goto L3;
4545 expand_omp_for_static_chunk (struct omp_region *region, struct omp_for_data *fd)
4547 tree n, s0, e0, e, t;
4548 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
4549 tree type, itype, v_main, v_back, v_extra;
4550 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
4551 basic_block trip_update_bb, cont_bb, fin_bb;
4552 gimple_stmt_iterator si;
4556 itype = type = TREE_TYPE (fd->loop.v);
4557 if (POINTER_TYPE_P (type))
4558 itype = signed_type_for (type);
4560 entry_bb = region->entry;
4561 se = split_block (entry_bb, last_stmt (entry_bb));
4563 iter_part_bb = se->dest;
4564 cont_bb = region->cont;
4565 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
4566 gcc_assert (BRANCH_EDGE (iter_part_bb)->dest
4567 == FALLTHRU_EDGE (cont_bb)->dest);
4568 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
4569 body_bb = single_succ (seq_start_bb);
4570 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
4571 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
4572 fin_bb = FALLTHRU_EDGE (cont_bb)->dest;
4573 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
4574 exit_bb = region->exit;
4576 /* Trip and adjustment setup goes in ENTRY_BB. */
4577 si = gsi_last_bb (entry_bb);
4578 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_FOR);
4580 t = fold_binary (fd->loop.cond_code, boolean_type_node,
4581 fold_convert (type, fd->loop.n1),
4582 fold_convert (type, fd->loop.n2));
4583 if (TYPE_UNSIGNED (type)
4584 && (t == NULL_TREE || !integer_onep (t)))
4587 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
4588 n1 = force_gimple_operand_gsi (&si, n1, true, NULL_TREE,
4589 true, GSI_SAME_STMT);
4590 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
4591 n2 = force_gimple_operand_gsi (&si, n2, true, NULL_TREE,
4592 true, GSI_SAME_STMT);
4593 stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
4594 NULL_TREE, NULL_TREE);
4595 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
4596 if (walk_tree (gimple_cond_lhs_ptr (stmt),
4597 expand_omp_regimplify_p, NULL, NULL)
4598 || walk_tree (gimple_cond_rhs_ptr (stmt),
4599 expand_omp_regimplify_p, NULL, NULL))
4601 si = gsi_for_stmt (stmt);
4602 gimple_regimplify_operands (stmt, &si);
4604 se = split_block (entry_bb, stmt);
4605 se->flags = EDGE_TRUE_VALUE;
4606 entry_bb = se->dest;
4607 se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
4608 se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
4609 se->probability = REG_BR_PROB_BASE / 2000 - 1;
4610 if (gimple_in_ssa_p (cfun))
4612 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
4613 for (si = gsi_start_phis (fin_bb);
4614 !gsi_end_p (si); gsi_next (&si))
4616 gimple phi = gsi_stmt (si);
4617 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
4618 se, UNKNOWN_LOCATION);
4621 si = gsi_last_bb (entry_bb);
4624 t = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS), 0);
4625 t = fold_convert (itype, t);
4626 nthreads = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
4627 true, GSI_SAME_STMT);
4629 t = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM), 0);
4630 t = fold_convert (itype, t);
4631 threadid = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
4632 true, GSI_SAME_STMT);
4635 = force_gimple_operand_gsi (&si, fold_convert (type, fd->loop.n1),
4636 true, NULL_TREE, true, GSI_SAME_STMT);
4638 = force_gimple_operand_gsi (&si, fold_convert (itype, fd->loop.n2),
4639 true, NULL_TREE, true, GSI_SAME_STMT);
4641 = force_gimple_operand_gsi (&si, fold_convert (itype, fd->loop.step),
4642 true, NULL_TREE, true, GSI_SAME_STMT);
4644 = force_gimple_operand_gsi (&si, fold_convert (itype, fd->chunk_size),
4645 true, NULL_TREE, true, GSI_SAME_STMT);
4647 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
4648 t = fold_build2 (PLUS_EXPR, itype, fd->loop.step, t);
4649 t = fold_build2 (PLUS_EXPR, itype, t, fd->loop.n2);
4650 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, fd->loop.n1));
4651 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
4652 t = fold_build2 (TRUNC_DIV_EXPR, itype,
4653 fold_build1 (NEGATE_EXPR, itype, t),
4654 fold_build1 (NEGATE_EXPR, itype, fd->loop.step));
4656 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, fd->loop.step);
4657 t = fold_convert (itype, t);
4658 n = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
4659 true, GSI_SAME_STMT);
4661 trip_var = create_tmp_reg (itype, ".trip");
4662 if (gimple_in_ssa_p (cfun))
4664 trip_init = make_ssa_name (trip_var, NULL);
4665 trip_main = make_ssa_name (trip_var, NULL);
4666 trip_back = make_ssa_name (trip_var, NULL);
4670 trip_init = trip_var;
4671 trip_main = trip_var;
4672 trip_back = trip_var;
4675 stmt = gimple_build_assign (trip_init, build_int_cst (itype, 0));
4676 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
4678 t = fold_build2 (MULT_EXPR, itype, threadid, fd->chunk_size);
4679 t = fold_build2 (MULT_EXPR, itype, t, fd->loop.step);
4680 if (POINTER_TYPE_P (type))
4681 t = fold_build_pointer_plus (fd->loop.n1, t);
4683 t = fold_build2 (PLUS_EXPR, type, t, fd->loop.n1);
4684 v_extra = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
4685 true, GSI_SAME_STMT);
4687 /* Remove the GIMPLE_OMP_FOR. */
4688 gsi_remove (&si, true);
4690 /* Iteration space partitioning goes in ITER_PART_BB. */
4691 si = gsi_last_bb (iter_part_bb);
4693 t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads);
4694 t = fold_build2 (PLUS_EXPR, itype, t, threadid);
4695 t = fold_build2 (MULT_EXPR, itype, t, fd->chunk_size);
4696 s0 = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
4697 false, GSI_CONTINUE_LINKING);
4699 t = fold_build2 (PLUS_EXPR, itype, s0, fd->chunk_size);
4700 t = fold_build2 (MIN_EXPR, itype, t, n);
4701 e0 = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
4702 false, GSI_CONTINUE_LINKING);
4704 t = build2 (LT_EXPR, boolean_type_node, s0, n);
4705 gsi_insert_after (&si, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING);
4707 /* Setup code for sequential iteration goes in SEQ_START_BB. */
4708 si = gsi_start_bb (seq_start_bb);
4710 t = fold_convert (itype, s0);
4711 t = fold_build2 (MULT_EXPR, itype, t, fd->loop.step);
4712 if (POINTER_TYPE_P (type))
4713 t = fold_build_pointer_plus (fd->loop.n1, t);
4715 t = fold_build2 (PLUS_EXPR, type, t, fd->loop.n1);
4716 t = force_gimple_operand_gsi (&si, t,
4718 && TREE_ADDRESSABLE (fd->loop.v),
4719 NULL_TREE, false, GSI_CONTINUE_LINKING);
4720 stmt = gimple_build_assign (fd->loop.v, t);
4721 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
4723 t = fold_convert (itype, e0);
4724 t = fold_build2 (MULT_EXPR, itype, t, fd->loop.step);
4725 if (POINTER_TYPE_P (type))
4726 t = fold_build_pointer_plus (fd->loop.n1, t);
4728 t = fold_build2 (PLUS_EXPR, type, t, fd->loop.n1);
4729 e = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
4730 false, GSI_CONTINUE_LINKING);
4732 /* The code controlling the sequential loop goes in CONT_BB,
4733 replacing the GIMPLE_OMP_CONTINUE. */
4734 si = gsi_last_bb (cont_bb);
4735 stmt = gsi_stmt (si);
4736 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
4737 v_main = gimple_omp_continue_control_use (stmt);
4738 v_back = gimple_omp_continue_control_def (stmt);
4740 if (POINTER_TYPE_P (type))
4741 t = fold_build_pointer_plus (v_main, fd->loop.step);
4743 t = fold_build2 (PLUS_EXPR, type, v_main, fd->loop.step);
4744 if (DECL_P (v_back) && TREE_ADDRESSABLE (v_back))
4745 t = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
4746 true, GSI_SAME_STMT);
4747 stmt = gimple_build_assign (v_back, t);
4748 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
4750 t = build2 (fd->loop.cond_code, boolean_type_node,
4751 DECL_P (v_back) && TREE_ADDRESSABLE (v_back)
4753 gsi_insert_before (&si, gimple_build_cond_empty (t), GSI_SAME_STMT);
4755 /* Remove GIMPLE_OMP_CONTINUE. */
4756 gsi_remove (&si, true);
4758 /* Trip update code goes into TRIP_UPDATE_BB. */
4759 si = gsi_start_bb (trip_update_bb);
4761 t = build_int_cst (itype, 1);
4762 t = build2 (PLUS_EXPR, itype, trip_main, t);
4763 stmt = gimple_build_assign (trip_back, t);
4764 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
4766 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
4767 si = gsi_last_bb (exit_bb);
4768 if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
4769 force_gimple_operand_gsi (&si, build_omp_barrier (), false, NULL_TREE,
4770 false, GSI_SAME_STMT);
4771 gsi_remove (&si, true);
4773 /* Connect the new blocks. */
4774 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
4775 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
4777 find_edge (cont_bb, body_bb)->flags = EDGE_TRUE_VALUE;
4778 find_edge (cont_bb, trip_update_bb)->flags = EDGE_FALSE_VALUE;
4780 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb);
4782 if (gimple_in_ssa_p (cfun))
4784 gimple_stmt_iterator psi;
4787 edge_var_map_vector *head;
4791 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
4792 remove arguments of the phi nodes in fin_bb. We need to create
4793 appropriate phi nodes in iter_part_bb instead. */
4794 se = single_pred_edge (fin_bb);
4795 re = single_succ_edge (trip_update_bb);
4796 head = redirect_edge_var_map_vector (re);
4797 ene = single_succ_edge (entry_bb);
4799 psi = gsi_start_phis (fin_bb);
4800 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
4801 gsi_next (&psi), ++i)
4804 source_location locus;
4806 phi = gsi_stmt (psi);
4807 t = gimple_phi_result (phi);
4808 gcc_assert (t == redirect_edge_var_map_result (vm));
4809 nphi = create_phi_node (t, iter_part_bb);
4811 t = PHI_ARG_DEF_FROM_EDGE (phi, se);
4812 locus = gimple_phi_arg_location_from_edge (phi, se);
4814 /* A special case -- fd->loop.v is not yet computed in
4815 iter_part_bb, we need to use v_extra instead. */
4816 if (t == fd->loop.v)
4818 add_phi_arg (nphi, t, ene, locus);
4819 locus = redirect_edge_var_map_location (vm);
4820 add_phi_arg (nphi, redirect_edge_var_map_def (vm), re, locus);
4822 gcc_assert (!gsi_end_p (psi) && i == head->length ());
4823 redirect_edge_var_map_clear (re);
4826 psi = gsi_start_phis (fin_bb);
4827 if (gsi_end_p (psi))
4829 remove_phi_node (&psi, false);
4832 /* Make phi node for trip. */
4833 phi = create_phi_node (trip_main, iter_part_bb);
4834 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb),
4836 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb),
4840 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
4841 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
4842 recompute_dominator (CDI_DOMINATORS, iter_part_bb));
4843 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
4844 recompute_dominator (CDI_DOMINATORS, fin_bb));
4845 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
4846 recompute_dominator (CDI_DOMINATORS, seq_start_bb));
4847 set_immediate_dominator (CDI_DOMINATORS, body_bb,
4848 recompute_dominator (CDI_DOMINATORS, body_bb));
4852 /* Expand the OpenMP loop defined by REGION. */
4855 expand_omp_for (struct omp_region *region)
4857 struct omp_for_data fd;
4858 struct omp_for_data_loop *loops;
4861 = (struct omp_for_data_loop *)
4862 alloca (gimple_omp_for_collapse (last_stmt (region->entry))
4863 * sizeof (struct omp_for_data_loop));
4864 extract_omp_for_data (last_stmt (region->entry), &fd, loops);
4865 region->sched_kind = fd.sched_kind;
4867 gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
4868 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
4869 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
4872 gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
4873 BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
4874 FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
4877 if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
4880 && region->cont != NULL)
4882 if (fd.chunk_size == NULL)
4883 expand_omp_for_static_nochunk (region, &fd);
4885 expand_omp_for_static_chunk (region, &fd);
4889 int fn_index, start_ix, next_ix;
4891 if (fd.chunk_size == NULL
4892 && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
4893 fd.chunk_size = integer_zero_node;
4894 gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
4895 fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
4896 ? 3 : fd.sched_kind;
4897 fn_index += fd.have_ordered * 4;
4898 start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
4899 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
4900 if (fd.iter_type == long_long_unsigned_type_node)
4902 start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
4903 - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
4904 next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
4905 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
4907 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
4908 (enum built_in_function) next_ix);
4911 if (gimple_in_ssa_p (cfun))
4912 update_ssa (TODO_update_ssa_only_virtuals);
4916 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
4918 v = GOMP_sections_start (n);
4935 v = GOMP_sections_next ();
4940 If this is a combined parallel sections, replace the call to
4941 GOMP_sections_start with call to GOMP_sections_next. */
4944 expand_omp_sections (struct omp_region *region)
4946 tree t, u, vin = NULL, vmain, vnext, l2;
4947 vec<tree> label_vec;
4949 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
4950 gimple_stmt_iterator si, switch_si;
4951 gimple sections_stmt, stmt, cont;
4954 struct omp_region *inner;
4956 bool exit_reachable = region->cont != NULL;
4958 gcc_assert (region->exit != NULL);
4959 entry_bb = region->entry;
4960 l0_bb = single_succ (entry_bb);
4961 l1_bb = region->cont;
4962 l2_bb = region->exit;
4963 if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb)
4964 l2 = gimple_block_label (l2_bb);
4967 /* This can happen if there are reductions. */
4968 len = EDGE_COUNT (l0_bb->succs);
4969 gcc_assert (len > 0);
4970 e = EDGE_SUCC (l0_bb, len - 1);
4971 si = gsi_last_bb (e->dest);
4974 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
4975 l2 = gimple_block_label (e->dest);
4977 FOR_EACH_EDGE (e, ei, l0_bb->succs)
4979 si = gsi_last_bb (e->dest);
4981 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
4983 l2 = gimple_block_label (e->dest);
4989 default_bb = create_empty_bb (l1_bb->prev_bb);
4991 default_bb = create_empty_bb (l0_bb);
4993 /* We will build a switch() with enough cases for all the
4994 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
4995 and a default case to abort if something goes wrong. */
4996 len = EDGE_COUNT (l0_bb->succs);
4998 /* Use vec::quick_push on label_vec throughout, since we know the size
5000 label_vec.create (len);
5002 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
5003 GIMPLE_OMP_SECTIONS statement. */
5004 si = gsi_last_bb (entry_bb);
5005 sections_stmt = gsi_stmt (si);
5006 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
5007 vin = gimple_omp_sections_control (sections_stmt);
5008 if (!is_combined_parallel (region))
5010 /* If we are not inside a combined parallel+sections region,
5011 call GOMP_sections_start. */
5012 t = build_int_cst (unsigned_type_node,
5013 exit_reachable ? len - 1 : len);
5014 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
5015 stmt = gimple_build_call (u, 1, t);
5019 /* Otherwise, call GOMP_sections_next. */
5020 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
5021 stmt = gimple_build_call (u, 0);
5023 gimple_call_set_lhs (stmt, vin);
5024 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
5025 gsi_remove (&si, true);
5027 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
5029 switch_si = gsi_last_bb (l0_bb);
5030 gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH);
5033 cont = last_stmt (l1_bb);
5034 gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE);
5035 vmain = gimple_omp_continue_control_use (cont);
5036 vnext = gimple_omp_continue_control_def (cont);
5044 t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2);
5045 label_vec.quick_push (t);
5048 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
5049 for (inner = region->inner, casei = 1;
5051 inner = inner->next, i++, casei++)
5053 basic_block s_entry_bb, s_exit_bb;
5055 /* Skip optional reduction region. */
5056 if (inner->type == GIMPLE_OMP_ATOMIC_LOAD)
5063 s_entry_bb = inner->entry;
5064 s_exit_bb = inner->exit;
5066 t = gimple_block_label (s_entry_bb);
5067 u = build_int_cst (unsigned_type_node, casei);
5068 u = build_case_label (u, NULL, t);
5069 label_vec.quick_push (u);
5071 si = gsi_last_bb (s_entry_bb);
5072 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION);
5073 gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si)));
5074 gsi_remove (&si, true);
5075 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
5077 if (s_exit_bb == NULL)
5080 si = gsi_last_bb (s_exit_bb);
5081 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
5082 gsi_remove (&si, true);
5084 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
5087 /* Error handling code goes in DEFAULT_BB. */
5088 t = gimple_block_label (default_bb);
5089 u = build_case_label (NULL, NULL, t);
5090 make_edge (l0_bb, default_bb, 0);
5092 stmt = gimple_build_switch (vmain, u, label_vec);
5093 gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT);
5094 gsi_remove (&switch_si, true);
5095 label_vec.release ();
5097 si = gsi_start_bb (default_bb);
5098 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
5099 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
5105 /* Code to get the next section goes in L1_BB. */
5106 si = gsi_last_bb (l1_bb);
5107 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
5109 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
5110 stmt = gimple_build_call (bfn_decl, 0);
5111 gimple_call_set_lhs (stmt, vnext);
5112 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
5113 gsi_remove (&si, true);
5115 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
5118 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
5119 si = gsi_last_bb (l2_bb);
5120 if (gimple_omp_return_nowait_p (gsi_stmt (si)))
5121 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT);
5123 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END);
5124 stmt = gimple_build_call (t, 0);
5125 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
5126 gsi_remove (&si, true);
5128 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
5132 /* Expand code for an OpenMP single directive. We've already expanded
5133 much of the code, here we simply place the GOMP_barrier call. */
5136 expand_omp_single (struct omp_region *region)
5138 basic_block entry_bb, exit_bb;
5139 gimple_stmt_iterator si;
5140 bool need_barrier = false;
5142 entry_bb = region->entry;
5143 exit_bb = region->exit;
5145 si = gsi_last_bb (entry_bb);
5146 /* The terminal barrier at the end of a GOMP_single_copy sequence cannot
5147 be removed. We need to ensure that the thread that entered the single
5148 does not exit before the data is copied out by the other threads. */
5149 if (find_omp_clause (gimple_omp_single_clauses (gsi_stmt (si)),
5150 OMP_CLAUSE_COPYPRIVATE))
5151 need_barrier = true;
5152 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE);
5153 gsi_remove (&si, true);
5154 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
5156 si = gsi_last_bb (exit_bb);
5157 if (!gimple_omp_return_nowait_p (gsi_stmt (si)) || need_barrier)
5158 force_gimple_operand_gsi (&si, build_omp_barrier (), false, NULL_TREE,
5159 false, GSI_SAME_STMT);
5160 gsi_remove (&si, true);
5161 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
5165 /* Generic expansion for OpenMP synchronization directives: master,
5166 ordered and critical. All we need to do here is remove the entry
5167 and exit markers for REGION. */
5170 expand_omp_synch (struct omp_region *region)
5172 basic_block entry_bb, exit_bb;
5173 gimple_stmt_iterator si;
5175 entry_bb = region->entry;
5176 exit_bb = region->exit;
5178 si = gsi_last_bb (entry_bb);
5179 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
5180 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
5181 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
5182 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL);
5183 gsi_remove (&si, true);
5184 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
5188 si = gsi_last_bb (exit_bb);
5189 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
5190 gsi_remove (&si, true);
5191 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
5195 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
5196 operation as a normal volatile load. */
5199 expand_omp_atomic_load (basic_block load_bb, tree addr,
5200 tree loaded_val, int index)
5202 enum built_in_function tmpbase;
5203 gimple_stmt_iterator gsi;
5204 basic_block store_bb;
5207 tree decl, call, type, itype;
5209 gsi = gsi_last_bb (load_bb);
5210 stmt = gsi_stmt (gsi);
5211 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
5212 loc = gimple_location (stmt);
5214 /* ??? If the target does not implement atomic_load_optab[mode], and mode
5215 is smaller than word size, then expand_atomic_load assumes that the load
5216 is atomic. We could avoid the builtin entirely in this case. */
5218 tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
5219 decl = builtin_decl_explicit (tmpbase);
5220 if (decl == NULL_TREE)
5223 type = TREE_TYPE (loaded_val);
5224 itype = TREE_TYPE (TREE_TYPE (decl));
5226 call = build_call_expr_loc (loc, decl, 2, addr,
5227 build_int_cst (NULL, MEMMODEL_RELAXED));
5228 if (!useless_type_conversion_p (type, itype))
5229 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
5230 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
5232 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
5233 gsi_remove (&gsi, true);
5235 store_bb = single_succ (load_bb);
5236 gsi = gsi_last_bb (store_bb);
5237 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
5238 gsi_remove (&gsi, true);
5240 if (gimple_in_ssa_p (cfun))
5241 update_ssa (TODO_update_ssa_no_phi);
5246 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
5247 operation as a normal volatile store. */
5250 expand_omp_atomic_store (basic_block load_bb, tree addr,
5251 tree loaded_val, tree stored_val, int index)
5253 enum built_in_function tmpbase;
5254 gimple_stmt_iterator gsi;
5255 basic_block store_bb = single_succ (load_bb);
5258 tree decl, call, type, itype;
5259 enum machine_mode imode;
5262 gsi = gsi_last_bb (load_bb);
5263 stmt = gsi_stmt (gsi);
5264 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
5266 /* If the load value is needed, then this isn't a store but an exchange. */
5267 exchange = gimple_omp_atomic_need_value_p (stmt);
5269 gsi = gsi_last_bb (store_bb);
5270 stmt = gsi_stmt (gsi);
5271 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE);
5272 loc = gimple_location (stmt);
5274 /* ??? If the target does not implement atomic_store_optab[mode], and mode
5275 is smaller than word size, then expand_atomic_store assumes that the store
5276 is atomic. We could avoid the builtin entirely in this case. */
5278 tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N);
5279 tmpbase = (enum built_in_function) ((int) tmpbase + index + 1);
5280 decl = builtin_decl_explicit (tmpbase);
5281 if (decl == NULL_TREE)
5284 type = TREE_TYPE (stored_val);
5286 /* Dig out the type of the function's second argument. */
5287 itype = TREE_TYPE (decl);
5288 itype = TYPE_ARG_TYPES (itype);
5289 itype = TREE_CHAIN (itype);
5290 itype = TREE_VALUE (itype);
5291 imode = TYPE_MODE (itype);
5293 if (exchange && !can_atomic_exchange_p (imode, true))
5296 if (!useless_type_conversion_p (itype, type))
5297 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
5298 call = build_call_expr_loc (loc, decl, 3, addr, stored_val,
5299 build_int_cst (NULL, MEMMODEL_RELAXED));
5302 if (!useless_type_conversion_p (type, itype))
5303 call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
5304 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
5307 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
5308 gsi_remove (&gsi, true);
5310 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
5311 gsi = gsi_last_bb (load_bb);
5312 gsi_remove (&gsi, true);
5314 if (gimple_in_ssa_p (cfun))
5315 update_ssa (TODO_update_ssa_no_phi);
5320 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
5321 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
5322 size of the data type, and thus usable to find the index of the builtin
5323 decl. Returns false if the expression is not of the proper form. */
5326 expand_omp_atomic_fetch_op (basic_block load_bb,
5327 tree addr, tree loaded_val,
5328 tree stored_val, int index)
5330 enum built_in_function oldbase, newbase, tmpbase;
5331 tree decl, itype, call;
5333 basic_block store_bb = single_succ (load_bb);
5334 gimple_stmt_iterator gsi;
5337 enum tree_code code;
5338 bool need_old, need_new;
5339 enum machine_mode imode;
5341 /* We expect to find the following sequences:
5344 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
5347 val = tmp OP something; (or: something OP tmp)
5348 GIMPLE_OMP_STORE (val)
5350 ???FIXME: Allow a more flexible sequence.
5351 Perhaps use data flow to pick the statements.
5355 gsi = gsi_after_labels (store_bb);
5356 stmt = gsi_stmt (gsi);
5357 loc = gimple_location (stmt);
5358 if (!is_gimple_assign (stmt))
5361 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
5363 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
5364 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb));
5365 gcc_checking_assert (!need_old || !need_new);
5367 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
5370 /* Check for one of the supported fetch-op operations. */
5371 code = gimple_assign_rhs_code (stmt);
5375 case POINTER_PLUS_EXPR:
5376 oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N;
5377 newbase = BUILT_IN_ATOMIC_ADD_FETCH_N;
5380 oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N;
5381 newbase = BUILT_IN_ATOMIC_SUB_FETCH_N;
5384 oldbase = BUILT_IN_ATOMIC_FETCH_AND_N;
5385 newbase = BUILT_IN_ATOMIC_AND_FETCH_N;
5388 oldbase = BUILT_IN_ATOMIC_FETCH_OR_N;
5389 newbase = BUILT_IN_ATOMIC_OR_FETCH_N;
5392 oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N;
5393 newbase = BUILT_IN_ATOMIC_XOR_FETCH_N;
5399 /* Make sure the expression is of the proper form. */
5400 if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
5401 rhs = gimple_assign_rhs2 (stmt);
5402 else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
5403 && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
5404 rhs = gimple_assign_rhs1 (stmt);
5408 tmpbase = ((enum built_in_function)
5409 ((need_new ? newbase : oldbase) + index + 1));
5410 decl = builtin_decl_explicit (tmpbase);
5411 if (decl == NULL_TREE)
5413 itype = TREE_TYPE (TREE_TYPE (decl));
5414 imode = TYPE_MODE (itype);
5416 /* We could test all of the various optabs involved, but the fact of the
5417 matter is that (with the exception of i486 vs i586 and xadd) all targets
5418 that support any atomic operaton optab also implements compare-and-swap.
5419 Let optabs.c take care of expanding any compare-and-swap loop. */
5420 if (!can_compare_and_swap_p (imode, true))
5423 gsi = gsi_last_bb (load_bb);
5424 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
5426 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
5427 It only requires that the operation happen atomically. Thus we can
5428 use the RELAXED memory model. */
5429 call = build_call_expr_loc (loc, decl, 3, addr,
5430 fold_convert_loc (loc, itype, rhs),
5431 build_int_cst (NULL, MEMMODEL_RELAXED));
5433 if (need_old || need_new)
5435 lhs = need_old ? loaded_val : stored_val;
5436 call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
5437 call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call);
5440 call = fold_convert_loc (loc, void_type_node, call);
5441 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
5442 gsi_remove (&gsi, true);
5444 gsi = gsi_last_bb (store_bb);
5445 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
5446 gsi_remove (&gsi, true);
5447 gsi = gsi_last_bb (store_bb);
5448 gsi_remove (&gsi, true);
5450 if (gimple_in_ssa_p (cfun))
5451 update_ssa (TODO_update_ssa_no_phi);
5456 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
5460 newval = rhs; // with oldval replacing *addr in rhs
5461 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
5462 if (oldval != newval)
5465 INDEX is log2 of the size of the data type, and thus usable to find the
5466 index of the builtin decl. */
5469 expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
5470 tree addr, tree loaded_val, tree stored_val,
5473 tree loadedi, storedi, initial, new_storedi, old_vali;
5474 tree type, itype, cmpxchg, iaddr;
5475 gimple_stmt_iterator si;
5476 basic_block loop_header = single_succ (load_bb);
5479 enum built_in_function fncode;
5481 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
5482 order to use the RELAXED memory model effectively. */
5483 fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
5485 cmpxchg = builtin_decl_explicit (fncode);
5486 if (cmpxchg == NULL_TREE)
5488 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
5489 itype = TREE_TYPE (TREE_TYPE (cmpxchg));
5491 if (!can_compare_and_swap_p (TYPE_MODE (itype), true))
5494 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
5495 si = gsi_last_bb (load_bb);
5496 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
5498 /* For floating-point values, we'll need to view-convert them to integers
5499 so that we can perform the atomic compare and swap. Simplify the
5500 following code by always setting up the "i"ntegral variables. */
5501 if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
5505 iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode,
5508 = force_gimple_operand_gsi (&si,
5509 fold_convert (TREE_TYPE (iaddr), addr),
5510 false, NULL_TREE, true, GSI_SAME_STMT);
5511 stmt = gimple_build_assign (iaddr, iaddr_val);
5512 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
5513 loadedi = create_tmp_var (itype, NULL);
5514 if (gimple_in_ssa_p (cfun))
5515 loadedi = make_ssa_name (loadedi, NULL);
5520 loadedi = loaded_val;
5524 = force_gimple_operand_gsi (&si,
5525 build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)),
5527 build_int_cst (TREE_TYPE (iaddr), 0)),
5528 true, NULL_TREE, true, GSI_SAME_STMT);
5530 /* Move the value to the LOADEDI temporary. */
5531 if (gimple_in_ssa_p (cfun))
5533 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
5534 phi = create_phi_node (loadedi, loop_header);
5535 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
5539 gsi_insert_before (&si,
5540 gimple_build_assign (loadedi, initial),
5542 if (loadedi != loaded_val)
5544 gimple_stmt_iterator gsi2;
5547 x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
5548 gsi2 = gsi_start_bb (loop_header);
5549 if (gimple_in_ssa_p (cfun))
5552 x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
5553 true, GSI_SAME_STMT);
5554 stmt = gimple_build_assign (loaded_val, x);
5555 gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
5559 x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
5560 force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
5561 true, GSI_SAME_STMT);
5564 gsi_remove (&si, true);
5566 si = gsi_last_bb (store_bb);
5567 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
5570 storedi = stored_val;
5573 force_gimple_operand_gsi (&si,
5574 build1 (VIEW_CONVERT_EXPR, itype,
5575 stored_val), true, NULL_TREE, true,
5578 /* Build the compare&swap statement. */
5579 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
5580 new_storedi = force_gimple_operand_gsi (&si,
5581 fold_convert (TREE_TYPE (loadedi),
5584 true, GSI_SAME_STMT);
5586 if (gimple_in_ssa_p (cfun))
5590 old_vali = create_tmp_var (TREE_TYPE (loadedi), NULL);
5591 stmt = gimple_build_assign (old_vali, loadedi);
5592 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
5594 stmt = gimple_build_assign (loadedi, new_storedi);
5595 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
5598 /* Note that we always perform the comparison as an integer, even for
5599 floating point. This allows the atomic operation to properly
5600 succeed even with NaNs and -0.0. */
5601 stmt = gimple_build_cond_empty
5602 (build2 (NE_EXPR, boolean_type_node,
5603 new_storedi, old_vali));
5604 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
5607 e = single_succ_edge (store_bb);
5608 e->flags &= ~EDGE_FALLTHRU;
5609 e->flags |= EDGE_FALSE_VALUE;
5611 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
5613 /* Copy the new value to loadedi (we already did that before the condition
5614 if we are not in SSA). */
5615 if (gimple_in_ssa_p (cfun))
5617 phi = gimple_seq_first_stmt (phi_nodes (loop_header));
5618 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
5621 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
5622 gsi_remove (&si, true);
5624 if (gimple_in_ssa_p (cfun))
5625 update_ssa (TODO_update_ssa_no_phi);
5630 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
5632 GOMP_atomic_start ();
5636 The result is not globally atomic, but works so long as all parallel
5637 references are within #pragma omp atomic directives. According to
5638 responses received from omp@openmp.org, appears to be within spec.
5639 Which makes sense, since that's how several other compilers handle
5640 this situation as well.
5641 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
5642 expanding. STORED_VAL is the operand of the matching
5643 GIMPLE_OMP_ATOMIC_STORE.
5646 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
5650 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
5655 expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
5656 tree addr, tree loaded_val, tree stored_val)
5658 gimple_stmt_iterator si;
5662 si = gsi_last_bb (load_bb);
5663 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
5665 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
5666 t = build_call_expr (t, 0);
5667 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
5669 stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr));
5670 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
5671 gsi_remove (&si, true);
5673 si = gsi_last_bb (store_bb);
5674 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
5676 stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)),
5678 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
5680 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
5681 t = build_call_expr (t, 0);
5682 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
5683 gsi_remove (&si, true);
5685 if (gimple_in_ssa_p (cfun))
5686 update_ssa (TODO_update_ssa_no_phi);
5690 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
5691 using expand_omp_atomic_fetch_op. If it failed, we try to
5692 call expand_omp_atomic_pipeline, and if it fails too, the
5693 ultimate fallback is wrapping the operation in a mutex
5694 (expand_omp_atomic_mutex). REGION is the atomic region built
5695 by build_omp_regions_1(). */
5698 expand_omp_atomic (struct omp_region *region)
5700 basic_block load_bb = region->entry, store_bb = region->exit;
5701 gimple load = last_stmt (load_bb), store = last_stmt (store_bb);
5702 tree loaded_val = gimple_omp_atomic_load_lhs (load);
5703 tree addr = gimple_omp_atomic_load_rhs (load);
5704 tree stored_val = gimple_omp_atomic_store_val (store);
5705 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
5706 HOST_WIDE_INT index;
5708 /* Make sure the type is one of the supported sizes. */
5709 index = tree_low_cst (TYPE_SIZE_UNIT (type), 1);
5710 index = exact_log2 (index);
5711 if (index >= 0 && index <= 4)
5713 unsigned int align = TYPE_ALIGN_UNIT (type);
5715 /* __sync builtins require strict data alignment. */
5716 if (exact_log2 (align) >= index)
5719 if (loaded_val == stored_val
5720 && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
5721 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
5722 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
5723 && expand_omp_atomic_load (load_bb, addr, loaded_val, index))
5727 if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
5728 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
5729 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
5730 && store_bb == single_succ (load_bb)
5731 && first_stmt (store_bb) == store
5732 && expand_omp_atomic_store (load_bb, addr, loaded_val,
5736 /* When possible, use specialized atomic update functions. */
5737 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
5738 && store_bb == single_succ (load_bb)
5739 && expand_omp_atomic_fetch_op (load_bb, addr,
5740 loaded_val, stored_val, index))
5743 /* If we don't have specialized __sync builtins, try and implement
5744 as a compare and swap loop. */
5745 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
5746 loaded_val, stored_val, index))
5751 /* The ultimate fallback is wrapping the operation in a mutex. */
5752 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
5756 /* Expand the parallel region tree rooted at REGION. Expansion
5757 proceeds in depth-first order. Innermost regions are expanded
5758 first. This way, parallel regions that require a new function to
5759 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
5760 internal dependencies in their body. */
5763 expand_omp (struct omp_region *region)
5767 location_t saved_location;
5769 /* First, determine whether this is a combined parallel+workshare
5771 if (region->type == GIMPLE_OMP_PARALLEL)
5772 determine_parallel_type (region);
5775 expand_omp (region->inner);
5777 saved_location = input_location;
5778 if (gimple_has_location (last_stmt (region->entry)))
5779 input_location = gimple_location (last_stmt (region->entry));
5781 switch (region->type)
5783 case GIMPLE_OMP_PARALLEL:
5784 case GIMPLE_OMP_TASK:
5785 expand_omp_taskreg (region);
5788 case GIMPLE_OMP_FOR:
5789 expand_omp_for (region);
5792 case GIMPLE_OMP_SECTIONS:
5793 expand_omp_sections (region);
5796 case GIMPLE_OMP_SECTION:
5797 /* Individual omp sections are handled together with their
5798 parent GIMPLE_OMP_SECTIONS region. */
5801 case GIMPLE_OMP_SINGLE:
5802 expand_omp_single (region);
5805 case GIMPLE_OMP_MASTER:
5806 case GIMPLE_OMP_ORDERED:
5807 case GIMPLE_OMP_CRITICAL:
5808 expand_omp_synch (region);
5811 case GIMPLE_OMP_ATOMIC_LOAD:
5812 expand_omp_atomic (region);
5819 input_location = saved_location;
5820 region = region->next;
5825 /* Helper for build_omp_regions. Scan the dominator tree starting at
5826 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
5827 true, the function ends once a single tree is built (otherwise, whole
5828 forest of OMP constructs may be built). */
5831 build_omp_regions_1 (basic_block bb, struct omp_region *parent,
5834 gimple_stmt_iterator gsi;
5838 gsi = gsi_last_bb (bb);
5839 if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi)))
5841 struct omp_region *region;
5842 enum gimple_code code;
5844 stmt = gsi_stmt (gsi);
5845 code = gimple_code (stmt);
5846 if (code == GIMPLE_OMP_RETURN)
5848 /* STMT is the return point out of region PARENT. Mark it
5849 as the exit point and make PARENT the immediately
5850 enclosing region. */
5851 gcc_assert (parent);
5854 parent = parent->outer;
5856 else if (code == GIMPLE_OMP_ATOMIC_STORE)
5858 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
5859 GIMPLE_OMP_RETURN, but matches with
5860 GIMPLE_OMP_ATOMIC_LOAD. */
5861 gcc_assert (parent);
5862 gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD);
5865 parent = parent->outer;
5868 else if (code == GIMPLE_OMP_CONTINUE)
5870 gcc_assert (parent);
5873 else if (code == GIMPLE_OMP_SECTIONS_SWITCH)
5875 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
5876 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
5881 /* Otherwise, this directive becomes the parent for a new
5883 region = new_omp_region (bb, code, parent);
5888 if (single_tree && !parent)
5891 for (son = first_dom_son (CDI_DOMINATORS, bb);
5893 son = next_dom_son (CDI_DOMINATORS, son))
5894 build_omp_regions_1 (son, parent, single_tree);
5897 /* Builds the tree of OMP regions rooted at ROOT, storing it to
5901 build_omp_regions_root (basic_block root)
5903 gcc_assert (root_omp_region == NULL);
5904 build_omp_regions_1 (root, NULL, true);
5905 gcc_assert (root_omp_region != NULL);
5908 /* Expands omp construct (and its subconstructs) starting in HEAD. */
5911 omp_expand_local (basic_block head)
5913 build_omp_regions_root (head);
5914 if (dump_file && (dump_flags & TDF_DETAILS))
5916 fprintf (dump_file, "\nOMP region tree\n\n");
5917 dump_omp_region (dump_file, root_omp_region, 0);
5918 fprintf (dump_file, "\n");
5921 remove_exit_barriers (root_omp_region);
5922 expand_omp (root_omp_region);
5924 free_omp_regions ();
5927 /* Scan the CFG and build a tree of OMP regions. Return the root of
5928 the OMP region tree. */
5931 build_omp_regions (void)
5933 gcc_assert (root_omp_region == NULL);
5934 calculate_dominance_info (CDI_DOMINATORS);
5935 build_omp_regions_1 (ENTRY_BLOCK_PTR, NULL, false);
5938 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
5941 execute_expand_omp (void)
5943 build_omp_regions ();
5945 if (!root_omp_region)
5950 fprintf (dump_file, "\nOMP region tree\n\n");
5951 dump_omp_region (dump_file, root_omp_region, 0);
5952 fprintf (dump_file, "\n");
5955 remove_exit_barriers (root_omp_region);
5957 expand_omp (root_omp_region);
5959 cleanup_tree_cfg ();
5961 free_omp_regions ();
5966 /* OMP expansion -- the default pass, run before creation of SSA form. */
5969 gate_expand_omp (void)
5971 return (flag_openmp != 0 && !seen_error ());
5974 struct gimple_opt_pass pass_expand_omp =
5978 "ompexp", /* name */
5979 OPTGROUP_NONE, /* optinfo_flags */
5980 gate_expand_omp, /* gate */
5981 execute_expand_omp, /* execute */
5984 0, /* static_pass_number */
5985 TV_NONE, /* tv_id */
5986 PROP_gimple_any, /* properties_required */
5987 0, /* properties_provided */
5988 0, /* properties_destroyed */
5989 0, /* todo_flags_start */
5990 0 /* todo_flags_finish */
5994 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
5996 /* Lower the OpenMP sections directive in the current statement in GSI_P.
5997 CTX is the enclosing OMP context for the current statement. */
6000 lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
6002 tree block, control;
6003 gimple_stmt_iterator tgsi;
6004 gimple stmt, new_stmt, bind, t;
6005 gimple_seq ilist, dlist, olist, new_body;
6006 struct gimplify_ctx gctx;
6008 stmt = gsi_stmt (*gsi_p);
6010 push_gimplify_context (&gctx);
6014 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
6015 &ilist, &dlist, ctx);
6017 new_body = gimple_omp_body (stmt);
6018 gimple_omp_set_body (stmt, NULL);
6019 tgsi = gsi_start (new_body);
6020 for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
6025 sec_start = gsi_stmt (tgsi);
6026 sctx = maybe_lookup_ctx (sec_start);
6029 lower_omp (gimple_omp_body_ptr (sec_start), sctx);
6030 gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
6031 GSI_CONTINUE_LINKING);
6032 gimple_omp_set_body (sec_start, NULL);
6034 if (gsi_one_before_end_p (tgsi))
6036 gimple_seq l = NULL;
6037 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
6039 gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
6040 gimple_omp_section_set_last (sec_start);
6043 gsi_insert_after (&tgsi, gimple_build_omp_return (false),
6044 GSI_CONTINUE_LINKING);
6047 block = make_node (BLOCK);
6048 bind = gimple_build_bind (NULL, new_body, block);
6051 lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx);
6053 block = make_node (BLOCK);
6054 new_stmt = gimple_build_bind (NULL, NULL, block);
6055 gsi_replace (gsi_p, new_stmt, true);
6057 pop_gimplify_context (new_stmt);
6058 gimple_bind_append_vars (new_stmt, ctx->block_vars);
6059 BLOCK_VARS (block) = gimple_bind_vars (bind);
6060 if (BLOCK_VARS (block))
6061 TREE_USED (block) = 1;
6064 gimple_seq_add_seq (&new_body, ilist);
6065 gimple_seq_add_stmt (&new_body, stmt);
6066 gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
6067 gimple_seq_add_stmt (&new_body, bind);
6069 control = create_tmp_var (unsigned_type_node, ".section");
6070 t = gimple_build_omp_continue (control, control);
6071 gimple_omp_sections_set_control (stmt, control);
6072 gimple_seq_add_stmt (&new_body, t);
6074 gimple_seq_add_seq (&new_body, olist);
6075 gimple_seq_add_seq (&new_body, dlist);
6077 new_body = maybe_catch_exception (new_body);
6079 t = gimple_build_omp_return
6080 (!!find_omp_clause (gimple_omp_sections_clauses (stmt),
6081 OMP_CLAUSE_NOWAIT));
6082 gimple_seq_add_stmt (&new_body, t);
6084 gimple_bind_set_body (new_stmt, new_body);
6088 /* A subroutine of lower_omp_single. Expand the simple form of
6089 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
6091 if (GOMP_single_start ())
6093 [ GOMP_barrier (); ] -> unless 'nowait' is present.
6095 FIXME. It may be better to delay expanding the logic of this until
6096 pass_expand_omp. The expanded logic may make the job more difficult
6097 to a synchronization analysis pass. */
6100 lower_omp_single_simple (gimple single_stmt, gimple_seq *pre_p)
6102 location_t loc = gimple_location (single_stmt);
6103 tree tlabel = create_artificial_label (loc);
6104 tree flabel = create_artificial_label (loc);
6108 decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START);
6109 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)), NULL);
6110 call = gimple_build_call (decl, 0);
6111 gimple_call_set_lhs (call, lhs);
6112 gimple_seq_add_stmt (pre_p, call);
6114 cond = gimple_build_cond (EQ_EXPR, lhs,
6115 fold_convert_loc (loc, TREE_TYPE (lhs),
6118 gimple_seq_add_stmt (pre_p, cond);
6119 gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
6120 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
6121 gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
6125 /* A subroutine of lower_omp_single. Expand the simple form of
6126 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
6128 #pragma omp single copyprivate (a, b, c)
6130 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
6133 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
6139 GOMP_single_copy_end (©out);
6150 FIXME. It may be better to delay expanding the logic of this until
6151 pass_expand_omp. The expanded logic may make the job more difficult
6152 to a synchronization analysis pass. */
6155 lower_omp_single_copy (gimple single_stmt, gimple_seq *pre_p, omp_context *ctx)
6157 tree ptr_type, t, l0, l1, l2, bfn_decl;
6158 gimple_seq copyin_seq;
6159 location_t loc = gimple_location (single_stmt);
6161 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
6163 ptr_type = build_pointer_type (ctx->record_type);
6164 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
6166 l0 = create_artificial_label (loc);
6167 l1 = create_artificial_label (loc);
6168 l2 = create_artificial_label (loc);
6170 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
6171 t = build_call_expr_loc (loc, bfn_decl, 0);
6172 t = fold_convert_loc (loc, ptr_type, t);
6173 gimplify_assign (ctx->receiver_decl, t, pre_p);
6175 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
6176 build_int_cst (ptr_type, 0));
6177 t = build3 (COND_EXPR, void_type_node, t,
6178 build_and_jump (&l0), build_and_jump (&l1));
6179 gimplify_and_add (t, pre_p);
6181 gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
6183 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
6186 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
6189 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
6190 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
6191 t = build_call_expr_loc (loc, bfn_decl, 1, t);
6192 gimplify_and_add (t, pre_p);
6194 t = build_and_jump (&l2);
6195 gimplify_and_add (t, pre_p);
6197 gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
6199 gimple_seq_add_seq (pre_p, copyin_seq);
6201 gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
6205 /* Expand code for an OpenMP single directive. */
6208 lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
6211 gimple t, bind, single_stmt = gsi_stmt (*gsi_p);
6212 gimple_seq bind_body, dlist;
6213 struct gimplify_ctx gctx;
6215 push_gimplify_context (&gctx);
6217 block = make_node (BLOCK);
6218 bind = gimple_build_bind (NULL, NULL, block);
6219 gsi_replace (gsi_p, bind, true);
6222 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
6223 &bind_body, &dlist, ctx);
6224 lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
6226 gimple_seq_add_stmt (&bind_body, single_stmt);
6228 if (ctx->record_type)
6229 lower_omp_single_copy (single_stmt, &bind_body, ctx);
6231 lower_omp_single_simple (single_stmt, &bind_body);
6233 gimple_omp_set_body (single_stmt, NULL);
6235 gimple_seq_add_seq (&bind_body, dlist);
6237 bind_body = maybe_catch_exception (bind_body);
6239 t = gimple_build_omp_return
6240 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt),
6241 OMP_CLAUSE_NOWAIT));
6242 gimple_seq_add_stmt (&bind_body, t);
6243 gimple_bind_set_body (bind, bind_body);
6245 pop_gimplify_context (bind);
6247 gimple_bind_append_vars (bind, ctx->block_vars);
6248 BLOCK_VARS (block) = ctx->block_vars;
6249 if (BLOCK_VARS (block))
6250 TREE_USED (block) = 1;
6254 /* Expand code for an OpenMP master directive. */
6257 lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
6259 tree block, lab = NULL, x, bfn_decl;
6260 gimple stmt = gsi_stmt (*gsi_p), bind;
6261 location_t loc = gimple_location (stmt);
6263 struct gimplify_ctx gctx;
6265 push_gimplify_context (&gctx);
6267 block = make_node (BLOCK);
6268 bind = gimple_build_bind (NULL, NULL, block);
6269 gsi_replace (gsi_p, bind, true);
6270 gimple_bind_add_stmt (bind, stmt);
6272 bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
6273 x = build_call_expr_loc (loc, bfn_decl, 0);
6274 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node);
6275 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
6277 gimplify_and_add (x, &tseq);
6278 gimple_bind_add_seq (bind, tseq);
6280 lower_omp (gimple_omp_body_ptr (stmt), ctx);
6281 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
6282 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
6283 gimple_omp_set_body (stmt, NULL);
6285 gimple_bind_add_stmt (bind, gimple_build_label (lab));
6287 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
6289 pop_gimplify_context (bind);
6291 gimple_bind_append_vars (bind, ctx->block_vars);
6292 BLOCK_VARS (block) = ctx->block_vars;
6296 /* Expand code for an OpenMP ordered directive. */
6299 lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
6302 gimple stmt = gsi_stmt (*gsi_p), bind, x;
6303 struct gimplify_ctx gctx;
6305 push_gimplify_context (&gctx);
6307 block = make_node (BLOCK);
6308 bind = gimple_build_bind (NULL, NULL, block);
6309 gsi_replace (gsi_p, bind, true);
6310 gimple_bind_add_stmt (bind, stmt);
6312 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
6314 gimple_bind_add_stmt (bind, x);
6316 lower_omp (gimple_omp_body_ptr (stmt), ctx);
6317 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
6318 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
6319 gimple_omp_set_body (stmt, NULL);
6321 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END), 0);
6322 gimple_bind_add_stmt (bind, x);
6324 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
6326 pop_gimplify_context (bind);
6328 gimple_bind_append_vars (bind, ctx->block_vars);
6329 BLOCK_VARS (block) = gimple_bind_vars (bind);
6333 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
6334 substitution of a couple of function calls. But in the NAMED case,
6335 requires that languages coordinate a symbol name. It is therefore
6336 best put here in common code. */
6338 static GTY((param1_is (tree), param2_is (tree)))
6339 splay_tree critical_name_mutexes;
6342 lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
6345 tree name, lock, unlock;
6346 gimple stmt = gsi_stmt (*gsi_p), bind;
6347 location_t loc = gimple_location (stmt);
6349 struct gimplify_ctx gctx;
6351 name = gimple_omp_critical_name (stmt);
6357 if (!critical_name_mutexes)
6358 critical_name_mutexes
6359 = splay_tree_new_ggc (splay_tree_compare_pointers,
6360 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s,
6361 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s);
6363 n = splay_tree_lookup (critical_name_mutexes, (splay_tree_key) name);
6368 decl = create_tmp_var_raw (ptr_type_node, NULL);
6370 new_str = ACONCAT ((".gomp_critical_user_",
6371 IDENTIFIER_POINTER (name), NULL));
6372 DECL_NAME (decl) = get_identifier (new_str);
6373 TREE_PUBLIC (decl) = 1;
6374 TREE_STATIC (decl) = 1;
6375 DECL_COMMON (decl) = 1;
6376 DECL_ARTIFICIAL (decl) = 1;
6377 DECL_IGNORED_P (decl) = 1;
6378 varpool_finalize_decl (decl);
6380 splay_tree_insert (critical_name_mutexes, (splay_tree_key) name,
6381 (splay_tree_value) decl);
6384 decl = (tree) n->value;
6386 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
6387 lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl));
6389 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
6390 unlock = build_call_expr_loc (loc, unlock, 1,
6391 build_fold_addr_expr_loc (loc, decl));
6395 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
6396 lock = build_call_expr_loc (loc, lock, 0);
6398 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
6399 unlock = build_call_expr_loc (loc, unlock, 0);
6402 push_gimplify_context (&gctx);
6404 block = make_node (BLOCK);
6405 bind = gimple_build_bind (NULL, NULL, block);
6406 gsi_replace (gsi_p, bind, true);
6407 gimple_bind_add_stmt (bind, stmt);
6409 tbody = gimple_bind_body (bind);
6410 gimplify_and_add (lock, &tbody);
6411 gimple_bind_set_body (bind, tbody);
6413 lower_omp (gimple_omp_body_ptr (stmt), ctx);
6414 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
6415 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
6416 gimple_omp_set_body (stmt, NULL);
6418 tbody = gimple_bind_body (bind);
6419 gimplify_and_add (unlock, &tbody);
6420 gimple_bind_set_body (bind, tbody);
6422 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
6424 pop_gimplify_context (bind);
6425 gimple_bind_append_vars (bind, ctx->block_vars);
6426 BLOCK_VARS (block) = gimple_bind_vars (bind);
6430 /* A subroutine of lower_omp_for. Generate code to emit the predicate
6431 for a lastprivate clause. Given a loop control predicate of (V
6432 cond N2), we gate the clause on (!(V cond N2)). The lowered form
6433 is appended to *DLIST, iterator initialization is appended to
6437 lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
6438 gimple_seq *dlist, struct omp_context *ctx)
6440 tree clauses, cond, vinit;
6441 enum tree_code cond_code;
6444 cond_code = fd->loop.cond_code;
6445 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
6447 /* When possible, use a strict equality expression. This can let VRP
6448 type optimizations deduce the value and remove a copy. */
6449 if (host_integerp (fd->loop.step, 0))
6451 HOST_WIDE_INT step = TREE_INT_CST_LOW (fd->loop.step);
6452 if (step == 1 || step == -1)
6453 cond_code = EQ_EXPR;
6456 cond = build2 (cond_code, boolean_type_node, fd->loop.v, fd->loop.n2);
6458 clauses = gimple_omp_for_clauses (fd->for_stmt);
6460 lower_lastprivate_clauses (clauses, cond, &stmts, ctx);
6461 if (!gimple_seq_empty_p (stmts))
6463 gimple_seq_add_seq (&stmts, *dlist);
6466 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
6467 vinit = fd->loop.n1;
6468 if (cond_code == EQ_EXPR
6469 && host_integerp (fd->loop.n2, 0)
6470 && ! integer_zerop (fd->loop.n2))
6471 vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
6473 /* Initialize the iterator variable, so that threads that don't execute
6474 any iterations don't execute the lastprivate clauses by accident. */
6475 gimplify_assign (fd->loop.v, vinit, body_p);
6480 /* Lower code for an OpenMP loop directive. */
6483 lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
6486 struct omp_for_data fd;
6487 gimple stmt = gsi_stmt (*gsi_p), new_stmt;
6488 gimple_seq omp_for_body, body, dlist;
6490 struct gimplify_ctx gctx;
6492 push_gimplify_context (&gctx);
6494 lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
6495 lower_omp (gimple_omp_body_ptr (stmt), ctx);
6497 block = make_node (BLOCK);
6498 new_stmt = gimple_build_bind (NULL, NULL, block);
6499 /* Replace at gsi right away, so that 'stmt' is no member
6500 of a sequence anymore as we're going to add to to a different
6502 gsi_replace (gsi_p, new_stmt, true);
6504 /* Move declaration of temporaries in the loop body before we make
6506 omp_for_body = gimple_omp_body (stmt);
6507 if (!gimple_seq_empty_p (omp_for_body)
6508 && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
6510 tree vars = gimple_bind_vars (gimple_seq_first_stmt (omp_for_body));
6511 gimple_bind_append_vars (new_stmt, vars);
6514 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
6517 lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx);
6518 gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt));
6520 /* Lower the header expressions. At this point, we can assume that
6521 the header is of the form:
6523 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
6525 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
6526 using the .omp_data_s mapping, if needed. */
6527 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
6529 rhs_p = gimple_omp_for_initial_ptr (stmt, i);
6530 if (!is_gimple_min_invariant (*rhs_p))
6531 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
6533 rhs_p = gimple_omp_for_final_ptr (stmt, i);
6534 if (!is_gimple_min_invariant (*rhs_p))
6535 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
6537 rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
6538 if (!is_gimple_min_invariant (*rhs_p))
6539 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
6542 /* Once lowered, extract the bounds and clauses. */
6543 extract_omp_for_data (stmt, &fd, NULL);
6545 lower_omp_for_lastprivate (&fd, &body, &dlist, ctx);
6547 gimple_seq_add_stmt (&body, stmt);
6548 gimple_seq_add_seq (&body, gimple_omp_body (stmt));
6550 gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
6553 /* After the loop, add exit clauses. */
6554 lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx);
6555 gimple_seq_add_seq (&body, dlist);
6557 body = maybe_catch_exception (body);
6559 /* Region exit marker goes at the end of the loop body. */
6560 gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait));
6562 pop_gimplify_context (new_stmt);
6564 gimple_bind_append_vars (new_stmt, ctx->block_vars);
6565 BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
6566 if (BLOCK_VARS (block))
6567 TREE_USED (block) = 1;
6569 gimple_bind_set_body (new_stmt, body);
6570 gimple_omp_set_body (stmt, NULL);
6571 gimple_omp_for_set_pre_body (stmt, NULL);
6574 /* Callback for walk_stmts. Check if the current statement only contains
6575 GIMPLE_OMP_FOR or GIMPLE_OMP_PARALLEL. */
6578 check_combined_parallel (gimple_stmt_iterator *gsi_p,
6579 bool *handled_ops_p,
6580 struct walk_stmt_info *wi)
6582 int *info = (int *) wi->info;
6583 gimple stmt = gsi_stmt (*gsi_p);
6585 *handled_ops_p = true;
6586 switch (gimple_code (stmt))
6590 case GIMPLE_OMP_FOR:
6591 case GIMPLE_OMP_SECTIONS:
6592 *info = *info == 0 ? 1 : -1;
6601 struct omp_taskcopy_context
6603 /* This field must be at the beginning, as we do "inheritance": Some
6604 callback functions for tree-inline.c (e.g., omp_copy_decl)
6605 receive a copy_body_data pointer that is up-casted to an
6606 omp_context pointer. */
6612 task_copyfn_copy_decl (tree var, copy_body_data *cb)
6614 struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
6616 if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
6617 return create_tmp_var (TREE_TYPE (var), NULL);
6623 task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
6625 tree name, new_fields = NULL, type, f;
6627 type = lang_hooks.types.make_type (RECORD_TYPE);
6628 name = DECL_NAME (TYPE_NAME (orig_type));
6629 name = build_decl (gimple_location (tcctx->ctx->stmt),
6630 TYPE_DECL, name, type);
6631 TYPE_NAME (type) = name;
6633 for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
6635 tree new_f = copy_node (f);
6636 DECL_CONTEXT (new_f) = type;
6637 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
6638 TREE_CHAIN (new_f) = new_fields;
6639 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
6640 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
6641 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
6644 *pointer_map_insert (tcctx->cb.decl_map, f) = new_f;
6646 TYPE_FIELDS (type) = nreverse (new_fields);
6651 /* Create task copyfn. */
6654 create_task_copyfn (gimple task_stmt, omp_context *ctx)
6656 struct function *child_cfun;
6657 tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
6658 tree record_type, srecord_type, bind, list;
6659 bool record_needs_remap = false, srecord_needs_remap = false;
6661 struct omp_taskcopy_context tcctx;
6662 struct gimplify_ctx gctx;
6663 location_t loc = gimple_location (task_stmt);
6665 child_fn = gimple_omp_task_copy_fn (task_stmt);
6666 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
6667 gcc_assert (child_cfun->cfg == NULL);
6668 DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
6670 /* Reset DECL_CONTEXT on function arguments. */
6671 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
6672 DECL_CONTEXT (t) = child_fn;
6674 /* Populate the function. */
6675 push_gimplify_context (&gctx);
6676 push_cfun (child_cfun);
6678 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
6679 TREE_SIDE_EFFECTS (bind) = 1;
6681 DECL_SAVED_TREE (child_fn) = bind;
6682 DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
6684 /* Remap src and dst argument types if needed. */
6685 record_type = ctx->record_type;
6686 srecord_type = ctx->srecord_type;
6687 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
6688 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
6690 record_needs_remap = true;
6693 for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
6694 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
6696 srecord_needs_remap = true;
6700 if (record_needs_remap || srecord_needs_remap)
6702 memset (&tcctx, '\0', sizeof (tcctx));
6703 tcctx.cb.src_fn = ctx->cb.src_fn;
6704 tcctx.cb.dst_fn = child_fn;
6705 tcctx.cb.src_node = cgraph_get_node (tcctx.cb.src_fn);
6706 gcc_checking_assert (tcctx.cb.src_node);
6707 tcctx.cb.dst_node = tcctx.cb.src_node;
6708 tcctx.cb.src_cfun = ctx->cb.src_cfun;
6709 tcctx.cb.copy_decl = task_copyfn_copy_decl;
6710 tcctx.cb.eh_lp_nr = 0;
6711 tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
6712 tcctx.cb.decl_map = pointer_map_create ();
6715 if (record_needs_remap)
6716 record_type = task_copyfn_remap_type (&tcctx, record_type);
6717 if (srecord_needs_remap)
6718 srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
6721 tcctx.cb.decl_map = NULL;
6723 arg = DECL_ARGUMENTS (child_fn);
6724 TREE_TYPE (arg) = build_pointer_type (record_type);
6725 sarg = DECL_CHAIN (arg);
6726 TREE_TYPE (sarg) = build_pointer_type (srecord_type);
6728 /* First pass: initialize temporaries used in record_type and srecord_type
6729 sizes and field offsets. */
6730 if (tcctx.cb.decl_map)
6731 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
6732 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
6736 decl = OMP_CLAUSE_DECL (c);
6737 p = (tree *) pointer_map_contains (tcctx.cb.decl_map, decl);
6740 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
6741 sf = (tree) n->value;
6742 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
6743 src = build_simple_mem_ref_loc (loc, sarg);
6744 src = omp_build_component_ref (src, sf);
6745 t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
6746 append_to_statement_list (t, &list);
6749 /* Second pass: copy shared var pointers and copy construct non-VLA
6750 firstprivate vars. */
6751 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
6752 switch (OMP_CLAUSE_CODE (c))
6754 case OMP_CLAUSE_SHARED:
6755 decl = OMP_CLAUSE_DECL (c);
6756 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
6759 f = (tree) n->value;
6760 if (tcctx.cb.decl_map)
6761 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
6762 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
6763 sf = (tree) n->value;
6764 if (tcctx.cb.decl_map)
6765 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
6766 src = build_simple_mem_ref_loc (loc, sarg);
6767 src = omp_build_component_ref (src, sf);
6768 dst = build_simple_mem_ref_loc (loc, arg);
6769 dst = omp_build_component_ref (dst, f);
6770 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
6771 append_to_statement_list (t, &list);
6773 case OMP_CLAUSE_FIRSTPRIVATE:
6774 decl = OMP_CLAUSE_DECL (c);
6775 if (is_variable_sized (decl))
6777 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
6780 f = (tree) n->value;
6781 if (tcctx.cb.decl_map)
6782 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
6783 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
6786 sf = (tree) n->value;
6787 if (tcctx.cb.decl_map)
6788 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
6789 src = build_simple_mem_ref_loc (loc, sarg);
6790 src = omp_build_component_ref (src, sf);
6791 if (use_pointer_for_field (decl, NULL) || is_reference (decl))
6792 src = build_simple_mem_ref_loc (loc, src);
6796 dst = build_simple_mem_ref_loc (loc, arg);
6797 dst = omp_build_component_ref (dst, f);
6798 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
6799 append_to_statement_list (t, &list);
6801 case OMP_CLAUSE_PRIVATE:
6802 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
6804 decl = OMP_CLAUSE_DECL (c);
6805 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
6806 f = (tree) n->value;
6807 if (tcctx.cb.decl_map)
6808 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
6809 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
6812 sf = (tree) n->value;
6813 if (tcctx.cb.decl_map)
6814 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
6815 src = build_simple_mem_ref_loc (loc, sarg);
6816 src = omp_build_component_ref (src, sf);
6817 if (use_pointer_for_field (decl, NULL))
6818 src = build_simple_mem_ref_loc (loc, src);
6822 dst = build_simple_mem_ref_loc (loc, arg);
6823 dst = omp_build_component_ref (dst, f);
6824 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
6825 append_to_statement_list (t, &list);
6831 /* Last pass: handle VLA firstprivates. */
6832 if (tcctx.cb.decl_map)
6833 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
6834 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
6838 decl = OMP_CLAUSE_DECL (c);
6839 if (!is_variable_sized (decl))
6841 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
6844 f = (tree) n->value;
6845 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
6846 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
6847 ind = DECL_VALUE_EXPR (decl);
6848 gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
6849 gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
6850 n = splay_tree_lookup (ctx->sfield_map,
6851 (splay_tree_key) TREE_OPERAND (ind, 0));
6852 sf = (tree) n->value;
6853 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
6854 src = build_simple_mem_ref_loc (loc, sarg);
6855 src = omp_build_component_ref (src, sf);
6856 src = build_simple_mem_ref_loc (loc, src);
6857 dst = build_simple_mem_ref_loc (loc, arg);
6858 dst = omp_build_component_ref (dst, f);
6859 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
6860 append_to_statement_list (t, &list);
6861 n = splay_tree_lookup (ctx->field_map,
6862 (splay_tree_key) TREE_OPERAND (ind, 0));
6863 df = (tree) n->value;
6864 df = *(tree *) pointer_map_contains (tcctx.cb.decl_map, df);
6865 ptr = build_simple_mem_ref_loc (loc, arg);
6866 ptr = omp_build_component_ref (ptr, df);
6867 t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
6868 build_fold_addr_expr_loc (loc, dst));
6869 append_to_statement_list (t, &list);
6872 t = build1 (RETURN_EXPR, void_type_node, NULL);
6873 append_to_statement_list (t, &list);
6875 if (tcctx.cb.decl_map)
6876 pointer_map_destroy (tcctx.cb.decl_map);
6877 pop_gimplify_context (NULL);
6878 BIND_EXPR_BODY (bind) = list;
6882 /* Lower the OpenMP parallel or task directive in the current statement
6883 in GSI_P. CTX holds context information for the directive. */
6886 lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
6890 gimple stmt = gsi_stmt (*gsi_p);
6891 gimple par_bind, bind;
6892 gimple_seq par_body, olist, ilist, par_olist, par_ilist, new_body;
6893 struct gimplify_ctx gctx;
6894 location_t loc = gimple_location (stmt);
6896 clauses = gimple_omp_taskreg_clauses (stmt);
6897 par_bind = gimple_seq_first_stmt (gimple_omp_body (stmt));
6898 par_body = gimple_bind_body (par_bind);
6899 child_fn = ctx->cb.dst_fn;
6900 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
6901 && !gimple_omp_parallel_combined_p (stmt))
6903 struct walk_stmt_info wi;
6906 memset (&wi, 0, sizeof (wi));
6909 walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
6911 gimple_omp_parallel_set_combined_p (stmt, true);
6913 if (ctx->srecord_type)
6914 create_task_copyfn (stmt, ctx);
6916 push_gimplify_context (&gctx);
6920 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx);
6921 lower_omp (&par_body, ctx);
6922 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL)
6923 lower_reduction_clauses (clauses, &par_olist, ctx);
6925 /* Declare all the variables created by mapping and the variables
6926 declared in the scope of the parallel body. */
6927 record_vars_into (ctx->block_vars, child_fn);
6928 record_vars_into (gimple_bind_vars (par_bind), child_fn);
6930 if (ctx->record_type)
6933 = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
6934 : ctx->record_type, ".omp_data_o");
6935 DECL_NAMELESS (ctx->sender_decl) = 1;
6936 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
6937 gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
6942 lower_send_clauses (clauses, &ilist, &olist, ctx);
6943 lower_send_shared_vars (&ilist, &olist, ctx);
6945 /* Once all the expansions are done, sequence all the different
6946 fragments inside gimple_omp_body. */
6950 if (ctx->record_type)
6952 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
6953 /* fixup_child_record_type might have changed receiver_decl's type. */
6954 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
6955 gimple_seq_add_stmt (&new_body,
6956 gimple_build_assign (ctx->receiver_decl, t));
6959 gimple_seq_add_seq (&new_body, par_ilist);
6960 gimple_seq_add_seq (&new_body, par_body);
6961 gimple_seq_add_seq (&new_body, par_olist);
6962 new_body = maybe_catch_exception (new_body);
6963 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
6964 gimple_omp_set_body (stmt, new_body);
6966 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
6967 gsi_replace (gsi_p, bind, true);
6968 gimple_bind_add_seq (bind, ilist);
6969 gimple_bind_add_stmt (bind, stmt);
6970 gimple_bind_add_seq (bind, olist);
6972 pop_gimplify_context (NULL);
6975 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
6976 regimplified. If DATA is non-NULL, lower_omp_1 is outside
6977 of OpenMP context, but with task_shared_vars set. */
6980 lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
6985 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
6986 if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t))
6989 if (task_shared_vars
6991 && bitmap_bit_p (task_shared_vars, DECL_UID (t)))
6994 /* If a global variable has been privatized, TREE_CONSTANT on
6995 ADDR_EXPR might be wrong. */
6996 if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
6997 recompute_tree_invariant_for_addr_expr (t);
6999 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
7004 lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
7006 gimple stmt = gsi_stmt (*gsi_p);
7007 struct walk_stmt_info wi;
7009 if (gimple_has_location (stmt))
7010 input_location = gimple_location (stmt);
7012 if (task_shared_vars)
7013 memset (&wi, '\0', sizeof (wi));
7015 /* If we have issued syntax errors, avoid doing any heavy lifting.
7016 Just replace the OpenMP directives with a NOP to avoid
7017 confusing RTL expansion. */
7018 if (seen_error () && is_gimple_omp (stmt))
7020 gsi_replace (gsi_p, gimple_build_nop (), true);
7024 switch (gimple_code (stmt))
7027 if ((ctx || task_shared_vars)
7028 && (walk_tree (gimple_cond_lhs_ptr (stmt), lower_omp_regimplify_p,
7029 ctx ? NULL : &wi, NULL)
7030 || walk_tree (gimple_cond_rhs_ptr (stmt), lower_omp_regimplify_p,
7031 ctx ? NULL : &wi, NULL)))
7032 gimple_regimplify_operands (stmt, gsi_p);
7035 lower_omp (gimple_catch_handler_ptr (stmt), ctx);
7037 case GIMPLE_EH_FILTER:
7038 lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
7041 lower_omp (gimple_try_eval_ptr (stmt), ctx);
7042 lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
7044 case GIMPLE_TRANSACTION:
7045 lower_omp (gimple_transaction_body_ptr (stmt), ctx);
7048 lower_omp (gimple_bind_body_ptr (stmt), ctx);
7050 case GIMPLE_OMP_PARALLEL:
7051 case GIMPLE_OMP_TASK:
7052 ctx = maybe_lookup_ctx (stmt);
7053 lower_omp_taskreg (gsi_p, ctx);
7055 case GIMPLE_OMP_FOR:
7056 ctx = maybe_lookup_ctx (stmt);
7058 lower_omp_for (gsi_p, ctx);
7060 case GIMPLE_OMP_SECTIONS:
7061 ctx = maybe_lookup_ctx (stmt);
7063 lower_omp_sections (gsi_p, ctx);
7065 case GIMPLE_OMP_SINGLE:
7066 ctx = maybe_lookup_ctx (stmt);
7068 lower_omp_single (gsi_p, ctx);
7070 case GIMPLE_OMP_MASTER:
7071 ctx = maybe_lookup_ctx (stmt);
7073 lower_omp_master (gsi_p, ctx);
7075 case GIMPLE_OMP_ORDERED:
7076 ctx = maybe_lookup_ctx (stmt);
7078 lower_omp_ordered (gsi_p, ctx);
7080 case GIMPLE_OMP_CRITICAL:
7081 ctx = maybe_lookup_ctx (stmt);
7083 lower_omp_critical (gsi_p, ctx);
7085 case GIMPLE_OMP_ATOMIC_LOAD:
7086 if ((ctx || task_shared_vars)
7087 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt),
7088 lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
7089 gimple_regimplify_operands (stmt, gsi_p);
7092 if ((ctx || task_shared_vars)
7093 && walk_gimple_op (stmt, lower_omp_regimplify_p,
7095 gimple_regimplify_operands (stmt, gsi_p);
7101 lower_omp (gimple_seq *body, omp_context *ctx)
7103 location_t saved_location = input_location;
7104 gimple_stmt_iterator gsi;
7105 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
7106 lower_omp_1 (&gsi, ctx);
7107 input_location = saved_location;
7110 /* Main entry point. */
7113 execute_lower_omp (void)
7117 /* This pass always runs, to provide PROP_gimple_lomp.
7118 But there is nothing to do unless -fopenmp is given. */
7119 if (flag_openmp == 0)
7122 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
7123 delete_omp_context);
7125 body = gimple_body (current_function_decl);
7126 scan_omp (&body, NULL);
7127 gcc_assert (taskreg_nesting_level == 0);
7129 if (all_contexts->root)
7131 struct gimplify_ctx gctx;
7133 if (task_shared_vars)
7134 push_gimplify_context (&gctx);
7135 lower_omp (&body, NULL);
7136 if (task_shared_vars)
7137 pop_gimplify_context (NULL);
7142 splay_tree_delete (all_contexts);
7143 all_contexts = NULL;
7145 BITMAP_FREE (task_shared_vars);
7149 struct gimple_opt_pass pass_lower_omp =
7153 "omplower", /* name */
7154 OPTGROUP_NONE, /* optinfo_flags */
7156 execute_lower_omp, /* execute */
7159 0, /* static_pass_number */
7160 TV_NONE, /* tv_id */
7161 PROP_gimple_any, /* properties_required */
7162 PROP_gimple_lomp, /* properties_provided */
7163 0, /* properties_destroyed */
7164 0, /* todo_flags_start */
7165 0 /* todo_flags_finish */
7169 /* The following is a utility to diagnose OpenMP structured block violations.
7170 It is not part of the "omplower" pass, as that's invoked too late. It
7171 should be invoked by the respective front ends after gimplification. */
7173 static splay_tree all_labels;
7175 /* Check for mismatched contexts and generate an error if needed. Return
7176 true if an error is detected. */
7179 diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
7180 gimple branch_ctx, gimple label_ctx)
7182 if (label_ctx == branch_ctx)
7187 Previously we kept track of the label's entire context in diagnose_sb_[12]
7188 so we could traverse it and issue a correct "exit" or "enter" error
7189 message upon a structured block violation.
7191 We built the context by building a list with tree_cons'ing, but there is
7192 no easy counterpart in gimple tuples. It seems like far too much work
7193 for issuing exit/enter error messages. If someone really misses the
7194 distinct error message... patches welcome.
7198 /* Try to avoid confusing the user by producing and error message
7199 with correct "exit" or "enter" verbiage. We prefer "exit"
7200 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
7201 if (branch_ctx == NULL)
7207 if (TREE_VALUE (label_ctx) == branch_ctx)
7212 label_ctx = TREE_CHAIN (label_ctx);
7217 error ("invalid exit from OpenMP structured block");
7219 error ("invalid entry to OpenMP structured block");
7222 /* If it's obvious we have an invalid entry, be specific about the error. */
7223 if (branch_ctx == NULL)
7224 error ("invalid entry to OpenMP structured block");
7226 /* Otherwise, be vague and lazy, but efficient. */
7227 error ("invalid branch to/from an OpenMP structured block");
7229 gsi_replace (gsi_p, gimple_build_nop (), false);
7233 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
7234 where each label is found. */
7237 diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
7238 struct walk_stmt_info *wi)
7240 gimple context = (gimple) wi->info;
7241 gimple inner_context;
7242 gimple stmt = gsi_stmt (*gsi_p);
7244 *handled_ops_p = true;
7246 switch (gimple_code (stmt))
7250 case GIMPLE_OMP_PARALLEL:
7251 case GIMPLE_OMP_TASK:
7252 case GIMPLE_OMP_SECTIONS:
7253 case GIMPLE_OMP_SINGLE:
7254 case GIMPLE_OMP_SECTION:
7255 case GIMPLE_OMP_MASTER:
7256 case GIMPLE_OMP_ORDERED:
7257 case GIMPLE_OMP_CRITICAL:
7258 /* The minimal context here is just the current OMP construct. */
7259 inner_context = stmt;
7260 wi->info = inner_context;
7261 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
7265 case GIMPLE_OMP_FOR:
7266 inner_context = stmt;
7267 wi->info = inner_context;
7268 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
7270 walk_gimple_seq (gimple_omp_for_pre_body (stmt),
7271 diagnose_sb_1, NULL, wi);
7272 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
7277 splay_tree_insert (all_labels, (splay_tree_key) gimple_label_label (stmt),
7278 (splay_tree_value) context);
7288 /* Pass 2: Check each branch and see if its context differs from that of
7289 the destination label's context. */
7292 diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
7293 struct walk_stmt_info *wi)
7295 gimple context = (gimple) wi->info;
7297 gimple stmt = gsi_stmt (*gsi_p);
7299 *handled_ops_p = true;
7301 switch (gimple_code (stmt))
7305 case GIMPLE_OMP_PARALLEL:
7306 case GIMPLE_OMP_TASK:
7307 case GIMPLE_OMP_SECTIONS:
7308 case GIMPLE_OMP_SINGLE:
7309 case GIMPLE_OMP_SECTION:
7310 case GIMPLE_OMP_MASTER:
7311 case GIMPLE_OMP_ORDERED:
7312 case GIMPLE_OMP_CRITICAL:
7314 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
7318 case GIMPLE_OMP_FOR:
7320 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
7322 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
7323 diagnose_sb_2, NULL, wi);
7324 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
7330 tree lab = gimple_cond_true_label (stmt);
7333 n = splay_tree_lookup (all_labels,
7334 (splay_tree_key) lab);
7335 diagnose_sb_0 (gsi_p, context,
7336 n ? (gimple) n->value : NULL);
7338 lab = gimple_cond_false_label (stmt);
7341 n = splay_tree_lookup (all_labels,
7342 (splay_tree_key) lab);
7343 diagnose_sb_0 (gsi_p, context,
7344 n ? (gimple) n->value : NULL);
7351 tree lab = gimple_goto_dest (stmt);
7352 if (TREE_CODE (lab) != LABEL_DECL)
7355 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
7356 diagnose_sb_0 (gsi_p, context, n ? (gimple) n->value : NULL);
7363 for (i = 0; i < gimple_switch_num_labels (stmt); ++i)
7365 tree lab = CASE_LABEL (gimple_switch_label (stmt, i));
7366 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
7367 if (n && diagnose_sb_0 (gsi_p, context, (gimple) n->value))
7374 diagnose_sb_0 (gsi_p, context, NULL);
7385 diagnose_omp_structured_block_errors (void)
7387 struct walk_stmt_info wi;
7388 gimple_seq body = gimple_body (current_function_decl);
7390 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
7392 memset (&wi, 0, sizeof (wi));
7393 walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
7395 memset (&wi, 0, sizeof (wi));
7396 wi.want_locations = true;
7397 walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
7399 gimple_set_body (current_function_decl, body);
7401 splay_tree_delete (all_labels);
7408 gate_diagnose_omp_blocks (void)
7410 return flag_openmp != 0;
7413 struct gimple_opt_pass pass_diagnose_omp_blocks =
7417 "*diagnose_omp_blocks", /* name */
7418 OPTGROUP_NONE, /* optinfo_flags */
7419 gate_diagnose_omp_blocks, /* gate */
7420 diagnose_omp_structured_block_errors, /* execute */
7423 0, /* static_pass_number */
7424 TV_NONE, /* tv_id */
7425 PROP_gimple_any, /* properties_required */
7426 0, /* properties_provided */
7427 0, /* properties_destroyed */
7428 0, /* todo_flags_start */
7429 0, /* todo_flags_finish */
7433 #include "gt-omp-low.h"