1 /* A pass for lowering trees to RTL.
2 Copyright (C) 2004-2022 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
11 GCC is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "tree-pass.h"
34 #include "regs.h" /* For reg_renumber. */
38 #include "diagnostic.h"
39 #include "fold-const.h"
41 #include "stor-layout.h"
43 #include "print-tree.h"
47 #include "cfgcleanup.h"
52 #include "internal-fn.h"
54 #include "gimple-iterator.h"
55 #include "gimple-expr.h"
56 #include "gimple-walk.h"
61 #include "gimple-pretty-print.h"
64 #include "tree-inline.h"
65 #include "value-prof.h"
66 #include "tree-ssa-live.h"
67 #include "tree-outof-ssa.h"
69 #include "insn-attr.h" /* For INSN_SCHEDULING. */
70 #include "stringpool.h"
73 #include "tree-ssa-address.h"
78 /* Some systems use __main in a way incompatible with its use in gcc, in these
79 cases use the macros NAME__MAIN to give a quoted symbol and SYMBOL__MAIN to
80 give the same symbol without quotes for an alternative entry point. You
81 must define both, or neither. */
83 #define NAME__MAIN "__main"
86 /* This variable holds information helping the rewriting of SSA trees
90 /* This variable holds the currently expanded gimple statement for purposes
91 of comminucating the profile info to the builtin expanders. */
92 gimple *currently_expanding_gimple_stmt;
94 static rtx expand_debug_expr (tree);
96 static bool defer_stack_allocation (tree, bool);
98 static void record_alignment_for_reg_var (unsigned int);
100 /* Return an expression tree corresponding to the RHS of GIMPLE
104 gimple_assign_rhs_to_tree (gimple *stmt)
107 switch (gimple_assign_rhs_class (stmt))
109 case GIMPLE_TERNARY_RHS:
110 t = build3 (gimple_assign_rhs_code (stmt),
111 TREE_TYPE (gimple_assign_lhs (stmt)),
112 gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt),
113 gimple_assign_rhs3 (stmt));
115 case GIMPLE_BINARY_RHS:
116 t = build2 (gimple_assign_rhs_code (stmt),
117 TREE_TYPE (gimple_assign_lhs (stmt)),
118 gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt));
120 case GIMPLE_UNARY_RHS:
121 t = build1 (gimple_assign_rhs_code (stmt),
122 TREE_TYPE (gimple_assign_lhs (stmt)),
123 gimple_assign_rhs1 (stmt));
125 case GIMPLE_SINGLE_RHS:
127 t = gimple_assign_rhs1 (stmt);
128 /* Avoid modifying this tree in place below. */
129 if ((gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t)
130 && gimple_location (stmt) != EXPR_LOCATION (t))
131 || (gimple_block (stmt) && currently_expanding_to_rtl
140 if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t))
141 SET_EXPR_LOCATION (t, gimple_location (stmt));
147 #ifndef STACK_ALIGNMENT_NEEDED
148 #define STACK_ALIGNMENT_NEEDED 1
151 #define SSAVAR(x) (TREE_CODE (x) == SSA_NAME ? SSA_NAME_VAR (x) : x)
153 /* Choose either CUR or NEXT as the leader DECL for a partition.
154 Prefer ignored decls, to simplify debug dumps and reduce ambiguity
155 out of the same user variable being in multiple partitions (this is
156 less likely for compiler-introduced temps). */
159 leader_merge (tree cur, tree next)
161 if (cur == NULL || cur == next)
164 if (DECL_P (cur) && DECL_IGNORED_P (cur))
167 if (DECL_P (next) && DECL_IGNORED_P (next))
173 /* Associate declaration T with storage space X. If T is no
174 SSA name this is exactly SET_DECL_RTL, otherwise make the
175 partition of T associated with X. */
177 set_rtl (tree t, rtx x)
179 gcc_checking_assert (!x
180 || !(TREE_CODE (t) == SSA_NAME || is_gimple_reg (t))
181 || (use_register_for_decl (t)
183 || (GET_CODE (x) == CONCAT
184 && (REG_P (XEXP (x, 0))
185 || SUBREG_P (XEXP (x, 0)))
186 && (REG_P (XEXP (x, 1))
187 || SUBREG_P (XEXP (x, 1))))
188 /* We need to accept PARALLELs for RESUT_DECLs
189 because of vector types with BLKmode returned
190 in multiple registers, but they are supposed
191 to be uncoalesced. */
192 || (GET_CODE (x) == PARALLEL
194 && TREE_CODE (SSAVAR (t)) == RESULT_DECL
195 && (GET_MODE (x) == BLKmode
196 || !flag_tree_coalesce_vars)))
197 : (MEM_P (x) || x == pc_rtx
198 || (GET_CODE (x) == CONCAT
199 && MEM_P (XEXP (x, 0))
200 && MEM_P (XEXP (x, 1))))));
201 /* Check that the RTL for SSA_NAMEs and gimple-reg PARM_DECLs and
202 RESULT_DECLs has the expected mode. For memory, we accept
203 unpromoted modes, since that's what we're likely to get. For
204 PARM_DECLs and RESULT_DECLs, we'll have been called by
205 set_parm_rtl, which will give us the default def, so we don't
206 have to compute it ourselves. For RESULT_DECLs, we accept mode
207 mismatches too, as long as we have BLKmode or are not coalescing
208 across variables, so that we don't reject BLKmode PARALLELs or
210 gcc_checking_assert (!x || x == pc_rtx || TREE_CODE (t) != SSA_NAME
212 && TREE_CODE (SSAVAR (t)) == RESULT_DECL
213 && (promote_ssa_mode (t, NULL) == BLKmode
214 || !flag_tree_coalesce_vars))
215 || !use_register_for_decl (t)
216 || GET_MODE (x) == promote_ssa_mode (t, NULL));
221 tree cur = NULL_TREE;
229 else if (SUBREG_P (xm))
231 gcc_assert (subreg_lowpart_p (xm));
232 xm = SUBREG_REG (xm);
235 else if (GET_CODE (xm) == CONCAT)
240 else if (GET_CODE (xm) == PARALLEL)
242 xm = XVECEXP (xm, 0, 0);
243 gcc_assert (GET_CODE (xm) == EXPR_LIST);
247 else if (xm == pc_rtx)
252 tree next = skip ? cur : leader_merge (cur, SSAVAR (t) ? SSAVAR (t) : t);
257 set_mem_attributes (x,
258 next && TREE_CODE (next) == SSA_NAME
262 set_reg_attrs_for_decl_rtl (next, x);
266 if (TREE_CODE (t) == SSA_NAME)
268 int part = var_to_partition (SA.map, t);
269 if (part != NO_PARTITION)
271 if (SA.partition_to_pseudo[part])
272 gcc_assert (SA.partition_to_pseudo[part] == x);
273 else if (x != pc_rtx)
274 SA.partition_to_pseudo[part] = x;
276 /* For the benefit of debug information at -O0 (where
277 vartracking doesn't run) record the place also in the base
278 DECL. For PARMs and RESULTs, do so only when setting the
280 if (x && x != pc_rtx && SSA_NAME_VAR (t)
281 && (VAR_P (SSA_NAME_VAR (t))
282 || SSA_NAME_IS_DEFAULT_DEF (t)))
284 tree var = SSA_NAME_VAR (t);
285 /* If we don't yet have something recorded, just record it now. */
286 if (!DECL_RTL_SET_P (var))
287 SET_DECL_RTL (var, x);
288 /* If we have it set already to "multiple places" don't
290 else if (DECL_RTL (var) == pc_rtx)
292 /* If we have something recorded and it's not the same place
293 as we want to record now, we have multiple partitions for the
294 same base variable, with different places. We can't just
295 randomly chose one, hence we have to say that we don't know.
296 This only happens with optimization, and there var-tracking
297 will figure out the right thing. */
298 else if (DECL_RTL (var) != x)
299 SET_DECL_RTL (var, pc_rtx);
306 /* This structure holds data relevant to one variable that will be
307 placed in a stack slot. */
314 /* Initially, the size of the variable. Later, the size of the partition,
315 if this variable becomes it's partition's representative. */
318 /* The *byte* alignment required for this variable. Or as, with the
319 size, the alignment for this partition. */
322 /* The partition representative. */
323 size_t representative;
325 /* The next stack variable in the partition, or EOC. */
328 /* The numbers of conflicting stack variables. */
332 #define EOC ((size_t)-1)
334 /* We have an array of such objects while deciding allocation. */
335 static class stack_var *stack_vars;
336 static size_t stack_vars_alloc;
337 static size_t stack_vars_num;
338 static hash_map<tree, size_t> *decl_to_stack_part;
340 /* Conflict bitmaps go on this obstack. This allows us to destroy
341 all of them in one big sweep. */
342 static bitmap_obstack stack_var_bitmap_obstack;
344 /* An array of indices such that stack_vars[stack_vars_sorted[i]].size
345 is non-decreasing. */
346 static size_t *stack_vars_sorted;
348 /* The phase of the stack frame. This is the known misalignment of
349 virtual_stack_vars_rtx from PREFERRED_STACK_BOUNDARY. That is,
350 (frame_offset+frame_phase) % PREFERRED_STACK_BOUNDARY == 0. */
351 static int frame_phase;
353 /* Used during expand_used_vars to remember if we saw any decls for
354 which we'd like to enable stack smashing protection. */
355 static bool has_protected_decls;
357 /* Used during expand_used_vars. Remember if we say a character buffer
358 smaller than our cutoff threshold. Used for -Wstack-protector. */
359 static bool has_short_buffer;
361 /* Compute the byte alignment to use for DECL. Ignore alignment
362 we can't do with expected alignment of the stack boundary. */
365 align_local_variable (tree decl, bool really_expand)
369 if (TREE_CODE (decl) == SSA_NAME)
371 tree type = TREE_TYPE (decl);
372 machine_mode mode = TYPE_MODE (type);
374 align = TYPE_ALIGN (type);
376 && align < GET_MODE_ALIGNMENT (mode))
377 align = GET_MODE_ALIGNMENT (mode);
380 align = LOCAL_DECL_ALIGNMENT (decl);
382 if (hwasan_sanitize_stack_p ())
383 align = MAX (align, (unsigned) HWASAN_TAG_GRANULE_SIZE * BITS_PER_UNIT);
385 if (TREE_CODE (decl) != SSA_NAME && really_expand)
386 /* Don't change DECL_ALIGN when called from estimated_stack_frame_size.
387 That is done before IPA and could bump alignment based on host
388 backend even for offloaded code which wants different
389 LOCAL_DECL_ALIGNMENT. */
390 SET_DECL_ALIGN (decl, align);
392 return align / BITS_PER_UNIT;
395 /* Align given offset BASE with ALIGN. Truncate up if ALIGN_UP is true,
396 down otherwise. Return truncated BASE value. */
398 static inline unsigned HOST_WIDE_INT
399 align_base (HOST_WIDE_INT base, unsigned HOST_WIDE_INT align, bool align_up)
401 return align_up ? (base + align - 1) & -align : base & -align;
404 /* Allocate SIZE bytes at byte alignment ALIGN from the stack frame.
405 Return the frame offset. */
408 alloc_stack_frame_space (poly_int64 size, unsigned HOST_WIDE_INT align)
410 poly_int64 offset, new_frame_offset;
412 if (FRAME_GROWS_DOWNWARD)
415 = aligned_lower_bound (frame_offset - frame_phase - size,
416 align) + frame_phase;
417 offset = new_frame_offset;
422 = aligned_upper_bound (frame_offset - frame_phase,
423 align) + frame_phase;
424 offset = new_frame_offset;
425 new_frame_offset += size;
427 frame_offset = new_frame_offset;
429 if (frame_offset_overflow (frame_offset, cfun->decl))
430 frame_offset = offset = 0;
435 /* Ensure that the stack is aligned to ALIGN bytes.
436 Return the new frame offset. */
438 align_frame_offset (unsigned HOST_WIDE_INT align)
440 return alloc_stack_frame_space (0, align);
443 /* Accumulate DECL into STACK_VARS. */
446 add_stack_var (tree decl, bool really_expand)
450 if (stack_vars_num >= stack_vars_alloc)
452 if (stack_vars_alloc)
453 stack_vars_alloc = stack_vars_alloc * 3 / 2;
455 stack_vars_alloc = 32;
457 = XRESIZEVEC (class stack_var, stack_vars, stack_vars_alloc);
459 if (!decl_to_stack_part)
460 decl_to_stack_part = new hash_map<tree, size_t>;
462 v = &stack_vars[stack_vars_num];
463 decl_to_stack_part->put (decl, stack_vars_num);
466 tree size = TREE_CODE (decl) == SSA_NAME
467 ? TYPE_SIZE_UNIT (TREE_TYPE (decl))
468 : DECL_SIZE_UNIT (decl);
469 v->size = tree_to_poly_uint64 (size);
470 /* Ensure that all variables have size, so that &a != &b for any two
471 variables that are simultaneously live. */
472 if (known_eq (v->size, 0U))
474 v->alignb = align_local_variable (decl, really_expand);
475 /* An alignment of zero can mightily confuse us later. */
476 gcc_assert (v->alignb != 0);
478 /* All variables are initially in their own partition. */
479 v->representative = stack_vars_num;
482 /* All variables initially conflict with no other. */
485 /* Ensure that this decl doesn't get put onto the list twice. */
486 set_rtl (decl, pc_rtx);
491 /* Make the decls associated with luid's X and Y conflict. */
494 add_stack_var_conflict (size_t x, size_t y)
496 class stack_var *a = &stack_vars[x];
497 class stack_var *b = &stack_vars[y];
501 a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
503 b->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
504 bitmap_set_bit (a->conflicts, y);
505 bitmap_set_bit (b->conflicts, x);
508 /* Check whether the decls associated with luid's X and Y conflict. */
511 stack_var_conflict_p (size_t x, size_t y)
513 class stack_var *a = &stack_vars[x];
514 class stack_var *b = &stack_vars[y];
517 /* Partitions containing an SSA name result from gimple registers
518 with things like unsupported modes. They are top-level and
519 hence conflict with everything else. */
520 if (TREE_CODE (a->decl) == SSA_NAME || TREE_CODE (b->decl) == SSA_NAME)
523 if (!a->conflicts || !b->conflicts)
525 return bitmap_bit_p (a->conflicts, y);
528 /* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
529 enter its partition number into bitmap DATA. */
532 visit_op (gimple *, tree op, tree, void *data)
534 bitmap active = (bitmap)data;
535 op = get_base_address (op);
538 && DECL_RTL_IF_SET (op) == pc_rtx)
540 size_t *v = decl_to_stack_part->get (op);
542 bitmap_set_bit (active, *v);
547 /* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
548 record conflicts between it and all currently active other partitions
552 visit_conflict (gimple *, tree op, tree, void *data)
554 bitmap active = (bitmap)data;
555 op = get_base_address (op);
558 && DECL_RTL_IF_SET (op) == pc_rtx)
560 size_t *v = decl_to_stack_part->get (op);
561 if (v && bitmap_set_bit (active, *v))
566 gcc_assert (num < stack_vars_num);
567 EXECUTE_IF_SET_IN_BITMAP (active, 0, i, bi)
568 add_stack_var_conflict (num, i);
574 /* Helper routine for add_scope_conflicts, calculating the active partitions
575 at the end of BB, leaving the result in WORK. We're called to generate
576 conflicts when FOR_CONFLICT is true, otherwise we're just tracking
580 add_scope_conflicts_1 (basic_block bb, bitmap work, bool for_conflict)
584 gimple_stmt_iterator gsi;
585 walk_stmt_load_store_addr_fn visit;
588 FOR_EACH_EDGE (e, ei, bb->preds)
589 bitmap_ior_into (work, (bitmap)e->src->aux);
593 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
595 gimple *stmt = gsi_stmt (gsi);
596 walk_stmt_load_store_addr_ops (stmt, work, NULL, NULL, visit);
598 for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
600 gimple *stmt = gsi_stmt (gsi);
602 if (gimple_clobber_p (stmt))
604 tree lhs = gimple_assign_lhs (stmt);
606 /* Nested function lowering might introduce LHSs
607 that are COMPONENT_REFs. */
610 if (DECL_RTL_IF_SET (lhs) == pc_rtx
611 && (v = decl_to_stack_part->get (lhs)))
612 bitmap_clear_bit (work, *v);
614 else if (!is_gimple_debug (stmt))
617 && visit == visit_op)
619 /* If this is the first real instruction in this BB we need
620 to add conflicts for everything live at this point now.
621 Unlike classical liveness for named objects we can't
622 rely on seeing a def/use of the names we're interested in.
623 There might merely be indirect loads/stores. We'd not add any
624 conflicts for such partitions. */
627 EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
629 class stack_var *a = &stack_vars[i];
631 a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
632 bitmap_ior_into (a->conflicts, work);
634 visit = visit_conflict;
636 walk_stmt_load_store_addr_ops (stmt, work, visit, visit, visit);
641 /* Generate stack partition conflicts between all partitions that are
642 simultaneously live. */
645 add_scope_conflicts (void)
649 bitmap work = BITMAP_ALLOC (NULL);
653 /* We approximate the live range of a stack variable by taking the first
654 mention of its name as starting point(s), and by the end-of-scope
655 death clobber added by gimplify as ending point(s) of the range.
656 This overapproximates in the case we for instance moved an address-taken
657 operation upward, without also moving a dereference to it upwards.
658 But it's conservatively correct as a variable never can hold values
659 before its name is mentioned at least once.
661 We then do a mostly classical bitmap liveness algorithm. */
663 FOR_ALL_BB_FN (bb, cfun)
664 bb->aux = BITMAP_ALLOC (&stack_var_bitmap_obstack);
666 rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
667 n_bbs = pre_and_rev_post_order_compute (NULL, rpo, false);
674 for (i = 0; i < n_bbs; i++)
677 bb = BASIC_BLOCK_FOR_FN (cfun, rpo[i]);
678 active = (bitmap)bb->aux;
679 add_scope_conflicts_1 (bb, work, false);
680 if (bitmap_ior_into (active, work))
685 FOR_EACH_BB_FN (bb, cfun)
686 add_scope_conflicts_1 (bb, work, true);
690 FOR_ALL_BB_FN (bb, cfun)
691 BITMAP_FREE (bb->aux);
694 /* A subroutine of partition_stack_vars. A comparison function for qsort,
695 sorting an array of indices by the properties of the object. */
698 stack_var_cmp (const void *a, const void *b)
700 size_t ia = *(const size_t *)a;
701 size_t ib = *(const size_t *)b;
702 unsigned int aligna = stack_vars[ia].alignb;
703 unsigned int alignb = stack_vars[ib].alignb;
704 poly_int64 sizea = stack_vars[ia].size;
705 poly_int64 sizeb = stack_vars[ib].size;
706 tree decla = stack_vars[ia].decl;
707 tree declb = stack_vars[ib].decl;
709 unsigned int uida, uidb;
711 /* Primary compare on "large" alignment. Large comes first. */
712 largea = (aligna * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
713 largeb = (alignb * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
714 if (largea != largeb)
715 return (int)largeb - (int)largea;
717 /* Secondary compare on size, decreasing */
718 int diff = compare_sizes_for_sort (sizeb, sizea);
722 /* Tertiary compare on true alignment, decreasing. */
728 /* Final compare on ID for sort stability, increasing.
729 Two SSA names are compared by their version, SSA names come before
730 non-SSA names, and two normal decls are compared by their DECL_UID. */
731 if (TREE_CODE (decla) == SSA_NAME)
733 if (TREE_CODE (declb) == SSA_NAME)
734 uida = SSA_NAME_VERSION (decla), uidb = SSA_NAME_VERSION (declb);
738 else if (TREE_CODE (declb) == SSA_NAME)
741 uida = DECL_UID (decla), uidb = DECL_UID (declb);
749 struct part_traits : unbounded_int_hashmap_traits <size_t, bitmap> {};
750 typedef hash_map<size_t, bitmap, part_traits> part_hashmap;
752 /* If the points-to solution *PI points to variables that are in a partition
753 together with other variables add all partition members to the pointed-to
757 add_partitioned_vars_to_ptset (struct pt_solution *pt,
758 part_hashmap *decls_to_partitions,
759 hash_set<bitmap> *visited, bitmap temp)
767 /* The pointed-to vars bitmap is shared, it is enough to
769 || visited->add (pt->vars))
774 /* By using a temporary bitmap to store all members of the partitions
775 we have to add we make sure to visit each of the partitions only
777 EXECUTE_IF_SET_IN_BITMAP (pt->vars, 0, i, bi)
779 || !bitmap_bit_p (temp, i))
780 && (part = decls_to_partitions->get (i)))
781 bitmap_ior_into (temp, *part);
782 if (!bitmap_empty_p (temp))
783 bitmap_ior_into (pt->vars, temp);
786 /* Update points-to sets based on partition info, so we can use them on RTL.
787 The bitmaps representing stack partitions will be saved until expand,
788 where partitioned decls used as bases in memory expressions will be
792 update_alias_info_with_stack_vars (void)
794 part_hashmap *decls_to_partitions = NULL;
796 tree var = NULL_TREE;
798 for (i = 0; i < stack_vars_num; i++)
802 struct ptr_info_def *pi;
804 /* Not interested in partitions with single variable. */
805 if (stack_vars[i].representative != i
806 || stack_vars[i].next == EOC)
809 if (!decls_to_partitions)
811 decls_to_partitions = new part_hashmap;
812 cfun->gimple_df->decls_to_pointers = new hash_map<tree, tree>;
815 /* Create an SSA_NAME that points to the partition for use
816 as base during alias-oracle queries on RTL for bases that
817 have been partitioned. */
818 if (var == NULL_TREE)
819 var = create_tmp_var (ptr_type_node);
820 name = make_ssa_name (var);
822 /* Create bitmaps representing partitions. They will be used for
823 points-to sets later, so use GGC alloc. */
824 part = BITMAP_GGC_ALLOC ();
825 for (j = i; j != EOC; j = stack_vars[j].next)
827 tree decl = stack_vars[j].decl;
828 unsigned int uid = DECL_PT_UID (decl);
829 bitmap_set_bit (part, uid);
830 decls_to_partitions->put (uid, part);
831 cfun->gimple_df->decls_to_pointers->put (decl, name);
832 if (TREE_ADDRESSABLE (decl))
833 TREE_ADDRESSABLE (name) = 1;
836 /* Make the SSA name point to all partition members. */
837 pi = get_ptr_info (name);
838 pt_solution_set (&pi->pt, part, false);
841 /* Make all points-to sets that contain one member of a partition
842 contain all members of the partition. */
843 if (decls_to_partitions)
847 hash_set<bitmap> visited;
848 bitmap temp = BITMAP_ALLOC (&stack_var_bitmap_obstack);
850 FOR_EACH_SSA_NAME (i, name, cfun)
852 struct ptr_info_def *pi;
854 if (POINTER_TYPE_P (TREE_TYPE (name))
855 && ((pi = SSA_NAME_PTR_INFO (name)) != NULL))
856 add_partitioned_vars_to_ptset (&pi->pt, decls_to_partitions,
860 add_partitioned_vars_to_ptset (&cfun->gimple_df->escaped,
861 decls_to_partitions, &visited, temp);
863 delete decls_to_partitions;
868 /* A subroutine of partition_stack_vars. The UNION portion of a UNION/FIND
869 partitioning algorithm. Partitions A and B are known to be non-conflicting.
870 Merge them into a single partition A. */
873 union_stack_vars (size_t a, size_t b)
875 class stack_var *vb = &stack_vars[b];
879 gcc_assert (stack_vars[b].next == EOC);
880 /* Add B to A's partition. */
881 stack_vars[b].next = stack_vars[a].next;
882 stack_vars[b].representative = a;
883 stack_vars[a].next = b;
885 /* Make sure A is big enough to hold B. */
886 stack_vars[a].size = upper_bound (stack_vars[a].size, stack_vars[b].size);
888 /* Update the required alignment of partition A to account for B. */
889 if (stack_vars[a].alignb < stack_vars[b].alignb)
890 stack_vars[a].alignb = stack_vars[b].alignb;
892 /* Update the interference graph and merge the conflicts. */
895 EXECUTE_IF_SET_IN_BITMAP (vb->conflicts, 0, u, bi)
896 add_stack_var_conflict (a, stack_vars[u].representative);
897 BITMAP_FREE (vb->conflicts);
901 /* A subroutine of expand_used_vars. Binpack the variables into
902 partitions constrained by the interference graph. The overall
903 algorithm used is as follows:
905 Sort the objects by size in descending order.
910 Look for the largest non-conflicting object B with size <= S.
917 partition_stack_vars (void)
919 size_t si, sj, n = stack_vars_num;
921 stack_vars_sorted = XNEWVEC (size_t, stack_vars_num);
922 for (si = 0; si < n; ++si)
923 stack_vars_sorted[si] = si;
928 qsort (stack_vars_sorted, n, sizeof (size_t), stack_var_cmp);
930 for (si = 0; si < n; ++si)
932 size_t i = stack_vars_sorted[si];
933 unsigned int ialign = stack_vars[i].alignb;
934 poly_int64 isize = stack_vars[i].size;
936 /* Ignore objects that aren't partition representatives. If we
937 see a var that is not a partition representative, it must
938 have been merged earlier. */
939 if (stack_vars[i].representative != i)
942 for (sj = si + 1; sj < n; ++sj)
944 size_t j = stack_vars_sorted[sj];
945 unsigned int jalign = stack_vars[j].alignb;
946 poly_int64 jsize = stack_vars[j].size;
948 /* Ignore objects that aren't partition representatives. */
949 if (stack_vars[j].representative != j)
952 /* Do not mix objects of "small" (supported) alignment
953 and "large" (unsupported) alignment. */
954 if ((ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
955 != (jalign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT))
958 /* For Address Sanitizer do not mix objects with different
959 sizes, as the shorter vars wouldn't be adequately protected.
960 Don't do that for "large" (unsupported) alignment objects,
961 those aren't protected anyway. */
962 if (asan_sanitize_stack_p ()
963 && maybe_ne (isize, jsize)
964 && ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
967 /* Ignore conflicting objects. */
968 if (stack_var_conflict_p (i, j))
971 /* UNION the objects, placing J at OFFSET. */
972 union_stack_vars (i, j);
976 update_alias_info_with_stack_vars ();
979 /* A debugging aid for expand_used_vars. Dump the generated partitions. */
982 dump_stack_var_partition (void)
984 size_t si, i, j, n = stack_vars_num;
986 for (si = 0; si < n; ++si)
988 i = stack_vars_sorted[si];
990 /* Skip variables that aren't partition representatives, for now. */
991 if (stack_vars[i].representative != i)
994 fprintf (dump_file, "Partition %lu: size ", (unsigned long) i);
995 print_dec (stack_vars[i].size, dump_file);
996 fprintf (dump_file, " align %u\n", stack_vars[i].alignb);
998 for (j = i; j != EOC; j = stack_vars[j].next)
1000 fputc ('\t', dump_file);
1001 print_generic_expr (dump_file, stack_vars[j].decl, dump_flags);
1003 fputc ('\n', dump_file);
1007 /* Assign rtl to DECL at BASE + OFFSET. */
1010 expand_one_stack_var_at (tree decl, rtx base, unsigned base_align,
1016 /* If this fails, we've overflowed the stack frame. Error nicely? */
1017 gcc_assert (known_eq (offset, trunc_int_for_mode (offset, Pmode)));
1019 if (hwasan_sanitize_stack_p ())
1020 x = targetm.memtag.add_tag (base, offset,
1021 hwasan_current_frame_tag ());
1023 x = plus_constant (Pmode, base, offset);
1025 x = gen_rtx_MEM (TREE_CODE (decl) == SSA_NAME
1026 ? TYPE_MODE (TREE_TYPE (decl))
1027 : DECL_MODE (decl), x);
1029 /* Set alignment we actually gave this decl if it isn't an SSA name.
1030 If it is we generate stack slots only accidentally so it isn't as
1031 important, we'll simply set the alignment directly on the MEM. */
1033 if (stack_vars_base_reg_p (base))
1034 offset -= frame_phase;
1035 align = known_alignment (offset);
1036 align *= BITS_PER_UNIT;
1037 if (align == 0 || align > base_align)
1040 if (TREE_CODE (decl) != SSA_NAME)
1042 /* One would think that we could assert that we're not decreasing
1043 alignment here, but (at least) the i386 port does exactly this
1044 via the MINIMUM_ALIGNMENT hook. */
1046 SET_DECL_ALIGN (decl, align);
1047 DECL_USER_ALIGN (decl) = 0;
1052 set_mem_align (x, align);
1055 class stack_vars_data
1058 /* Vector of offset pairs, always end of some padding followed
1059 by start of the padding that needs Address Sanitizer protection.
1060 The vector is in reversed, highest offset pairs come first. */
1061 auto_vec<HOST_WIDE_INT> asan_vec;
1063 /* Vector of partition representative decls in between the paddings. */
1064 auto_vec<tree> asan_decl_vec;
1066 /* Base pseudo register for Address Sanitizer protected automatic vars. */
1069 /* Alignment needed for the Address Sanitizer protected automatic vars. */
1070 unsigned int asan_alignb;
1073 /* A subroutine of expand_used_vars. Give each partition representative
1074 a unique location within the stack frame. Update each partition member
1075 with that location. */
1077 expand_stack_vars (bool (*pred) (size_t), class stack_vars_data *data)
1079 size_t si, i, j, n = stack_vars_num;
1080 poly_uint64 large_size = 0, large_alloc = 0;
1081 rtx large_base = NULL;
1082 rtx large_untagged_base = NULL;
1083 unsigned large_align = 0;
1084 bool large_allocation_done = false;
1087 /* Determine if there are any variables requiring "large" alignment.
1088 Since these are dynamically allocated, we only process these if
1089 no predicate involved. */
1090 large_align = stack_vars[stack_vars_sorted[0]].alignb * BITS_PER_UNIT;
1091 if (pred == NULL && large_align > MAX_SUPPORTED_STACK_ALIGNMENT)
1093 /* Find the total size of these variables. */
1094 for (si = 0; si < n; ++si)
1098 i = stack_vars_sorted[si];
1099 alignb = stack_vars[i].alignb;
1101 /* All "large" alignment decls come before all "small" alignment
1102 decls, but "large" alignment decls are not sorted based on
1103 their alignment. Increase large_align to track the largest
1104 required alignment. */
1105 if ((alignb * BITS_PER_UNIT) > large_align)
1106 large_align = alignb * BITS_PER_UNIT;
1108 /* Stop when we get to the first decl with "small" alignment. */
1109 if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1112 /* Skip variables that aren't partition representatives. */
1113 if (stack_vars[i].representative != i)
1116 /* Skip variables that have already had rtl assigned. See also
1117 add_stack_var where we perpetrate this pc_rtx hack. */
1118 decl = stack_vars[i].decl;
1119 if (TREE_CODE (decl) == SSA_NAME
1120 ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
1121 : DECL_RTL (decl) != pc_rtx)
1124 large_size = aligned_upper_bound (large_size, alignb);
1125 large_size += stack_vars[i].size;
1129 for (si = 0; si < n; ++si)
1132 unsigned base_align, alignb;
1133 poly_int64 offset = 0;
1135 i = stack_vars_sorted[si];
1137 /* Skip variables that aren't partition representatives, for now. */
1138 if (stack_vars[i].representative != i)
1141 /* Skip variables that have already had rtl assigned. See also
1142 add_stack_var where we perpetrate this pc_rtx hack. */
1143 decl = stack_vars[i].decl;
1144 if (TREE_CODE (decl) == SSA_NAME
1145 ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
1146 : DECL_RTL (decl) != pc_rtx)
1149 /* Check the predicate to see whether this variable should be
1150 allocated in this pass. */
1151 if (pred && !pred (i))
1154 base = (hwasan_sanitize_stack_p ()
1155 ? hwasan_frame_base ()
1156 : virtual_stack_vars_rtx);
1157 alignb = stack_vars[i].alignb;
1158 if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1160 poly_int64 hwasan_orig_offset;
1161 if (hwasan_sanitize_stack_p ())
1163 /* There must be no tag granule "shared" between different
1164 objects. This means that no HWASAN_TAG_GRANULE_SIZE byte
1165 chunk can have more than one object in it.
1167 We ensure this by forcing the end of the last bit of data to
1168 be aligned to HWASAN_TAG_GRANULE_SIZE bytes here, and setting
1169 the start of each variable to be aligned to
1170 HWASAN_TAG_GRANULE_SIZE bytes in `align_local_variable`.
1172 We can't align just one of the start or end, since there are
1173 untagged things stored on the stack which we do not align to
1174 HWASAN_TAG_GRANULE_SIZE bytes. If we only aligned the start
1175 or the end of tagged objects then untagged objects could end
1176 up sharing the first granule of a tagged object or sharing the
1177 last granule of a tagged object respectively. */
1178 hwasan_orig_offset = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1179 gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
1181 /* ASAN description strings don't yet have a syntax for expressing
1182 polynomial offsets. */
1183 HOST_WIDE_INT prev_offset;
1184 if (asan_sanitize_stack_p ()
1186 && frame_offset.is_constant (&prev_offset)
1187 && stack_vars[i].size.is_constant ())
1189 if (data->asan_vec.is_empty ())
1191 align_frame_offset (ASAN_RED_ZONE_SIZE);
1192 prev_offset = frame_offset.to_constant ();
1194 prev_offset = align_base (prev_offset,
1195 ASAN_MIN_RED_ZONE_SIZE,
1196 !FRAME_GROWS_DOWNWARD);
1197 tree repr_decl = NULL_TREE;
1198 unsigned HOST_WIDE_INT size
1199 = asan_var_and_redzone_size (stack_vars[i].size.to_constant ());
1200 if (data->asan_vec.is_empty ())
1201 size = MAX (size, ASAN_RED_ZONE_SIZE);
1203 unsigned HOST_WIDE_INT alignment = MAX (alignb,
1204 ASAN_MIN_RED_ZONE_SIZE);
1205 offset = alloc_stack_frame_space (size, alignment);
1207 data->asan_vec.safe_push (prev_offset);
1208 /* Allocating a constant amount of space from a constant
1209 starting offset must give a constant result. */
1210 data->asan_vec.safe_push ((offset + stack_vars[i].size)
1212 /* Find best representative of the partition.
1213 Prefer those with DECL_NAME, even better
1214 satisfying asan_protect_stack_decl predicate. */
1215 for (j = i; j != EOC; j = stack_vars[j].next)
1216 if (asan_protect_stack_decl (stack_vars[j].decl)
1217 && DECL_NAME (stack_vars[j].decl))
1219 repr_decl = stack_vars[j].decl;
1222 else if (repr_decl == NULL_TREE
1223 && DECL_P (stack_vars[j].decl)
1224 && DECL_NAME (stack_vars[j].decl))
1225 repr_decl = stack_vars[j].decl;
1226 if (repr_decl == NULL_TREE)
1227 repr_decl = stack_vars[i].decl;
1228 data->asan_decl_vec.safe_push (repr_decl);
1230 /* Make sure a representative is unpoison if another
1231 variable in the partition is handled by
1232 use-after-scope sanitization. */
1233 if (asan_handled_variables != NULL
1234 && !asan_handled_variables->contains (repr_decl))
1236 for (j = i; j != EOC; j = stack_vars[j].next)
1237 if (asan_handled_variables->contains (stack_vars[j].decl))
1240 asan_handled_variables->add (repr_decl);
1243 data->asan_alignb = MAX (data->asan_alignb, alignb);
1244 if (data->asan_base == NULL)
1245 data->asan_base = gen_reg_rtx (Pmode);
1246 base = data->asan_base;
1248 if (!STRICT_ALIGNMENT)
1249 base_align = crtl->max_used_stack_slot_alignment;
1251 base_align = MAX (crtl->max_used_stack_slot_alignment,
1252 GET_MODE_ALIGNMENT (SImode)
1253 << ASAN_SHADOW_SHIFT);
1257 offset = alloc_stack_frame_space (stack_vars[i].size, alignb);
1258 base_align = crtl->max_used_stack_slot_alignment;
1260 if (hwasan_sanitize_stack_p ())
1262 /* Align again since the point of this alignment is to handle
1263 the "end" of the object (i.e. smallest address after the
1264 stack object). For FRAME_GROWS_DOWNWARD that requires
1265 aligning the stack before allocating, but for a frame that
1266 grows upwards that requires aligning the stack after
1269 Use `frame_offset` to record the offset value rather than
1270 `offset` since the `frame_offset` describes the extent
1271 allocated for this particular variable while `offset`
1272 describes the address that this variable starts at. */
1273 align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1274 hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1275 hwasan_orig_offset, frame_offset);
1281 /* Large alignment is only processed in the last pass. */
1285 /* If there were any variables requiring "large" alignment, allocate
1287 if (maybe_ne (large_size, 0U) && ! large_allocation_done)
1290 rtx large_allocsize;
1292 large_allocsize = gen_int_mode (large_size, Pmode);
1293 get_dynamic_stack_size (&large_allocsize, 0, large_align, NULL);
1294 loffset = alloc_stack_frame_space
1295 (rtx_to_poly_int64 (large_allocsize),
1296 PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT);
1297 large_base = get_dynamic_stack_base (loffset, large_align, base);
1298 large_allocation_done = true;
1301 gcc_assert (large_base != NULL);
1302 large_alloc = aligned_upper_bound (large_alloc, alignb);
1303 offset = large_alloc;
1304 large_alloc += stack_vars[i].size;
1305 if (hwasan_sanitize_stack_p ())
1307 /* An object with a large alignment requirement means that the
1308 alignment requirement is greater than the required alignment
1310 if (!large_untagged_base)
1312 = targetm.memtag.untagged_pointer (large_base, NULL_RTX);
1313 /* Ensure the end of the variable is also aligned correctly. */
1314 poly_int64 align_again
1315 = aligned_upper_bound (large_alloc, HWASAN_TAG_GRANULE_SIZE);
1316 /* For large allocations we always allocate a chunk of space
1317 (which is addressed by large_untagged_base/large_base) and
1318 then use positive offsets from that. Hence the farthest
1319 offset is `align_again` and the nearest offset from the base
1321 hwasan_record_stack_var (large_untagged_base, large_base,
1322 offset, align_again);
1326 base_align = large_align;
1329 /* Create rtl for each variable based on their location within the
1331 for (j = i; j != EOC; j = stack_vars[j].next)
1333 expand_one_stack_var_at (stack_vars[j].decl,
1334 base, base_align, offset);
1336 if (hwasan_sanitize_stack_p ())
1337 hwasan_increment_frame_tag ();
1340 gcc_assert (known_eq (large_alloc, large_size));
1343 /* Take into account all sizes of partitions and reset DECL_RTLs. */
1345 account_stack_vars (void)
1347 size_t si, j, i, n = stack_vars_num;
1348 poly_uint64 size = 0;
1350 for (si = 0; si < n; ++si)
1352 i = stack_vars_sorted[si];
1354 /* Skip variables that aren't partition representatives, for now. */
1355 if (stack_vars[i].representative != i)
1358 size += stack_vars[i].size;
1359 for (j = i; j != EOC; j = stack_vars[j].next)
1360 set_rtl (stack_vars[j].decl, NULL);
1365 /* Record the RTL assignment X for the default def of PARM. */
1368 set_parm_rtl (tree parm, rtx x)
1370 gcc_assert (TREE_CODE (parm) == PARM_DECL
1371 || TREE_CODE (parm) == RESULT_DECL);
1373 if (x && !MEM_P (x))
1375 unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (parm),
1376 TYPE_MODE (TREE_TYPE (parm)),
1377 TYPE_ALIGN (TREE_TYPE (parm)));
1379 /* If the variable alignment is very large we'll dynamicaly
1380 allocate it, which means that in-frame portion is just a
1381 pointer. ??? We've got a pseudo for sure here, do we
1382 actually dynamically allocate its spilling area if needed?
1383 ??? Isn't it a problem when Pmode alignment also exceeds
1384 MAX_SUPPORTED_STACK_ALIGNMENT, as can happen on cris and lm32? */
1385 if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1386 align = GET_MODE_ALIGNMENT (Pmode);
1388 record_alignment_for_reg_var (align);
1391 tree ssa = ssa_default_def (cfun, parm);
1393 return set_rtl (parm, x);
1395 int part = var_to_partition (SA.map, ssa);
1396 gcc_assert (part != NO_PARTITION);
1398 bool changed = bitmap_bit_p (SA.partitions_for_parm_default_defs, part);
1399 gcc_assert (changed);
1402 gcc_assert (DECL_RTL (parm) == x);
1405 /* A subroutine of expand_one_var. Called to immediately assign rtl
1406 to a variable to be allocated in the stack frame. */
1409 expand_one_stack_var_1 (tree var)
1413 unsigned byte_align;
1415 if (TREE_CODE (var) == SSA_NAME)
1417 tree type = TREE_TYPE (var);
1418 size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
1421 size = tree_to_poly_uint64 (DECL_SIZE_UNIT (var));
1423 byte_align = align_local_variable (var, true);
1425 /* We handle highly aligned variables in expand_stack_vars. */
1426 gcc_assert (byte_align * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT);
1429 if (hwasan_sanitize_stack_p ())
1431 /* Allocate zero bytes to align the stack. */
1432 poly_int64 hwasan_orig_offset
1433 = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1434 offset = alloc_stack_frame_space (size, byte_align);
1435 align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1436 base = hwasan_frame_base ();
1437 /* Use `frame_offset` to automatically account for machines where the
1438 frame grows upwards.
1440 `offset` will always point to the "start" of the stack object, which
1441 will be the smallest address, for ! FRAME_GROWS_DOWNWARD this is *not*
1442 the "furthest" offset from the base delimiting the current stack
1443 object. `frame_offset` will always delimit the extent that the frame.
1445 hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1446 hwasan_orig_offset, frame_offset);
1450 offset = alloc_stack_frame_space (size, byte_align);
1451 base = virtual_stack_vars_rtx;
1454 expand_one_stack_var_at (var, base,
1455 crtl->max_used_stack_slot_alignment, offset);
1457 if (hwasan_sanitize_stack_p ())
1458 hwasan_increment_frame_tag ();
1461 /* Wrapper for expand_one_stack_var_1 that checks SSA_NAMEs are
1462 already assigned some MEM. */
1465 expand_one_stack_var (tree var)
1467 if (TREE_CODE (var) == SSA_NAME)
1469 int part = var_to_partition (SA.map, var);
1470 if (part != NO_PARTITION)
1472 rtx x = SA.partition_to_pseudo[part];
1474 gcc_assert (MEM_P (x));
1479 return expand_one_stack_var_1 (var);
1482 /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1483 that will reside in a hard register. */
1486 expand_one_hard_reg_var (tree var)
1488 rest_of_decl_compilation (var, 0, 0);
1491 /* Record the alignment requirements of some variable assigned to a
1495 record_alignment_for_reg_var (unsigned int align)
1497 if (SUPPORTS_STACK_ALIGNMENT
1498 && crtl->stack_alignment_estimated < align)
1500 /* stack_alignment_estimated shouldn't change after stack
1501 realign decision made */
1502 gcc_assert (!crtl->stack_realign_processed);
1503 crtl->stack_alignment_estimated = align;
1506 /* stack_alignment_needed > PREFERRED_STACK_BOUNDARY is permitted.
1507 So here we only make sure stack_alignment_needed >= align. */
1508 if (crtl->stack_alignment_needed < align)
1509 crtl->stack_alignment_needed = align;
1510 if (crtl->max_used_stack_slot_alignment < align)
1511 crtl->max_used_stack_slot_alignment = align;
1514 /* Create RTL for an SSA partition. */
1517 expand_one_ssa_partition (tree var)
1519 int part = var_to_partition (SA.map, var);
1520 gcc_assert (part != NO_PARTITION);
1522 if (SA.partition_to_pseudo[part])
1525 unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1526 TYPE_MODE (TREE_TYPE (var)),
1527 TYPE_ALIGN (TREE_TYPE (var)));
1529 /* If the variable alignment is very large we'll dynamicaly allocate
1530 it, which means that in-frame portion is just a pointer. */
1531 if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1532 align = GET_MODE_ALIGNMENT (Pmode);
1534 record_alignment_for_reg_var (align);
1536 if (!use_register_for_decl (var))
1538 if (defer_stack_allocation (var, true))
1539 add_stack_var (var, true);
1541 expand_one_stack_var_1 (var);
1545 machine_mode reg_mode = promote_ssa_mode (var, NULL);
1546 rtx x = gen_reg_rtx (reg_mode);
1550 /* For a promoted variable, X will not be used directly but wrapped in a
1551 SUBREG with SUBREG_PROMOTED_VAR_P set, which means that the RTL land
1552 will assume that its upper bits can be inferred from its lower bits.
1553 Therefore, if X isn't initialized on every path from the entry, then
1554 we must do it manually in order to fulfill the above assumption. */
1555 if (reg_mode != TYPE_MODE (TREE_TYPE (var))
1556 && bitmap_bit_p (SA.partitions_for_undefined_values, part))
1557 emit_move_insn (x, CONST0_RTX (reg_mode));
1560 /* Record the association between the RTL generated for partition PART
1561 and the underlying variable of the SSA_NAME VAR. */
1564 adjust_one_expanded_partition_var (tree var)
1569 tree decl = SSA_NAME_VAR (var);
1571 int part = var_to_partition (SA.map, var);
1572 if (part == NO_PARTITION)
1575 rtx x = SA.partition_to_pseudo[part];
1584 /* Note if the object is a user variable. */
1585 if (decl && !DECL_ARTIFICIAL (decl))
1588 if (POINTER_TYPE_P (decl ? TREE_TYPE (decl) : TREE_TYPE (var)))
1589 mark_reg_pointer (x, get_pointer_alignment (var));
1592 /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1593 that will reside in a pseudo register. */
1596 expand_one_register_var (tree var)
1598 if (TREE_CODE (var) == SSA_NAME)
1600 int part = var_to_partition (SA.map, var);
1601 if (part != NO_PARTITION)
1603 rtx x = SA.partition_to_pseudo[part];
1605 gcc_assert (REG_P (x));
1612 tree type = TREE_TYPE (decl);
1613 machine_mode reg_mode = promote_decl_mode (decl, NULL);
1614 rtx x = gen_reg_rtx (reg_mode);
1618 /* Note if the object is a user variable. */
1619 if (!DECL_ARTIFICIAL (decl))
1622 if (POINTER_TYPE_P (type))
1623 mark_reg_pointer (x, get_pointer_alignment (var));
1626 /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL that
1627 has some associated error, e.g. its type is error-mark. We just need
1628 to pick something that won't crash the rest of the compiler. */
1631 expand_one_error_var (tree var)
1633 machine_mode mode = DECL_MODE (var);
1636 if (mode == BLKmode)
1637 x = gen_rtx_MEM (BLKmode, const0_rtx);
1638 else if (mode == VOIDmode)
1641 x = gen_reg_rtx (mode);
1643 SET_DECL_RTL (var, x);
1646 /* A subroutine of expand_one_var. VAR is a variable that will be
1647 allocated to the local stack frame. Return true if we wish to
1648 add VAR to STACK_VARS so that it will be coalesced with other
1649 variables. Return false to allocate VAR immediately.
1651 This function is used to reduce the number of variables considered
1652 for coalescing, which reduces the size of the quadratic problem. */
1655 defer_stack_allocation (tree var, bool toplevel)
1657 tree size_unit = TREE_CODE (var) == SSA_NAME
1658 ? TYPE_SIZE_UNIT (TREE_TYPE (var))
1659 : DECL_SIZE_UNIT (var);
1662 /* Whether the variable is small enough for immediate allocation not to be
1663 a problem with regard to the frame size. */
1665 = (poly_int_tree_p (size_unit, &size)
1666 && (estimated_poly_value (size)
1667 < param_min_size_for_stack_sharing));
1669 /* If stack protection is enabled, *all* stack variables must be deferred,
1670 so that we can re-order the strings to the top of the frame.
1671 Similarly for Address Sanitizer. */
1672 if (flag_stack_protect || asan_sanitize_stack_p ())
1675 unsigned int align = TREE_CODE (var) == SSA_NAME
1676 ? TYPE_ALIGN (TREE_TYPE (var))
1679 /* We handle "large" alignment via dynamic allocation. We want to handle
1680 this extra complication in only one place, so defer them. */
1681 if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1684 bool ignored = TREE_CODE (var) == SSA_NAME
1685 ? !SSAVAR (var) || DECL_IGNORED_P (SSA_NAME_VAR (var))
1686 : DECL_IGNORED_P (var);
1688 /* When optimization is enabled, DECL_IGNORED_P variables originally scoped
1689 might be detached from their block and appear at toplevel when we reach
1690 here. We want to coalesce them with variables from other blocks when
1691 the immediate contribution to the frame size would be noticeable. */
1692 if (toplevel && optimize > 0 && ignored && !smallish)
1695 /* Variables declared in the outermost scope automatically conflict
1696 with every other variable. The only reason to want to defer them
1697 at all is that, after sorting, we can more efficiently pack
1698 small variables in the stack frame. Continue to defer at -O2. */
1699 if (toplevel && optimize < 2)
1702 /* Without optimization, *most* variables are allocated from the
1703 stack, which makes the quadratic problem large exactly when we
1704 want compilation to proceed as quickly as possible. On the
1705 other hand, we don't want the function's stack frame size to
1706 get completely out of hand. So we avoid adding scalars and
1707 "small" aggregates to the list at all. */
1708 if (optimize == 0 && smallish)
1714 /* A subroutine of expand_used_vars. Expand one variable according to
1715 its flavor. Variables to be placed on the stack are not actually
1716 expanded yet, merely recorded.
1717 When REALLY_EXPAND is false, only add stack values to be allocated.
1718 Return stack usage this variable is supposed to take.
1722 expand_one_var (tree var, bool toplevel, bool really_expand,
1723 bitmap forced_stack_var = NULL)
1725 unsigned int align = BITS_PER_UNIT;
1730 if (TREE_TYPE (var) != error_mark_node && VAR_P (var))
1732 if (is_global_var (var))
1735 /* Because we don't know if VAR will be in register or on stack,
1736 we conservatively assume it will be on stack even if VAR is
1737 eventually put into register after RA pass. For non-automatic
1738 variables, which won't be on stack, we collect alignment of
1739 type and ignore user specified alignment. Similarly for
1740 SSA_NAMEs for which use_register_for_decl returns true. */
1741 if (TREE_STATIC (var)
1742 || DECL_EXTERNAL (var)
1743 || (TREE_CODE (origvar) == SSA_NAME && use_register_for_decl (var)))
1744 align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1745 TYPE_MODE (TREE_TYPE (var)),
1746 TYPE_ALIGN (TREE_TYPE (var)));
1747 else if (DECL_HAS_VALUE_EXPR_P (var)
1748 || (DECL_RTL_SET_P (var) && MEM_P (DECL_RTL (var))))
1749 /* Don't consider debug only variables with DECL_HAS_VALUE_EXPR_P set
1750 or variables which were assigned a stack slot already by
1751 expand_one_stack_var_at - in the latter case DECL_ALIGN has been
1752 changed from the offset chosen to it. */
1753 align = crtl->stack_alignment_estimated;
1755 align = MINIMUM_ALIGNMENT (var, DECL_MODE (var), DECL_ALIGN (var));
1757 /* If the variable alignment is very large we'll dynamicaly allocate
1758 it, which means that in-frame portion is just a pointer. */
1759 if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1760 align = GET_MODE_ALIGNMENT (Pmode);
1763 record_alignment_for_reg_var (align);
1766 if (TREE_CODE (origvar) == SSA_NAME)
1768 gcc_assert (!VAR_P (var)
1769 || (!DECL_EXTERNAL (var)
1770 && !DECL_HAS_VALUE_EXPR_P (var)
1771 && !TREE_STATIC (var)
1772 && TREE_TYPE (var) != error_mark_node
1773 && !DECL_HARD_REGISTER (var)
1776 if (!VAR_P (var) && TREE_CODE (origvar) != SSA_NAME)
1778 else if (DECL_EXTERNAL (var))
1780 else if (DECL_HAS_VALUE_EXPR_P (var))
1782 else if (TREE_STATIC (var))
1784 else if (TREE_CODE (origvar) != SSA_NAME && DECL_RTL_SET_P (var))
1786 else if (TREE_TYPE (var) == error_mark_node)
1789 expand_one_error_var (var);
1791 else if (VAR_P (var) && DECL_HARD_REGISTER (var))
1795 expand_one_hard_reg_var (var);
1796 if (!DECL_HARD_REGISTER (var))
1797 /* Invalid register specification. */
1798 expand_one_error_var (var);
1801 else if (use_register_for_decl (var)
1802 && (!forced_stack_var
1803 || !bitmap_bit_p (forced_stack_var, DECL_UID (var))))
1806 expand_one_register_var (origvar);
1808 else if (!poly_int_tree_p (DECL_SIZE_UNIT (var), &size)
1809 || !valid_constant_size_p (DECL_SIZE_UNIT (var)))
1811 /* Reject variables which cover more than half of the address-space. */
1814 if (DECL_NONLOCAL_FRAME (var))
1815 error_at (DECL_SOURCE_LOCATION (current_function_decl),
1816 "total size of local objects is too large");
1818 error_at (DECL_SOURCE_LOCATION (var),
1819 "size of variable %q+D is too large", var);
1820 expand_one_error_var (var);
1823 else if (defer_stack_allocation (var, toplevel))
1824 add_stack_var (origvar, really_expand);
1829 if (lookup_attribute ("naked",
1830 DECL_ATTRIBUTES (current_function_decl)))
1831 error ("cannot allocate stack for variable %q+D, naked function",
1834 expand_one_stack_var (origvar);
1841 /* A subroutine of expand_used_vars. Walk down through the BLOCK tree
1842 expanding variables. Those variables that can be put into registers
1843 are allocated pseudos; those that can't are put on the stack.
1845 TOPLEVEL is true if this is the outermost BLOCK. */
1848 expand_used_vars_for_block (tree block, bool toplevel, bitmap forced_stack_vars)
1852 /* Expand all variables at this level. */
1853 for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
1855 && ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
1856 || !DECL_NONSHAREABLE (t)))
1857 expand_one_var (t, toplevel, true, forced_stack_vars);
1859 /* Expand all variables at containing levels. */
1860 for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
1861 expand_used_vars_for_block (t, false, forced_stack_vars);
1864 /* A subroutine of expand_used_vars. Walk down through the BLOCK tree
1865 and clear TREE_USED on all local variables. */
1868 clear_tree_used (tree block)
1872 for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
1873 /* if (!TREE_STATIC (t) && !DECL_EXTERNAL (t)) */
1874 if ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
1875 || !DECL_NONSHAREABLE (t))
1878 for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
1879 clear_tree_used (t);
1882 /* Examine TYPE and determine a bit mask of the following features. */
1884 #define SPCT_HAS_LARGE_CHAR_ARRAY 1
1885 #define SPCT_HAS_SMALL_CHAR_ARRAY 2
1886 #define SPCT_HAS_ARRAY 4
1887 #define SPCT_HAS_AGGREGATE 8
1890 stack_protect_classify_type (tree type)
1892 unsigned int ret = 0;
1895 switch (TREE_CODE (type))
1898 t = TYPE_MAIN_VARIANT (TREE_TYPE (type));
1899 if (t == char_type_node
1900 || t == signed_char_type_node
1901 || t == unsigned_char_type_node)
1903 unsigned HOST_WIDE_INT max = param_ssp_buffer_size;
1904 unsigned HOST_WIDE_INT len;
1906 if (!TYPE_SIZE_UNIT (type)
1907 || !tree_fits_uhwi_p (TYPE_SIZE_UNIT (type)))
1910 len = tree_to_uhwi (TYPE_SIZE_UNIT (type));
1913 ret = SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_ARRAY;
1915 ret = SPCT_HAS_LARGE_CHAR_ARRAY | SPCT_HAS_ARRAY;
1918 ret = SPCT_HAS_ARRAY;
1922 case QUAL_UNION_TYPE:
1924 ret = SPCT_HAS_AGGREGATE;
1925 for (t = TYPE_FIELDS (type); t ; t = TREE_CHAIN (t))
1926 if (TREE_CODE (t) == FIELD_DECL)
1927 ret |= stack_protect_classify_type (TREE_TYPE (t));
1937 /* Return nonzero if DECL should be segregated into the "vulnerable" upper
1938 part of the local stack frame. Remember if we ever return nonzero for
1939 any variable in this function. The return value is the phase number in
1940 which the variable should be allocated. */
1943 stack_protect_decl_phase (tree decl)
1945 unsigned int bits = stack_protect_classify_type (TREE_TYPE (decl));
1948 if (bits & SPCT_HAS_SMALL_CHAR_ARRAY)
1949 has_short_buffer = true;
1951 tree attribs = DECL_ATTRIBUTES (current_function_decl);
1952 if (!lookup_attribute ("no_stack_protector", attribs)
1953 && (flag_stack_protect == SPCT_FLAG_ALL
1954 || flag_stack_protect == SPCT_FLAG_STRONG
1955 || (flag_stack_protect == SPCT_FLAG_EXPLICIT
1956 && lookup_attribute ("stack_protect", attribs))))
1958 if ((bits & (SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_LARGE_CHAR_ARRAY))
1959 && !(bits & SPCT_HAS_AGGREGATE))
1961 else if (bits & SPCT_HAS_ARRAY)
1965 ret = (bits & SPCT_HAS_LARGE_CHAR_ARRAY) != 0;
1968 has_protected_decls = true;
1973 /* Two helper routines that check for phase 1 and phase 2. These are used
1974 as callbacks for expand_stack_vars. */
1977 stack_protect_decl_phase_1 (size_t i)
1979 return stack_protect_decl_phase (stack_vars[i].decl) == 1;
1983 stack_protect_decl_phase_2 (size_t i)
1985 return stack_protect_decl_phase (stack_vars[i].decl) == 2;
1988 /* And helper function that checks for asan phase (with stack protector
1989 it is phase 3). This is used as callback for expand_stack_vars.
1990 Returns true if any of the vars in the partition need to be protected. */
1993 asan_decl_phase_3 (size_t i)
1997 if (asan_protect_stack_decl (stack_vars[i].decl))
1999 i = stack_vars[i].next;
2004 /* Ensure that variables in different stack protection phases conflict
2005 so that they are not merged and share the same stack slot.
2006 Return true if there are any address taken variables. */
2009 add_stack_protection_conflicts (void)
2011 size_t i, j, n = stack_vars_num;
2012 unsigned char *phase;
2015 phase = XNEWVEC (unsigned char, n);
2016 for (i = 0; i < n; ++i)
2018 phase[i] = stack_protect_decl_phase (stack_vars[i].decl);
2019 if (TREE_ADDRESSABLE (stack_vars[i].decl))
2023 for (i = 0; i < n; ++i)
2025 unsigned char ph_i = phase[i];
2026 for (j = i + 1; j < n; ++j)
2027 if (ph_i != phase[j])
2028 add_stack_var_conflict (i, j);
2035 /* Create a decl for the guard at the top of the stack frame. */
2038 create_stack_guard (void)
2040 tree guard = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
2041 VAR_DECL, NULL, ptr_type_node);
2042 TREE_THIS_VOLATILE (guard) = 1;
2043 TREE_USED (guard) = 1;
2044 expand_one_stack_var (guard);
2045 crtl->stack_protect_guard = guard;
2048 /* Prepare for expanding variables. */
2050 init_vars_expansion (void)
2052 /* Conflict bitmaps, and a few related temporary bitmaps, go here. */
2053 bitmap_obstack_initialize (&stack_var_bitmap_obstack);
2055 /* A map from decl to stack partition. */
2056 decl_to_stack_part = new hash_map<tree, size_t>;
2058 /* Initialize local stack smashing state. */
2059 has_protected_decls = false;
2060 has_short_buffer = false;
2061 if (hwasan_sanitize_stack_p ())
2062 hwasan_record_frame_init ();
2065 /* Free up stack variable graph data. */
2067 fini_vars_expansion (void)
2069 bitmap_obstack_release (&stack_var_bitmap_obstack);
2071 XDELETEVEC (stack_vars);
2072 if (stack_vars_sorted)
2073 XDELETEVEC (stack_vars_sorted);
2075 stack_vars_sorted = NULL;
2076 stack_vars_alloc = stack_vars_num = 0;
2077 delete decl_to_stack_part;
2078 decl_to_stack_part = NULL;
2081 /* Make a fair guess for the size of the stack frame of the function
2082 in NODE. This doesn't have to be exact, the result is only used in
2083 the inline heuristics. So we don't want to run the full stack var
2084 packing algorithm (which is quadratic in the number of stack vars).
2085 Instead, we calculate the total size of all stack vars. This turns
2086 out to be a pretty fair estimate -- packing of stack vars doesn't
2087 happen very often. */
2090 estimated_stack_frame_size (struct cgraph_node *node)
2092 poly_int64 size = 0;
2095 struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
2099 init_vars_expansion ();
2101 FOR_EACH_LOCAL_DECL (fn, i, var)
2102 if (auto_var_in_fn_p (var, fn->decl))
2103 size += expand_one_var (var, true, false);
2105 if (stack_vars_num > 0)
2107 /* Fake sorting the stack vars for account_stack_vars (). */
2108 stack_vars_sorted = XNEWVEC (size_t, stack_vars_num);
2109 for (i = 0; i < stack_vars_num; ++i)
2110 stack_vars_sorted[i] = i;
2111 size += account_stack_vars ();
2114 fini_vars_expansion ();
2116 return estimated_poly_value (size);
2119 /* Check if the current function has calls that use a return slot. */
2122 stack_protect_return_slot_p ()
2126 FOR_ALL_BB_FN (bb, cfun)
2127 for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2128 !gsi_end_p (gsi); gsi_next (&gsi))
2130 gimple *stmt = gsi_stmt (gsi);
2131 /* This assumes that calls to internal-only functions never
2132 use a return slot. */
2133 if (is_gimple_call (stmt)
2134 && !gimple_call_internal_p (stmt)
2135 && aggregate_value_p (TREE_TYPE (gimple_call_fntype (stmt)),
2136 gimple_call_fndecl (stmt)))
2142 /* Expand all variables used in the function. */
2145 expand_used_vars (bitmap forced_stack_vars)
2147 tree var, outer_block = DECL_INITIAL (current_function_decl);
2148 auto_vec<tree> maybe_local_decls;
2149 rtx_insn *var_end_seq = NULL;
2152 bool gen_stack_protect_signal = false;
2154 /* Compute the phase of the stack frame for this function. */
2156 int align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2157 int off = targetm.starting_frame_offset () % align;
2158 frame_phase = off ? align - off : 0;
2161 /* Set TREE_USED on all variables in the local_decls. */
2162 FOR_EACH_LOCAL_DECL (cfun, i, var)
2163 TREE_USED (var) = 1;
2164 /* Clear TREE_USED on all variables associated with a block scope. */
2165 clear_tree_used (DECL_INITIAL (current_function_decl));
2167 init_vars_expansion ();
2169 if (targetm.use_pseudo_pic_reg ())
2170 pic_offset_table_rtx = gen_reg_rtx (Pmode);
2172 for (i = 0; i < SA.map->num_partitions; i++)
2174 if (bitmap_bit_p (SA.partitions_for_parm_default_defs, i))
2177 tree var = partition_to_var (SA.map, i);
2179 gcc_assert (!virtual_operand_p (var));
2181 expand_one_ssa_partition (var);
2184 if (flag_stack_protect == SPCT_FLAG_STRONG)
2185 gen_stack_protect_signal = stack_protect_return_slot_p ();
2187 /* At this point all variables on the local_decls with TREE_USED
2188 set are not associated with any block scope. Lay them out. */
2190 len = vec_safe_length (cfun->local_decls);
2191 FOR_EACH_LOCAL_DECL (cfun, i, var)
2193 bool expand_now = false;
2195 /* Expanded above already. */
2196 if (is_gimple_reg (var))
2198 TREE_USED (var) = 0;
2201 /* We didn't set a block for static or extern because it's hard
2202 to tell the difference between a global variable (re)declared
2203 in a local scope, and one that's really declared there to
2204 begin with. And it doesn't really matter much, since we're
2205 not giving them stack space. Expand them now. */
2206 else if (TREE_STATIC (var) || DECL_EXTERNAL (var))
2209 /* Expand variables not associated with any block now. Those created by
2210 the optimizers could be live anywhere in the function. Those that
2211 could possibly have been scoped originally and detached from their
2212 block will have their allocation deferred so we coalesce them with
2213 others when optimization is enabled. */
2214 else if (TREE_USED (var))
2217 /* Finally, mark all variables on the list as used. We'll use
2218 this in a moment when we expand those associated with scopes. */
2219 TREE_USED (var) = 1;
2222 expand_one_var (var, true, true, forced_stack_vars);
2225 if (DECL_ARTIFICIAL (var) && !DECL_IGNORED_P (var))
2227 rtx rtl = DECL_RTL_IF_SET (var);
2229 /* Keep artificial non-ignored vars in cfun->local_decls
2230 chain until instantiate_decls. */
2231 if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2232 add_local_decl (cfun, var);
2233 else if (rtl == NULL_RTX)
2234 /* If rtl isn't set yet, which can happen e.g. with
2235 -fstack-protector, retry before returning from this
2237 maybe_local_decls.safe_push (var);
2241 /* We duplicated some of the decls in CFUN->LOCAL_DECLS.
2243 +-----------------+-----------------+
2244 | ...processed... | ...duplicates...|
2245 +-----------------+-----------------+
2247 +-- LEN points here.
2249 We just want the duplicates, as those are the artificial
2250 non-ignored vars that we want to keep until instantiate_decls.
2251 Move them down and truncate the array. */
2252 if (!vec_safe_is_empty (cfun->local_decls))
2253 cfun->local_decls->block_remove (0, len);
2255 /* At this point, all variables within the block tree with TREE_USED
2256 set are actually used by the optimized function. Lay them out. */
2257 expand_used_vars_for_block (outer_block, true, forced_stack_vars);
2259 tree attribs = DECL_ATTRIBUTES (current_function_decl);
2260 if (stack_vars_num > 0)
2262 bool has_addressable_vars = false;
2264 add_scope_conflicts ();
2266 /* If stack protection is enabled, we don't share space between
2267 vulnerable data and non-vulnerable data. */
2268 if (flag_stack_protect != 0
2269 && !lookup_attribute ("no_stack_protector", attribs)
2270 && (flag_stack_protect != SPCT_FLAG_EXPLICIT
2271 || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2272 && lookup_attribute ("stack_protect", attribs))))
2273 has_addressable_vars = add_stack_protection_conflicts ();
2275 if (flag_stack_protect == SPCT_FLAG_STRONG && has_addressable_vars)
2276 gen_stack_protect_signal = true;
2278 /* Now that we have collected all stack variables, and have computed a
2279 minimal interference graph, attempt to save some stack space. */
2280 partition_stack_vars ();
2282 dump_stack_var_partition ();
2286 if (!lookup_attribute ("no_stack_protector", attribs))
2287 switch (flag_stack_protect)
2290 create_stack_guard ();
2293 case SPCT_FLAG_STRONG:
2294 if (gen_stack_protect_signal
2295 || cfun->calls_alloca
2296 || has_protected_decls
2297 || lookup_attribute ("stack_protect", attribs))
2298 create_stack_guard ();
2301 case SPCT_FLAG_DEFAULT:
2302 if (cfun->calls_alloca
2303 || has_protected_decls
2304 || lookup_attribute ("stack_protect", attribs))
2305 create_stack_guard ();
2308 case SPCT_FLAG_EXPLICIT:
2309 if (lookup_attribute ("stack_protect", attribs))
2310 create_stack_guard ();
2317 /* Assign rtl to each variable based on these partitions. */
2318 if (stack_vars_num > 0)
2320 class stack_vars_data data;
2322 data.asan_base = NULL_RTX;
2323 data.asan_alignb = 0;
2325 /* Reorder decls to be protected by iterating over the variables
2326 array multiple times, and allocating out of each phase in turn. */
2327 /* ??? We could probably integrate this into the qsort we did
2328 earlier, such that we naturally see these variables first,
2329 and thus naturally allocate things in the right order. */
2330 if (has_protected_decls)
2332 /* Phase 1 contains only character arrays. */
2333 expand_stack_vars (stack_protect_decl_phase_1, &data);
2335 /* Phase 2 contains other kinds of arrays. */
2336 if (!lookup_attribute ("no_stack_protector", attribs)
2337 && (flag_stack_protect == SPCT_FLAG_ALL
2338 || flag_stack_protect == SPCT_FLAG_STRONG
2339 || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2340 && lookup_attribute ("stack_protect", attribs))))
2341 expand_stack_vars (stack_protect_decl_phase_2, &data);
2344 if (asan_sanitize_stack_p ())
2345 /* Phase 3, any partitions that need asan protection
2346 in addition to phase 1 and 2. */
2347 expand_stack_vars (asan_decl_phase_3, &data);
2349 /* ASAN description strings don't yet have a syntax for expressing
2350 polynomial offsets. */
2351 HOST_WIDE_INT prev_offset;
2352 if (!data.asan_vec.is_empty ()
2353 && frame_offset.is_constant (&prev_offset))
2355 HOST_WIDE_INT offset, sz, redzonesz;
2356 redzonesz = ASAN_RED_ZONE_SIZE;
2357 sz = data.asan_vec[0] - prev_offset;
2358 if (data.asan_alignb > ASAN_RED_ZONE_SIZE
2359 && data.asan_alignb <= 4096
2360 && sz + ASAN_RED_ZONE_SIZE >= (int) data.asan_alignb)
2361 redzonesz = ((sz + ASAN_RED_ZONE_SIZE + data.asan_alignb - 1)
2362 & ~(data.asan_alignb - HOST_WIDE_INT_1)) - sz;
2363 /* Allocating a constant amount of space from a constant
2364 starting offset must give a constant result. */
2365 offset = (alloc_stack_frame_space (redzonesz, ASAN_RED_ZONE_SIZE)
2367 data.asan_vec.safe_push (prev_offset);
2368 data.asan_vec.safe_push (offset);
2369 /* Leave space for alignment if STRICT_ALIGNMENT. */
2370 if (STRICT_ALIGNMENT)
2371 alloc_stack_frame_space ((GET_MODE_ALIGNMENT (SImode)
2372 << ASAN_SHADOW_SHIFT)
2373 / BITS_PER_UNIT, 1);
2376 = asan_emit_stack_protection (virtual_stack_vars_rtx,
2379 data.asan_vec.address (),
2380 data.asan_decl_vec.address (),
2381 data.asan_vec.length ());
2384 expand_stack_vars (NULL, &data);
2387 if (hwasan_sanitize_stack_p ())
2388 hwasan_emit_prologue ();
2389 if (asan_sanitize_allocas_p () && cfun->calls_alloca)
2390 var_end_seq = asan_emit_allocas_unpoison (virtual_stack_dynamic_rtx,
2391 virtual_stack_vars_rtx,
2393 else if (hwasan_sanitize_allocas_p () && cfun->calls_alloca)
2394 /* When using out-of-line instrumentation we only want to emit one function
2395 call for clearing the tags in a region of shadow stack. When there are
2396 alloca calls in this frame we want to emit a call using the
2397 virtual_stack_dynamic_rtx, but when not we use the hwasan_frame_extent
2398 rtx we created in expand_stack_vars. */
2399 var_end_seq = hwasan_emit_untag_frame (virtual_stack_dynamic_rtx,
2400 virtual_stack_vars_rtx);
2401 else if (hwasan_sanitize_stack_p ())
2402 /* If no variables were stored on the stack, `hwasan_get_frame_extent`
2403 will return NULL_RTX and hence `hwasan_emit_untag_frame` will return
2404 NULL (i.e. an empty sequence). */
2405 var_end_seq = hwasan_emit_untag_frame (hwasan_get_frame_extent (),
2406 virtual_stack_vars_rtx);
2408 fini_vars_expansion ();
2410 /* If there were any artificial non-ignored vars without rtl
2411 found earlier, see if deferred stack allocation hasn't assigned
2413 FOR_EACH_VEC_ELT_REVERSE (maybe_local_decls, i, var)
2415 rtx rtl = DECL_RTL_IF_SET (var);
2417 /* Keep artificial non-ignored vars in cfun->local_decls
2418 chain until instantiate_decls. */
2419 if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2420 add_local_decl (cfun, var);
2423 /* If the target requires that FRAME_OFFSET be aligned, do it. */
2424 if (STACK_ALIGNMENT_NEEDED)
2426 HOST_WIDE_INT align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2427 if (FRAME_GROWS_DOWNWARD)
2428 frame_offset = aligned_lower_bound (frame_offset, align);
2430 frame_offset = aligned_upper_bound (frame_offset, align);
2437 /* If we need to produce a detailed dump, print the tree representation
2438 for STMT to the dump file. SINCE is the last RTX after which the RTL
2439 generated for STMT should have been appended. */
2442 maybe_dump_rtl_for_gimple_stmt (gimple *stmt, rtx_insn *since)
2444 if (dump_file && (dump_flags & TDF_DETAILS))
2446 fprintf (dump_file, "\n;; ");
2447 print_gimple_stmt (dump_file, stmt, 0,
2448 TDF_SLIM | (dump_flags & TDF_LINENO));
2449 fprintf (dump_file, "\n");
2451 print_rtl (dump_file, since ? NEXT_INSN (since) : since);
2455 /* Maps the blocks that do not contain tree labels to rtx labels. */
2457 static hash_map<basic_block, rtx_code_label *> *lab_rtx_for_bb;
2459 /* Returns the label_rtx expression for a label starting basic block BB. */
2461 static rtx_code_label *
2462 label_rtx_for_bb (basic_block bb ATTRIBUTE_UNUSED)
2464 if (bb->flags & BB_RTL)
2465 return block_label (bb);
2467 rtx_code_label **elt = lab_rtx_for_bb->get (bb);
2471 /* Find the tree label if it is present. */
2472 gimple_stmt_iterator gsi = gsi_start_bb (bb);
2474 if (!gsi_end_p (gsi)
2475 && (lab_stmt = dyn_cast <glabel *> (gsi_stmt (gsi)))
2476 && !DECL_NONLOCAL (gimple_label_label (lab_stmt)))
2477 return jump_target_rtx (gimple_label_label (lab_stmt));
2479 rtx_code_label *l = gen_label_rtx ();
2480 lab_rtx_for_bb->put (bb, l);
2485 /* A subroutine of expand_gimple_cond. Given E, a fallthrough edge
2486 of a basic block where we just expanded the conditional at the end,
2487 possibly clean up the CFG and instruction sequence. LAST is the
2488 last instruction before the just emitted jump sequence. */
2491 maybe_cleanup_end_of_block (edge e, rtx_insn *last)
2493 /* Special case: when jumpif decides that the condition is
2494 trivial it emits an unconditional jump (and the necessary
2495 barrier). But we still have two edges, the fallthru one is
2496 wrong. purge_dead_edges would clean this up later. Unfortunately
2497 we have to insert insns (and split edges) before
2498 find_many_sub_basic_blocks and hence before purge_dead_edges.
2499 But splitting edges might create new blocks which depend on the
2500 fact that if there are two edges there's no barrier. So the
2501 barrier would get lost and verify_flow_info would ICE. Instead
2502 of auditing all edge splitters to care for the barrier (which
2503 normally isn't there in a cleaned CFG), fix it here. */
2504 if (BARRIER_P (get_last_insn ()))
2508 /* Now, we have a single successor block, if we have insns to
2509 insert on the remaining edge we potentially will insert
2510 it at the end of this block (if the dest block isn't feasible)
2511 in order to avoid splitting the edge. This insertion will take
2512 place in front of the last jump. But we might have emitted
2513 multiple jumps (conditional and one unconditional) to the
2514 same destination. Inserting in front of the last one then
2515 is a problem. See PR 40021. We fix this by deleting all
2516 jumps except the last unconditional one. */
2517 insn = PREV_INSN (get_last_insn ());
2518 /* Make sure we have an unconditional jump. Otherwise we're
2520 gcc_assert (JUMP_P (insn) && !any_condjump_p (insn));
2521 for (insn = PREV_INSN (insn); insn != last;)
2523 insn = PREV_INSN (insn);
2524 if (JUMP_P (NEXT_INSN (insn)))
2526 if (!any_condjump_p (NEXT_INSN (insn)))
2528 gcc_assert (BARRIER_P (NEXT_INSN (NEXT_INSN (insn))));
2529 delete_insn (NEXT_INSN (NEXT_INSN (insn)));
2531 delete_insn (NEXT_INSN (insn));
2537 /* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_COND.
2538 Returns a new basic block if we've terminated the current basic
2539 block and created a new one. */
2542 expand_gimple_cond (basic_block bb, gcond *stmt)
2544 basic_block new_bb, dest;
2547 rtx_insn *last2, *last;
2548 enum tree_code code;
2551 code = gimple_cond_code (stmt);
2552 op0 = gimple_cond_lhs (stmt);
2553 op1 = gimple_cond_rhs (stmt);
2554 /* We're sometimes presented with such code:
2558 This would expand to two comparisons which then later might
2559 be cleaned up by combine. But some pattern matchers like if-conversion
2560 work better when there's only one compare, so make up for this
2561 here as special exception if TER would have made the same change. */
2563 && TREE_CODE (op0) == SSA_NAME
2564 && TREE_CODE (TREE_TYPE (op0)) == BOOLEAN_TYPE
2565 && TREE_CODE (op1) == INTEGER_CST
2566 && ((gimple_cond_code (stmt) == NE_EXPR
2567 && integer_zerop (op1))
2568 || (gimple_cond_code (stmt) == EQ_EXPR
2569 && integer_onep (op1)))
2570 && bitmap_bit_p (SA.values, SSA_NAME_VERSION (op0)))
2572 gimple *second = SSA_NAME_DEF_STMT (op0);
2573 if (gimple_code (second) == GIMPLE_ASSIGN)
2575 enum tree_code code2 = gimple_assign_rhs_code (second);
2576 if (TREE_CODE_CLASS (code2) == tcc_comparison)
2579 op0 = gimple_assign_rhs1 (second);
2580 op1 = gimple_assign_rhs2 (second);
2582 /* If jumps are cheap and the target does not support conditional
2583 compare, turn some more codes into jumpy sequences. */
2584 else if (BRANCH_COST (optimize_insn_for_speed_p (), false) < 4
2585 && targetm.gen_ccmp_first == NULL)
2587 if ((code2 == BIT_AND_EXPR
2588 && TYPE_PRECISION (TREE_TYPE (op0)) == 1
2589 && TREE_CODE (gimple_assign_rhs2 (second)) != INTEGER_CST)
2590 || code2 == TRUTH_AND_EXPR)
2592 code = TRUTH_ANDIF_EXPR;
2593 op0 = gimple_assign_rhs1 (second);
2594 op1 = gimple_assign_rhs2 (second);
2596 else if (code2 == BIT_IOR_EXPR || code2 == TRUTH_OR_EXPR)
2598 code = TRUTH_ORIF_EXPR;
2599 op0 = gimple_assign_rhs1 (second);
2600 op1 = gimple_assign_rhs2 (second);
2606 /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
2607 into (x - C2) * C3 < C4. */
2608 if ((code == EQ_EXPR || code == NE_EXPR)
2609 && TREE_CODE (op0) == SSA_NAME
2610 && TREE_CODE (op1) == INTEGER_CST)
2611 code = maybe_optimize_mod_cmp (code, &op0, &op1);
2613 /* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
2614 if (!TYPE_UNSIGNED (TREE_TYPE (op0))
2615 && (code == LT_EXPR || code == LE_EXPR
2616 || code == GT_EXPR || code == GE_EXPR)
2617 && integer_zerop (op1)
2618 && TREE_CODE (op0) == SSA_NAME)
2619 maybe_optimize_sub_cmp_0 (code, &op0, &op1);
2621 last2 = last = get_last_insn ();
2623 extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
2624 set_curr_insn_location (gimple_location (stmt));
2626 /* These flags have no purpose in RTL land. */
2627 true_edge->flags &= ~EDGE_TRUE_VALUE;
2628 false_edge->flags &= ~EDGE_FALSE_VALUE;
2630 /* We can either have a pure conditional jump with one fallthru edge or
2631 two-way jump that needs to be decomposed into two basic blocks. */
2632 if (false_edge->dest == bb->next_bb)
2634 jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
2635 true_edge->probability);
2636 maybe_dump_rtl_for_gimple_stmt (stmt, last);
2637 if (true_edge->goto_locus != UNKNOWN_LOCATION)
2638 set_curr_insn_location (true_edge->goto_locus);
2639 false_edge->flags |= EDGE_FALLTHRU;
2640 maybe_cleanup_end_of_block (false_edge, last);
2643 if (true_edge->dest == bb->next_bb)
2645 jumpifnot_1 (code, op0, op1, label_rtx_for_bb (false_edge->dest),
2646 false_edge->probability);
2647 maybe_dump_rtl_for_gimple_stmt (stmt, last);
2648 if (false_edge->goto_locus != UNKNOWN_LOCATION)
2649 set_curr_insn_location (false_edge->goto_locus);
2650 true_edge->flags |= EDGE_FALLTHRU;
2651 maybe_cleanup_end_of_block (true_edge, last);
2655 jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
2656 true_edge->probability);
2657 last = get_last_insn ();
2658 if (false_edge->goto_locus != UNKNOWN_LOCATION)
2659 set_curr_insn_location (false_edge->goto_locus);
2660 emit_jump (label_rtx_for_bb (false_edge->dest));
2663 if (BARRIER_P (BB_END (bb)))
2664 BB_END (bb) = PREV_INSN (BB_END (bb));
2665 update_bb_for_insn (bb);
2667 new_bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
2668 dest = false_edge->dest;
2669 redirect_edge_succ (false_edge, new_bb);
2670 false_edge->flags |= EDGE_FALLTHRU;
2671 new_bb->count = false_edge->count ();
2672 loop_p loop = find_common_loop (bb->loop_father, dest->loop_father);
2673 add_bb_to_loop (new_bb, loop);
2674 if (loop->latch == bb
2675 && loop->header == dest)
2676 loop->latch = new_bb;
2677 make_single_succ_edge (new_bb, dest, 0);
2678 if (BARRIER_P (BB_END (new_bb)))
2679 BB_END (new_bb) = PREV_INSN (BB_END (new_bb));
2680 update_bb_for_insn (new_bb);
2682 maybe_dump_rtl_for_gimple_stmt (stmt, last2);
2684 if (true_edge->goto_locus != UNKNOWN_LOCATION)
2686 set_curr_insn_location (true_edge->goto_locus);
2687 true_edge->goto_locus = curr_insn_location ();
2693 /* Mark all calls that can have a transaction restart. */
2696 mark_transaction_restart_calls (gimple *stmt)
2698 struct tm_restart_node dummy;
2699 tm_restart_node **slot;
2701 if (!cfun->gimple_df->tm_restart)
2705 slot = cfun->gimple_df->tm_restart->find_slot (&dummy, NO_INSERT);
2708 struct tm_restart_node *n = *slot;
2709 tree list = n->label_or_list;
2712 for (insn = next_real_insn (get_last_insn ());
2714 insn = next_real_insn (insn))
2717 if (TREE_CODE (list) == LABEL_DECL)
2718 add_reg_note (insn, REG_TM, label_rtx (list));
2720 for (; list ; list = TREE_CHAIN (list))
2721 add_reg_note (insn, REG_TM, label_rtx (TREE_VALUE (list)));
2725 /* A subroutine of expand_gimple_stmt_1, expanding one GIMPLE_CALL
2729 expand_call_stmt (gcall *stmt)
2731 tree exp, decl, lhs;
2735 if (gimple_call_internal_p (stmt))
2737 expand_internal_call (stmt);
2741 /* If this is a call to a built-in function and it has no effect other
2742 than setting the lhs, try to implement it using an internal function
2744 decl = gimple_call_fndecl (stmt);
2745 if (gimple_call_lhs (stmt)
2746 && !gimple_has_side_effects (stmt)
2747 && (optimize || (decl && called_as_built_in (decl))))
2749 internal_fn ifn = replacement_internal_fn (stmt);
2750 if (ifn != IFN_LAST)
2752 expand_internal_call (ifn, stmt);
2757 exp = build_vl_exp (CALL_EXPR, gimple_call_num_args (stmt) + 3);
2759 CALL_EXPR_FN (exp) = gimple_call_fn (stmt);
2760 builtin_p = decl && fndecl_built_in_p (decl);
2762 /* If this is not a builtin function, the function type through which the
2763 call is made may be different from the type of the function. */
2766 = fold_convert (build_pointer_type (gimple_call_fntype (stmt)),
2767 CALL_EXPR_FN (exp));
2769 TREE_TYPE (exp) = gimple_call_return_type (stmt);
2770 CALL_EXPR_STATIC_CHAIN (exp) = gimple_call_chain (stmt);
2772 for (i = 0; i < gimple_call_num_args (stmt); i++)
2774 tree arg = gimple_call_arg (stmt, i);
2776 /* TER addresses into arguments of builtin functions so we have a
2777 chance to infer more correct alignment information. See PR39954. */
2779 && TREE_CODE (arg) == SSA_NAME
2780 && (def = get_gimple_for_ssa_name (arg))
2781 && gimple_assign_rhs_code (def) == ADDR_EXPR)
2782 arg = gimple_assign_rhs1 (def);
2783 CALL_EXPR_ARG (exp, i) = arg;
2786 if (gimple_has_side_effects (stmt)
2787 /* ??? Downstream in expand_expr_real_1 we assume that expressions
2788 w/o side-effects do not throw so work around this here. */
2789 || stmt_could_throw_p (cfun, stmt))
2790 TREE_SIDE_EFFECTS (exp) = 1;
2792 if (gimple_call_nothrow_p (stmt))
2793 TREE_NOTHROW (exp) = 1;
2795 CALL_EXPR_TAILCALL (exp) = gimple_call_tail_p (stmt);
2796 CALL_EXPR_MUST_TAIL_CALL (exp) = gimple_call_must_tail_p (stmt);
2797 CALL_EXPR_RETURN_SLOT_OPT (exp) = gimple_call_return_slot_opt_p (stmt);
2799 && fndecl_built_in_p (decl, BUILT_IN_NORMAL)
2800 && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (decl)))
2801 CALL_ALLOCA_FOR_VAR_P (exp) = gimple_call_alloca_for_var_p (stmt);
2803 CALL_FROM_THUNK_P (exp) = gimple_call_from_thunk_p (stmt);
2804 CALL_EXPR_VA_ARG_PACK (exp) = gimple_call_va_arg_pack_p (stmt);
2805 CALL_EXPR_BY_DESCRIPTOR (exp) = gimple_call_by_descriptor_p (stmt);
2806 SET_EXPR_LOCATION (exp, gimple_location (stmt));
2808 /* Must come after copying location. */
2809 copy_warning (exp, stmt);
2811 /* Ensure RTL is created for debug args. */
2812 if (decl && DECL_HAS_DEBUG_ARGS_P (decl))
2814 vec<tree, va_gc> **debug_args = decl_debug_args_lookup (decl);
2819 for (ix = 1; (*debug_args)->iterate (ix, &dtemp); ix += 2)
2821 gcc_assert (TREE_CODE (dtemp) == DEBUG_EXPR_DECL);
2822 expand_debug_expr (dtemp);
2826 rtx_insn *before_call = get_last_insn ();
2827 lhs = gimple_call_lhs (stmt);
2829 expand_assignment (lhs, exp, false);
2831 expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
2833 /* If the gimple call is an indirect call and has 'nocf_check'
2834 attribute find a generated CALL insn to mark it as no
2835 control-flow verification is needed. */
2836 if (gimple_call_nocf_check_p (stmt)
2837 && !gimple_call_fndecl (stmt))
2839 rtx_insn *last = get_last_insn ();
2840 while (!CALL_P (last)
2841 && last != before_call)
2842 last = PREV_INSN (last);
2844 if (last != before_call)
2845 add_reg_note (last, REG_CALL_NOCF_CHECK, const0_rtx);
2848 mark_transaction_restart_calls (stmt);
2852 /* Generate RTL for an asm statement (explicit assembler code).
2853 STRING is a STRING_CST node containing the assembler code text,
2854 or an ADDR_EXPR containing a STRING_CST. VOL nonzero means the
2855 insn is volatile; don't optimize it. */
2858 expand_asm_loc (tree string, int vol, location_t locus)
2862 body = gen_rtx_ASM_INPUT_loc (VOIDmode,
2863 ggc_strdup (TREE_STRING_POINTER (string)),
2866 MEM_VOLATILE_P (body) = vol;
2868 /* Non-empty basic ASM implicitly clobbers memory. */
2869 if (TREE_STRING_LENGTH (string) != 0)
2872 unsigned i, nclobbers;
2873 auto_vec<rtx> input_rvec, output_rvec;
2874 auto_vec<machine_mode> input_mode;
2875 auto_vec<const char *> constraints;
2876 auto_vec<rtx> clobber_rvec;
2877 HARD_REG_SET clobbered_regs;
2878 CLEAR_HARD_REG_SET (clobbered_regs);
2880 clob = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
2881 clobber_rvec.safe_push (clob);
2883 if (targetm.md_asm_adjust)
2884 targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
2885 constraints, clobber_rvec, clobbered_regs,
2889 nclobbers = clobber_rvec.length ();
2890 body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (1 + nclobbers));
2892 XVECEXP (body, 0, 0) = asm_op;
2893 for (i = 0; i < nclobbers; i++)
2894 XVECEXP (body, 0, i + 1) = gen_rtx_CLOBBER (VOIDmode, clobber_rvec[i]);
2900 /* Return the number of times character C occurs in string S. */
2902 n_occurrences (int c, const char *s)
2910 /* A subroutine of expand_asm_operands. Check that all operands have
2911 the same number of alternatives. Return true if so. */
2914 check_operand_nalternatives (const vec<const char *> &constraints)
2916 unsigned len = constraints.length();
2919 int nalternatives = n_occurrences (',', constraints[0]);
2921 if (nalternatives + 1 > MAX_RECOG_ALTERNATIVES)
2923 error ("too many alternatives in %<asm%>");
2927 for (unsigned i = 1; i < len; ++i)
2928 if (n_occurrences (',', constraints[i]) != nalternatives)
2930 error ("operand constraints for %<asm%> differ "
2931 "in number of alternatives");
2938 /* Check for overlap between registers marked in CLOBBERED_REGS and
2939 anything inappropriate in T. Emit error and return the register
2940 variable definition for error, NULL_TREE for ok. */
2943 tree_conflicts_with_clobbers_p (tree t, HARD_REG_SET *clobbered_regs,
2946 /* Conflicts between asm-declared register variables and the clobber
2947 list are not allowed. */
2948 tree overlap = tree_overlaps_hard_reg_set (t, clobbered_regs);
2952 error_at (loc, "%<asm%> specifier for variable %qE conflicts with "
2953 "%<asm%> clobber list", DECL_NAME (overlap));
2955 /* Reset registerness to stop multiple errors emitted for a single
2957 DECL_REGISTER (overlap) = 0;
2964 /* Check that the given REGNO spanning NREGS is a valid
2965 asm clobber operand. Some HW registers cannot be
2966 saved/restored, hence they should not be clobbered by
2969 asm_clobber_reg_is_valid (int regno, int nregs, const char *regname)
2971 bool is_valid = true;
2972 HARD_REG_SET regset;
2974 CLEAR_HARD_REG_SET (regset);
2976 add_range_to_hard_reg_set (®set, regno, nregs);
2978 /* Clobbering the PIC register is an error. */
2979 if (PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM
2980 && overlaps_hard_reg_set_p (regset, Pmode, PIC_OFFSET_TABLE_REGNUM))
2982 /* ??? Diagnose during gimplification? */
2983 error ("PIC register clobbered by %qs in %<asm%>", regname);
2986 else if (!in_hard_reg_set_p
2987 (accessible_reg_set, reg_raw_mode[regno], regno))
2989 /* ??? Diagnose during gimplification? */
2990 error ("the register %qs cannot be clobbered in %<asm%>"
2991 " for the current target", regname);
2995 /* Clobbering the stack pointer register is deprecated. GCC expects
2996 the value of the stack pointer after an asm statement to be the same
2997 as it was before, so no asm can validly clobber the stack pointer in
2998 the usual sense. Adding the stack pointer to the clobber list has
2999 traditionally had some undocumented and somewhat obscure side-effects. */
3000 if (overlaps_hard_reg_set_p (regset, Pmode, STACK_POINTER_REGNUM))
3002 crtl->sp_is_clobbered_by_asm = true;
3003 if (warning (OPT_Wdeprecated, "listing the stack pointer register"
3004 " %qs in a clobber list is deprecated", regname))
3005 inform (input_location, "the value of the stack pointer after"
3006 " an %<asm%> statement must be the same as it was before"
3013 /* Generate RTL for an asm statement with arguments.
3014 STRING is the instruction template.
3015 OUTPUTS is a list of output arguments (lvalues); INPUTS a list of inputs.
3016 Each output or input has an expression in the TREE_VALUE and
3017 a tree list in TREE_PURPOSE which in turn contains a constraint
3018 name in TREE_VALUE (or NULL_TREE) and a constraint string
3020 CLOBBERS is a list of STRING_CST nodes each naming a hard register
3021 that is clobbered by this insn.
3023 LABELS is a list of labels, and if LABELS is non-NULL, FALLTHRU_BB
3024 should be the fallthru basic block of the asm goto.
3026 Not all kinds of lvalue that may appear in OUTPUTS can be stored directly.
3027 Some elements of OUTPUTS may be replaced with trees representing temporary
3028 values. The caller should copy those temporary values to the originally
3031 VOL nonzero means the insn is volatile; don't optimize it. */
3034 expand_asm_stmt (gasm *stmt)
3036 class save_input_location
3041 explicit save_input_location(location_t where)
3043 old = input_location;
3044 input_location = where;
3047 ~save_input_location()
3049 input_location = old;
3053 location_t locus = gimple_location (stmt);
3055 if (gimple_asm_input_p (stmt))
3057 const char *s = gimple_asm_string (stmt);
3058 tree string = build_string (strlen (s), s);
3059 expand_asm_loc (string, gimple_asm_volatile_p (stmt), locus);
3063 /* There are some legacy diagnostics in here. */
3064 save_input_location s_i_l(locus);
3066 unsigned noutputs = gimple_asm_noutputs (stmt);
3067 unsigned ninputs = gimple_asm_ninputs (stmt);
3068 unsigned nlabels = gimple_asm_nlabels (stmt);
3070 bool error_seen = false;
3072 /* ??? Diagnose during gimplification? */
3073 if (ninputs + noutputs + nlabels > MAX_RECOG_OPERANDS)
3075 error_at (locus, "more than %d operands in %<asm%>", MAX_RECOG_OPERANDS);
3079 auto_vec<tree, MAX_RECOG_OPERANDS> output_tvec;
3080 auto_vec<tree, MAX_RECOG_OPERANDS> input_tvec;
3081 auto_vec<const char *, MAX_RECOG_OPERANDS> constraints;
3083 /* Copy the gimple vectors into new vectors that we can manipulate. */
3085 output_tvec.safe_grow (noutputs, true);
3086 input_tvec.safe_grow (ninputs, true);
3087 constraints.safe_grow (noutputs + ninputs, true);
3089 for (i = 0; i < noutputs; ++i)
3091 tree t = gimple_asm_output_op (stmt, i);
3092 output_tvec[i] = TREE_VALUE (t);
3093 constraints[i] = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3095 for (i = 0; i < ninputs; i++)
3097 tree t = gimple_asm_input_op (stmt, i);
3098 input_tvec[i] = TREE_VALUE (t);
3099 constraints[i + noutputs]
3100 = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3103 /* ??? Diagnose during gimplification? */
3104 if (! check_operand_nalternatives (constraints))
3107 /* Count the number of meaningful clobbered registers, ignoring what
3108 we would ignore later. */
3109 auto_vec<rtx> clobber_rvec;
3110 HARD_REG_SET clobbered_regs;
3111 CLEAR_HARD_REG_SET (clobbered_regs);
3113 if (unsigned n = gimple_asm_nclobbers (stmt))
3115 clobber_rvec.reserve (n);
3116 for (i = 0; i < n; i++)
3118 tree t = gimple_asm_clobber_op (stmt, i);
3119 const char *regname = TREE_STRING_POINTER (TREE_VALUE (t));
3122 j = decode_reg_name_and_count (regname, &nregs);
3127 /* ??? Diagnose during gimplification? */
3128 error_at (locus, "unknown register name %qs in %<asm%>",
3134 rtx x = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
3135 clobber_rvec.safe_push (x);
3139 /* Otherwise we should have -1 == empty string
3140 or -3 == cc, which is not a register. */
3141 gcc_assert (j == -1 || j == -3);
3145 for (int reg = j; reg < j + nregs; reg++)
3147 if (!asm_clobber_reg_is_valid (reg, nregs, regname))
3150 SET_HARD_REG_BIT (clobbered_regs, reg);
3151 rtx x = gen_rtx_REG (reg_raw_mode[reg], reg);
3152 clobber_rvec.safe_push (x);
3157 /* First pass over inputs and outputs checks validity and sets
3158 mark_addressable if needed. */
3159 /* ??? Diagnose during gimplification? */
3161 for (i = 0; i < noutputs; ++i)
3163 tree val = output_tvec[i];
3164 tree type = TREE_TYPE (val);
3165 const char *constraint;
3170 /* Try to parse the output constraint. If that fails, there's
3171 no point in going further. */
3172 constraint = constraints[i];
3173 if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
3174 &allows_mem, &allows_reg, &is_inout))
3177 /* If the output is a hard register, verify it doesn't conflict with
3178 any other operand's possible hard register use. */
3180 && REG_P (DECL_RTL (val))
3181 && HARD_REGISTER_P (DECL_RTL (val)))
3183 unsigned j, output_hregno = REGNO (DECL_RTL (val));
3184 bool early_clobber_p = strchr (constraints[i], '&') != NULL;
3185 unsigned long match;
3187 /* Verify the other outputs do not use the same hard register. */
3188 for (j = i + 1; j < noutputs; ++j)
3189 if (DECL_P (output_tvec[j])
3190 && REG_P (DECL_RTL (output_tvec[j]))
3191 && HARD_REGISTER_P (DECL_RTL (output_tvec[j]))
3192 && output_hregno == REGNO (DECL_RTL (output_tvec[j])))
3194 error_at (locus, "invalid hard register usage between output "
3199 /* Verify matching constraint operands use the same hard register
3200 and that the non-matching constraint operands do not use the same
3201 hard register if the output is an early clobber operand. */
3202 for (j = 0; j < ninputs; ++j)
3203 if (DECL_P (input_tvec[j])
3204 && REG_P (DECL_RTL (input_tvec[j]))
3205 && HARD_REGISTER_P (DECL_RTL (input_tvec[j])))
3207 unsigned input_hregno = REGNO (DECL_RTL (input_tvec[j]));
3208 switch (*constraints[j + noutputs])
3210 case '0': case '1': case '2': case '3': case '4':
3211 case '5': case '6': case '7': case '8': case '9':
3212 match = strtoul (constraints[j + noutputs], NULL, 10);
3219 && output_hregno != input_hregno)
3221 error_at (locus, "invalid hard register usage between "
3222 "output operand and matching constraint operand");
3225 else if (early_clobber_p
3227 && output_hregno == input_hregno)
3229 error_at (locus, "invalid hard register usage between "
3230 "earlyclobber operand and input operand");
3240 && REG_P (DECL_RTL (val))
3241 && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type))))
3242 mark_addressable (val);
3245 for (i = 0; i < ninputs; ++i)
3247 bool allows_reg, allows_mem;
3248 const char *constraint;
3250 constraint = constraints[i + noutputs];
3251 if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3252 constraints.address (),
3253 &allows_mem, &allows_reg))
3256 if (! allows_reg && allows_mem)
3257 mark_addressable (input_tvec[i]);
3260 /* Second pass evaluates arguments. */
3262 /* Make sure stack is consistent for asm goto. */
3264 do_pending_stack_adjust ();
3265 int old_generating_concat_p = generating_concat_p;
3267 /* Vector of RTX's of evaluated output operands. */
3268 auto_vec<rtx, MAX_RECOG_OPERANDS> output_rvec;
3269 auto_vec<int, MAX_RECOG_OPERANDS> inout_opnum;
3270 rtx_insn *after_rtl_seq = NULL, *after_rtl_end = NULL;
3272 output_rvec.safe_grow (noutputs, true);
3274 for (i = 0; i < noutputs; ++i)
3276 tree val = output_tvec[i];
3277 tree type = TREE_TYPE (val);
3278 bool is_inout, allows_reg, allows_mem, ok;
3281 ok = parse_output_constraint (&constraints[i], i, ninputs,
3282 noutputs, &allows_mem, &allows_reg,
3286 /* If an output operand is not a decl or indirect ref and our constraint
3287 allows a register, make a temporary to act as an intermediate.
3288 Make the asm insn write into that, then we will copy it to
3289 the real output operand. Likewise for promoted variables. */
3291 generating_concat_p = 0;
3293 gcc_assert (TREE_CODE (val) != INDIRECT_REF);
3294 if (((TREE_CODE (val) == MEM_REF
3295 && TREE_CODE (TREE_OPERAND (val, 0)) != ADDR_EXPR)
3298 && (allows_mem || REG_P (DECL_RTL (val)))
3299 && ! (REG_P (DECL_RTL (val))
3300 && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type)))
3303 || TREE_ADDRESSABLE (type)
3304 || (!tree_fits_poly_int64_p (TYPE_SIZE (type))
3305 && !known_size_p (max_int_size_in_bytes (type))))
3307 op = expand_expr (val, NULL_RTX, VOIDmode,
3308 !allows_reg ? EXPAND_MEMORY : EXPAND_WRITE);
3310 op = validize_mem (op);
3312 if (! allows_reg && !MEM_P (op))
3314 error_at (locus, "output number %d not directly addressable", i);
3317 if ((! allows_mem && MEM_P (op) && GET_MODE (op) != BLKmode)
3318 || GET_CODE (op) == CONCAT)
3321 op = gen_reg_rtx (GET_MODE (op));
3323 generating_concat_p = old_generating_concat_p;
3326 emit_move_insn (op, old_op);
3328 push_to_sequence2 (after_rtl_seq, after_rtl_end);
3329 emit_move_insn (old_op, op);
3330 after_rtl_seq = get_insns ();
3331 after_rtl_end = get_last_insn ();
3337 op = assign_temp (type, 0, 1);
3338 op = validize_mem (op);
3339 if (!MEM_P (op) && TREE_CODE (val) == SSA_NAME)
3340 set_reg_attrs_for_decl_rtl (SSA_NAME_VAR (val), op);
3342 generating_concat_p = old_generating_concat_p;
3344 push_to_sequence2 (after_rtl_seq, after_rtl_end);
3345 expand_assignment (val, make_tree (type, op), false);
3346 after_rtl_seq = get_insns ();
3347 after_rtl_end = get_last_insn ();
3350 output_rvec[i] = op;
3353 inout_opnum.safe_push (i);
3356 const char *str = gimple_asm_string (stmt);
3361 inout_opnum.truncate (0);
3362 output_rvec.truncate (0);
3363 clobber_rvec.truncate (0);
3364 constraints.truncate (0);
3365 CLEAR_HARD_REG_SET (clobbered_regs);
3369 auto_vec<rtx, MAX_RECOG_OPERANDS> input_rvec;
3370 auto_vec<machine_mode, MAX_RECOG_OPERANDS> input_mode;
3372 input_rvec.safe_grow (ninputs, true);
3373 input_mode.safe_grow (ninputs, true);
3375 generating_concat_p = 0;
3377 for (i = 0; i < ninputs; ++i)
3379 tree val = input_tvec[i];
3380 tree type = TREE_TYPE (val);
3381 bool allows_reg, allows_mem, ok;
3382 const char *constraint;
3385 constraint = constraints[i + noutputs];
3386 ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3387 constraints.address (),
3388 &allows_mem, &allows_reg);
3391 /* EXPAND_INITIALIZER will not generate code for valid initializer
3392 constants, but will still generate code for other types of operand.
3393 This is the behavior we want for constant constraints. */
3394 op = expand_expr (val, NULL_RTX, VOIDmode,
3395 allows_reg ? EXPAND_NORMAL
3396 : allows_mem ? EXPAND_MEMORY
3397 : EXPAND_INITIALIZER);
3399 /* Never pass a CONCAT to an ASM. */
3400 if (GET_CODE (op) == CONCAT)
3401 op = force_reg (GET_MODE (op), op);
3402 else if (MEM_P (op))
3403 op = validize_mem (op);
3405 if (asm_operand_ok (op, constraint, NULL) <= 0)
3407 if (allows_reg && TYPE_MODE (type) != BLKmode)
3408 op = force_reg (TYPE_MODE (type), op);
3409 else if (!allows_mem)
3410 warning_at (locus, 0, "%<asm%> operand %d probably does not match "
3411 "constraints", i + noutputs);
3412 else if (MEM_P (op))
3414 /* We won't recognize either volatile memory or memory
3415 with a queued address as available a memory_operand
3416 at this point. Ignore it: clearly this *is* a memory. */
3422 input_mode[i] = TYPE_MODE (type);
3425 /* For in-out operands, copy output rtx to input rtx. */
3426 unsigned ninout = inout_opnum.length ();
3427 for (i = 0; i < ninout; i++)
3429 int j = inout_opnum[i];
3430 rtx o = output_rvec[j];
3432 input_rvec.safe_push (o);
3433 input_mode.safe_push (GET_MODE (o));
3436 sprintf (buffer, "%d", j);
3437 constraints.safe_push (ggc_strdup (buffer));
3441 /* Sometimes we wish to automatically clobber registers across an asm.
3442 Case in point is when the i386 backend moved from cc0 to a hard reg --
3443 maintaining source-level compatibility means automatically clobbering
3444 the flags register. */
3445 rtx_insn *after_md_seq = NULL;
3446 if (targetm.md_asm_adjust)
3448 = targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
3449 constraints, clobber_rvec, clobbered_regs,
3452 /* Do not allow the hook to change the output and input count,
3453 lest it mess up the operand numbering. */
3454 gcc_assert (output_rvec.length() == noutputs);
3455 gcc_assert (input_rvec.length() == ninputs);
3456 gcc_assert (constraints.length() == noutputs + ninputs);
3458 /* But it certainly can adjust the clobbers. */
3459 unsigned nclobbers = clobber_rvec.length ();
3461 /* Third pass checks for easy conflicts. */
3462 /* ??? Why are we doing this on trees instead of rtx. */
3464 bool clobber_conflict_found = 0;
3465 for (i = 0; i < noutputs; ++i)
3466 if (tree_conflicts_with_clobbers_p (output_tvec[i], &clobbered_regs, locus))
3467 clobber_conflict_found = 1;
3468 for (i = 0; i < ninputs - ninout; ++i)
3469 if (tree_conflicts_with_clobbers_p (input_tvec[i], &clobbered_regs, locus))
3470 clobber_conflict_found = 1;
3472 /* Make vectors for the expression-rtx, constraint strings,
3473 and named operands. */
3475 rtvec argvec = rtvec_alloc (ninputs);
3476 rtvec constraintvec = rtvec_alloc (ninputs);
3477 rtvec labelvec = rtvec_alloc (nlabels);
3479 rtx body = gen_rtx_ASM_OPERANDS ((noutputs == 0 ? VOIDmode
3480 : GET_MODE (output_rvec[0])),
3482 "", 0, argvec, constraintvec,
3484 MEM_VOLATILE_P (body) = gimple_asm_volatile_p (stmt);
3486 for (i = 0; i < ninputs; ++i)
3488 ASM_OPERANDS_INPUT (body, i) = input_rvec[i];
3489 ASM_OPERANDS_INPUT_CONSTRAINT_EXP (body, i)
3490 = gen_rtx_ASM_INPUT_loc (input_mode[i],
3491 constraints[i + noutputs],
3495 /* Copy labels to the vector. */
3496 rtx_code_label *fallthru_label = NULL;
3499 basic_block fallthru_bb = NULL;
3500 edge fallthru = find_fallthru_edge (gimple_bb (stmt)->succs);
3502 fallthru_bb = fallthru->dest;
3504 for (i = 0; i < nlabels; ++i)
3506 tree label = TREE_VALUE (gimple_asm_label_op (stmt, i));
3508 /* If asm goto has any labels in the fallthru basic block, use
3509 a label that we emit immediately after the asm goto. Expansion
3510 may insert further instructions into the same basic block after
3511 asm goto and if we don't do this, insertion of instructions on
3512 the fallthru edge might misbehave. See PR58670. */
3513 if (fallthru_bb && label_to_block (cfun, label) == fallthru_bb)
3515 if (fallthru_label == NULL_RTX)
3516 fallthru_label = gen_label_rtx ();
3520 r = label_rtx (label);
3521 ASM_OPERANDS_LABEL (body, i) = gen_rtx_LABEL_REF (Pmode, r);
3525 /* Now, for each output, construct an rtx
3526 (set OUTPUT (asm_operands INSN OUTPUTCONSTRAINT OUTPUTNUMBER
3527 ARGVEC CONSTRAINTS OPNAMES))
3528 If there is more than one, put them inside a PARALLEL. */
3530 if (noutputs == 0 && nclobbers == 0)
3532 /* No output operands: put in a raw ASM_OPERANDS rtx. */
3534 emit_jump_insn (body);
3538 else if (noutputs == 1 && nclobbers == 0)
3540 ASM_OPERANDS_OUTPUT_CONSTRAINT (body) = constraints[0];
3542 emit_jump_insn (gen_rtx_SET (output_rvec[0], body));
3544 emit_insn (gen_rtx_SET (output_rvec[0], body));
3554 body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (num + nclobbers));
3556 /* For each output operand, store a SET. */
3557 for (i = 0; i < noutputs; ++i)
3559 rtx src, o = output_rvec[i];
3562 ASM_OPERANDS_OUTPUT_CONSTRAINT (obody) = constraints[0];
3567 src = gen_rtx_ASM_OPERANDS (GET_MODE (o),
3568 ASM_OPERANDS_TEMPLATE (obody),
3569 constraints[i], i, argvec,
3570 constraintvec, labelvec, locus);
3571 MEM_VOLATILE_P (src) = gimple_asm_volatile_p (stmt);
3573 XVECEXP (body, 0, i) = gen_rtx_SET (o, src);
3576 /* If there are no outputs (but there are some clobbers)
3577 store the bare ASM_OPERANDS into the PARALLEL. */
3579 XVECEXP (body, 0, i++) = obody;
3581 /* Store (clobber REG) for each clobbered register specified. */
3582 for (unsigned j = 0; j < nclobbers; ++j)
3584 rtx clobbered_reg = clobber_rvec[j];
3586 /* Do sanity check for overlap between clobbers and respectively
3587 input and outputs that hasn't been handled. Such overlap
3588 should have been detected and reported above. */
3589 if (!clobber_conflict_found && REG_P (clobbered_reg))
3591 /* We test the old body (obody) contents to avoid
3592 tripping over the under-construction body. */
3593 for (unsigned k = 0; k < noutputs; ++k)
3594 if (reg_overlap_mentioned_p (clobbered_reg, output_rvec[k]))
3595 internal_error ("%<asm%> clobber conflict with "
3598 for (unsigned k = 0; k < ninputs - ninout; ++k)
3599 if (reg_overlap_mentioned_p (clobbered_reg, input_rvec[k]))
3600 internal_error ("%<asm%> clobber conflict with "
3604 XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobbered_reg);
3608 emit_jump_insn (body);
3613 generating_concat_p = old_generating_concat_p;
3616 emit_label (fallthru_label);
3619 emit_insn (after_md_seq);
3623 emit_insn (after_rtl_seq);
3629 FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
3632 for (rtx_insn *curr = after_rtl_seq;
3634 curr = NEXT_INSN (curr))
3635 emit_insn (copy_insn (PATTERN (curr)));
3636 rtx_insn *copy = get_insns ();
3638 insert_insn_on_edge (copy, e);
3644 crtl->has_asm_statement = 1;
3647 /* Emit code to jump to the address
3648 specified by the pointer expression EXP. */
3651 expand_computed_goto (tree exp)
3653 rtx x = expand_normal (exp);
3655 do_pending_stack_adjust ();
3656 emit_indirect_jump (x);
3659 /* Generate RTL code for a `goto' statement with target label LABEL.
3660 LABEL should be a LABEL_DECL tree node that was or will later be
3661 defined with `expand_label'. */
3664 expand_goto (tree label)
3668 /* Check for a nonlocal goto to a containing function. Should have
3669 gotten translated to __builtin_nonlocal_goto. */
3670 tree context = decl_function_context (label);
3671 gcc_assert (!context || context == current_function_decl);
3674 emit_jump (jump_target_rtx (label));
3677 /* Output a return with no value. */
3680 expand_null_return_1 (void)
3682 clear_pending_stack_adjust ();
3683 do_pending_stack_adjust ();
3684 emit_jump (return_label);
3687 /* Generate RTL to return from the current function, with no value.
3688 (That is, we do not do anything about returning any value.) */
3691 expand_null_return (void)
3693 /* If this function was declared to return a value, but we
3694 didn't, clobber the return registers so that they are not
3695 propagated live to the rest of the function. */
3696 clobber_return_register ();
3698 expand_null_return_1 ();
3701 /* Generate RTL to return from the current function, with value VAL. */
3704 expand_value_return (rtx val)
3706 /* Copy the value to the return location unless it's already there. */
3708 tree decl = DECL_RESULT (current_function_decl);
3709 rtx return_reg = DECL_RTL (decl);
3710 if (return_reg != val)
3712 tree funtype = TREE_TYPE (current_function_decl);
3713 tree type = TREE_TYPE (decl);
3714 int unsignedp = TYPE_UNSIGNED (type);
3715 machine_mode old_mode = DECL_MODE (decl);
3717 if (DECL_BY_REFERENCE (decl))
3718 mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 2);
3720 mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 1);
3722 if (mode != old_mode)
3724 /* Some ABIs require scalar floating point modes to be returned
3725 in a wider scalar integer mode. We need to explicitly
3726 reinterpret to an integer mode of the correct precision
3727 before extending to the desired result. */
3728 if (SCALAR_INT_MODE_P (mode)
3729 && SCALAR_FLOAT_MODE_P (old_mode)
3730 && known_gt (GET_MODE_SIZE (mode), GET_MODE_SIZE (old_mode)))
3731 val = convert_float_to_wider_int (mode, old_mode, val);
3733 val = convert_modes (mode, old_mode, val, unsignedp);
3736 if (GET_CODE (return_reg) == PARALLEL)
3737 emit_group_load (return_reg, val, type, int_size_in_bytes (type));
3739 emit_move_insn (return_reg, val);
3742 expand_null_return_1 ();
3745 /* Generate RTL to evaluate the expression RETVAL and return it
3746 from the current function. */
3749 expand_return (tree retval)
3755 /* If function wants no value, give it none. */
3756 if (TREE_CODE (TREE_TYPE (TREE_TYPE (current_function_decl))) == VOID_TYPE)
3758 expand_normal (retval);
3759 expand_null_return ();
3763 if (retval == error_mark_node)
3765 /* Treat this like a return of no value from a function that
3767 expand_null_return ();
3770 else if ((TREE_CODE (retval) == MODIFY_EXPR
3771 || TREE_CODE (retval) == INIT_EXPR)
3772 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
3773 retval_rhs = TREE_OPERAND (retval, 1);
3775 retval_rhs = retval;
3777 result_rtl = DECL_RTL (DECL_RESULT (current_function_decl));
3779 /* If we are returning the RESULT_DECL, then the value has already
3780 been stored into it, so we don't have to do anything special. */
3781 if (TREE_CODE (retval_rhs) == RESULT_DECL)
3782 expand_value_return (result_rtl);
3784 /* If the result is an aggregate that is being returned in one (or more)
3785 registers, load the registers here. */
3787 else if (retval_rhs != 0
3788 && TYPE_MODE (TREE_TYPE (retval_rhs)) == BLKmode
3789 && REG_P (result_rtl))
3791 val = copy_blkmode_to_reg (GET_MODE (result_rtl), retval_rhs);
3794 /* Use the mode of the result value on the return register. */
3795 PUT_MODE (result_rtl, GET_MODE (val));
3796 expand_value_return (val);
3799 expand_null_return ();
3801 else if (retval_rhs != 0
3802 && !VOID_TYPE_P (TREE_TYPE (retval_rhs))
3803 && (REG_P (result_rtl)
3804 || (GET_CODE (result_rtl) == PARALLEL)))
3806 /* Compute the return value into a temporary (usually a pseudo reg). */
3808 = assign_temp (TREE_TYPE (DECL_RESULT (current_function_decl)), 0, 1);
3809 val = expand_expr (retval_rhs, val, GET_MODE (val), EXPAND_NORMAL);
3810 val = force_not_mem (val);
3811 expand_value_return (val);
3815 /* No hard reg used; calculate value into hard return reg. */
3816 expand_expr (retval, const0_rtx, VOIDmode, EXPAND_NORMAL);
3817 expand_value_return (result_rtl);
3821 /* Expand a clobber of LHS. If LHS is stored it in a multi-part
3822 register, tell the rtl optimizers that its value is no longer
3826 expand_clobber (tree lhs)
3830 rtx decl_rtl = DECL_RTL_IF_SET (lhs);
3831 if (decl_rtl && REG_P (decl_rtl))
3833 machine_mode decl_mode = GET_MODE (decl_rtl);
3834 if (maybe_gt (GET_MODE_SIZE (decl_mode),
3835 REGMODE_NATURAL_SIZE (decl_mode)))
3836 emit_clobber (decl_rtl);
3841 /* A subroutine of expand_gimple_stmt, expanding one gimple statement
3842 STMT that doesn't require special handling for outgoing edges. That
3843 is no tailcalls and no GIMPLE_COND. */
3846 expand_gimple_stmt_1 (gimple *stmt)
3850 set_curr_insn_location (gimple_location (stmt));
3852 switch (gimple_code (stmt))
3855 op0 = gimple_goto_dest (stmt);
3856 if (TREE_CODE (op0) == LABEL_DECL)
3859 expand_computed_goto (op0);
3862 expand_label (gimple_label_label (as_a <glabel *> (stmt)));
3865 case GIMPLE_PREDICT:
3869 gswitch *swtch = as_a <gswitch *> (stmt);
3870 if (gimple_switch_num_labels (swtch) == 1)
3871 expand_goto (CASE_LABEL (gimple_switch_default_label (swtch)));
3873 expand_case (swtch);
3877 expand_asm_stmt (as_a <gasm *> (stmt));
3880 expand_call_stmt (as_a <gcall *> (stmt));
3885 op0 = gimple_return_retval (as_a <greturn *> (stmt));
3887 /* If a return doesn't have a location, it very likely represents
3888 multiple user returns so we cannot let it inherit the location
3889 of the last statement of the previous basic block in RTL. */
3890 if (!gimple_has_location (stmt))
3891 set_curr_insn_location (cfun->function_end_locus);
3893 if (op0 && op0 != error_mark_node)
3895 tree result = DECL_RESULT (current_function_decl);
3897 /* If we are not returning the current function's RESULT_DECL,
3898 build an assignment to it. */
3901 /* I believe that a function's RESULT_DECL is unique. */
3902 gcc_assert (TREE_CODE (op0) != RESULT_DECL);
3904 /* ??? We'd like to use simply expand_assignment here,
3905 but this fails if the value is of BLKmode but the return
3906 decl is a register. expand_return has special handling
3907 for this combination, which eventually should move
3908 to common code. See comments there. Until then, let's
3909 build a modify expression :-/ */
3910 op0 = build2 (MODIFY_EXPR, TREE_TYPE (result),
3916 expand_null_return ();
3918 expand_return (op0);
3924 gassign *assign_stmt = as_a <gassign *> (stmt);
3925 tree lhs = gimple_assign_lhs (assign_stmt);
3927 /* Tree expand used to fiddle with |= and &= of two bitfield
3928 COMPONENT_REFs here. This can't happen with gimple, the LHS
3929 of binary assigns must be a gimple reg. */
3931 if (TREE_CODE (lhs) != SSA_NAME
3932 || gimple_assign_rhs_class (assign_stmt) == GIMPLE_SINGLE_RHS)
3934 tree rhs = gimple_assign_rhs1 (assign_stmt);
3935 gcc_assert (gimple_assign_rhs_class (assign_stmt)
3936 == GIMPLE_SINGLE_RHS);
3937 if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (rhs)
3938 /* Do not put locations on possibly shared trees. */
3939 && !is_gimple_min_invariant (rhs))
3940 SET_EXPR_LOCATION (rhs, gimple_location (stmt));
3941 if (TREE_CLOBBER_P (rhs))
3942 /* This is a clobber to mark the going out of scope for
3944 expand_clobber (lhs);
3946 expand_assignment (lhs, rhs,
3947 gimple_assign_nontemporal_move_p (
3953 bool nontemporal = gimple_assign_nontemporal_move_p (assign_stmt);
3954 struct separate_ops ops;
3955 bool promoted = false;
3957 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3958 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
3961 ops.code = gimple_assign_rhs_code (assign_stmt);
3962 ops.type = TREE_TYPE (lhs);
3963 switch (get_gimple_rhs_class (ops.code))
3965 case GIMPLE_TERNARY_RHS:
3966 ops.op2 = gimple_assign_rhs3 (assign_stmt);
3968 case GIMPLE_BINARY_RHS:
3969 ops.op1 = gimple_assign_rhs2 (assign_stmt);
3971 case GIMPLE_UNARY_RHS:
3972 ops.op0 = gimple_assign_rhs1 (assign_stmt);
3977 ops.location = gimple_location (stmt);
3979 /* If we want to use a nontemporal store, force the value to
3980 register first. If we store into a promoted register,
3981 don't directly expand to target. */
3982 temp = nontemporal || promoted ? NULL_RTX : target;
3983 temp = expand_expr_real_2 (&ops, temp, GET_MODE (target),
3990 int unsignedp = SUBREG_PROMOTED_SIGN (target);
3991 /* If TEMP is a VOIDmode constant, use convert_modes to make
3992 sure that we properly convert it. */
3993 if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
3995 temp = convert_modes (GET_MODE (target),
3996 TYPE_MODE (ops.type),
3998 temp = convert_modes (GET_MODE (SUBREG_REG (target)),
3999 GET_MODE (target), temp, unsignedp);
4002 convert_move (SUBREG_REG (target), temp, unsignedp);
4004 else if (nontemporal && emit_storent_insn (target, temp))
4008 temp = force_operand (temp, target);
4010 emit_move_insn (target, temp);
4021 /* Expand one gimple statement STMT and return the last RTL instruction
4022 before any of the newly generated ones.
4024 In addition to generating the necessary RTL instructions this also
4025 sets REG_EH_REGION notes if necessary and sets the current source
4026 location for diagnostics. */
4029 expand_gimple_stmt (gimple *stmt)
4031 location_t saved_location = input_location;
4032 rtx_insn *last = get_last_insn ();
4037 /* We need to save and restore the current source location so that errors
4038 discovered during expansion are emitted with the right location. But
4039 it would be better if the diagnostic routines used the source location
4040 embedded in the tree nodes rather than globals. */
4041 if (gimple_has_location (stmt))
4042 input_location = gimple_location (stmt);
4044 expand_gimple_stmt_1 (stmt);
4046 /* Free any temporaries used to evaluate this statement. */
4049 input_location = saved_location;
4051 /* Mark all insns that may trap. */
4052 lp_nr = lookup_stmt_eh_lp (stmt);
4056 for (insn = next_real_insn (last); insn;
4057 insn = next_real_insn (insn))
4059 if (! find_reg_note (insn, REG_EH_REGION, NULL_RTX)
4060 /* If we want exceptions for non-call insns, any
4061 may_trap_p instruction may throw. */
4062 && GET_CODE (PATTERN (insn)) != CLOBBER
4063 && GET_CODE (PATTERN (insn)) != USE
4064 && insn_could_throw_p (insn))
4065 make_reg_eh_region_note (insn, 0, lp_nr);
4072 /* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_CALL
4073 that has CALL_EXPR_TAILCALL set. Returns non-null if we actually
4074 generated a tail call (something that might be denied by the ABI
4075 rules governing the call; see calls.cc).
4077 Sets CAN_FALLTHRU if we generated a *conditional* tail call, and
4078 can still reach the rest of BB. The case here is __builtin_sqrt,
4079 where the NaN result goes through the external function (with a
4080 tailcall) and the normal result happens via a sqrt instruction. */
4083 expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru)
4085 rtx_insn *last2, *last;
4088 profile_probability probability;
4090 last2 = last = expand_gimple_stmt (stmt);
4092 for (last = NEXT_INSN (last); last; last = NEXT_INSN (last))
4093 if (CALL_P (last) && SIBLING_CALL_P (last))
4096 maybe_dump_rtl_for_gimple_stmt (stmt, last2);
4098 *can_fallthru = true;
4102 /* ??? Wouldn't it be better to just reset any pending stack adjust?
4103 Any instructions emitted here are about to be deleted. */
4104 do_pending_stack_adjust ();
4106 /* Remove any non-eh, non-abnormal edges that don't go to exit. */
4107 /* ??? I.e. the fallthrough edge. HOWEVER! If there were to be
4108 EH or abnormal edges, we shouldn't have created a tail call in
4109 the first place. So it seems to me we should just be removing
4110 all edges here, or redirecting the existing fallthru edge to
4113 probability = profile_probability::never ();
4115 for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
4117 if (!(e->flags & (EDGE_ABNORMAL | EDGE_EH)))
4119 if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
4120 e->dest->count -= e->count ();
4121 probability += e->probability;
4128 /* This is somewhat ugly: the call_expr expander often emits instructions
4129 after the sibcall (to perform the function return). These confuse the
4130 find_many_sub_basic_blocks code, so we need to get rid of these. */
4131 last = NEXT_INSN (last);
4132 gcc_assert (BARRIER_P (last));
4134 *can_fallthru = false;
4135 while (NEXT_INSN (last))
4137 /* For instance an sqrt builtin expander expands if with
4138 sibcall in the then and label for `else`. */
4139 if (LABEL_P (NEXT_INSN (last)))
4141 *can_fallthru = true;
4144 delete_insn (NEXT_INSN (last));
4147 e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_ABNORMAL
4149 e->probability = probability;
4151 update_bb_for_insn (bb);
4153 if (NEXT_INSN (last))
4155 bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
4158 if (BARRIER_P (last))
4159 BB_END (bb) = PREV_INSN (last);
4162 maybe_dump_rtl_for_gimple_stmt (stmt, last2);
4167 /* Return the difference between the floor and the truncated result of
4168 a signed division by OP1 with remainder MOD. */
4170 floor_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4172 /* (mod != 0 ? (op1 / mod < 0 ? -1 : 0) : 0) */
4173 return gen_rtx_IF_THEN_ELSE
4174 (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4175 gen_rtx_IF_THEN_ELSE
4176 (mode, gen_rtx_LT (BImode,
4177 gen_rtx_DIV (mode, op1, mod),
4179 constm1_rtx, const0_rtx),
4183 /* Return the difference between the ceil and the truncated result of
4184 a signed division by OP1 with remainder MOD. */
4186 ceil_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4188 /* (mod != 0 ? (op1 / mod > 0 ? 1 : 0) : 0) */
4189 return gen_rtx_IF_THEN_ELSE
4190 (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4191 gen_rtx_IF_THEN_ELSE
4192 (mode, gen_rtx_GT (BImode,
4193 gen_rtx_DIV (mode, op1, mod),
4195 const1_rtx, const0_rtx),
4199 /* Return the difference between the ceil and the truncated result of
4200 an unsigned division by OP1 with remainder MOD. */
4202 ceil_udiv_adjust (machine_mode mode, rtx mod, rtx op1 ATTRIBUTE_UNUSED)
4204 /* (mod != 0 ? 1 : 0) */
4205 return gen_rtx_IF_THEN_ELSE
4206 (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4207 const1_rtx, const0_rtx);
4210 /* Return the difference between the rounded and the truncated result
4211 of a signed division by OP1 with remainder MOD. Halfway cases are
4212 rounded away from zero, rather than to the nearest even number. */
4214 round_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4216 /* (abs (mod) >= abs (op1) - abs (mod)
4217 ? (op1 / mod > 0 ? 1 : -1)
4219 return gen_rtx_IF_THEN_ELSE
4220 (mode, gen_rtx_GE (BImode, gen_rtx_ABS (mode, mod),
4221 gen_rtx_MINUS (mode,
4222 gen_rtx_ABS (mode, op1),
4223 gen_rtx_ABS (mode, mod))),
4224 gen_rtx_IF_THEN_ELSE
4225 (mode, gen_rtx_GT (BImode,
4226 gen_rtx_DIV (mode, op1, mod),
4228 const1_rtx, constm1_rtx),
4232 /* Return the difference between the rounded and the truncated result
4233 of a unsigned division by OP1 with remainder MOD. Halfway cases
4234 are rounded away from zero, rather than to the nearest even
4237 round_udiv_adjust (machine_mode mode, rtx mod, rtx op1)
4239 /* (mod >= op1 - mod ? 1 : 0) */
4240 return gen_rtx_IF_THEN_ELSE
4241 (mode, gen_rtx_GE (BImode, mod,
4242 gen_rtx_MINUS (mode, op1, mod)),
4243 const1_rtx, const0_rtx);
4246 /* Convert X to MODE, that must be Pmode or ptr_mode, without emitting
4250 convert_debug_memory_address (scalar_int_mode mode, rtx x,
4253 #ifndef POINTERS_EXTEND_UNSIGNED
4254 gcc_assert (mode == Pmode
4255 || mode == targetm.addr_space.address_mode (as));
4256 gcc_assert (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode);
4260 gcc_assert (targetm.addr_space.valid_pointer_mode (mode, as));
4262 if (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode)
4265 /* X must have some form of address mode already. */
4266 scalar_int_mode xmode = as_a <scalar_int_mode> (GET_MODE (x));
4267 if (GET_MODE_PRECISION (mode) < GET_MODE_PRECISION (xmode))
4268 x = lowpart_subreg (mode, x, xmode);
4269 else if (POINTERS_EXTEND_UNSIGNED > 0)
4270 x = gen_rtx_ZERO_EXTEND (mode, x);
4271 else if (!POINTERS_EXTEND_UNSIGNED)
4272 x = gen_rtx_SIGN_EXTEND (mode, x);
4275 switch (GET_CODE (x))
4278 if ((SUBREG_PROMOTED_VAR_P (x)
4279 || (REG_P (SUBREG_REG (x)) && REG_POINTER (SUBREG_REG (x)))
4280 || (GET_CODE (SUBREG_REG (x)) == PLUS
4281 && REG_P (XEXP (SUBREG_REG (x), 0))
4282 && REG_POINTER (XEXP (SUBREG_REG (x), 0))
4283 && CONST_INT_P (XEXP (SUBREG_REG (x), 1))))
4284 && GET_MODE (SUBREG_REG (x)) == mode)
4285 return SUBREG_REG (x);
4288 temp = gen_rtx_LABEL_REF (mode, label_ref_label (x));
4289 LABEL_REF_NONLOCAL_P (temp) = LABEL_REF_NONLOCAL_P (x);
4292 temp = shallow_copy_rtx (x);
4293 PUT_MODE (temp, mode);
4296 temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4298 temp = gen_rtx_CONST (mode, temp);
4302 if (CONST_INT_P (XEXP (x, 1)))
4304 temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4306 return gen_rtx_fmt_ee (GET_CODE (x), mode, temp, XEXP (x, 1));
4312 /* Don't know how to express ptr_extend as operation in debug info. */
4315 #endif /* POINTERS_EXTEND_UNSIGNED */
4320 /* Map from SSA_NAMEs to corresponding DEBUG_EXPR_DECLs created
4321 by avoid_deep_ter_for_debug. */
4323 static hash_map<tree, tree> *deep_ter_debug_map;
4325 /* Split too deep TER chains for debug stmts using debug temporaries. */
4328 avoid_deep_ter_for_debug (gimple *stmt, int depth)
4330 use_operand_p use_p;
4332 FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
4334 tree use = USE_FROM_PTR (use_p);
4335 if (TREE_CODE (use) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (use))
4337 gimple *g = get_gimple_for_ssa_name (use);
4340 if (depth > 6 && !stmt_ends_bb_p (g))
4342 if (deep_ter_debug_map == NULL)
4343 deep_ter_debug_map = new hash_map<tree, tree>;
4345 tree &vexpr = deep_ter_debug_map->get_or_insert (use);
4348 vexpr = build_debug_expr_decl (TREE_TYPE (use));
4349 gimple *def_temp = gimple_build_debug_bind (vexpr, use, g);
4350 gimple_stmt_iterator gsi = gsi_for_stmt (g);
4351 gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
4352 avoid_deep_ter_for_debug (def_temp, 0);
4355 avoid_deep_ter_for_debug (g, depth + 1);
4359 /* Return an RTX equivalent to the value of the parameter DECL. */
4362 expand_debug_parm_decl (tree decl)
4364 rtx incoming = DECL_INCOMING_RTL (decl);
4367 && GET_MODE (incoming) != BLKmode
4368 && ((REG_P (incoming) && HARD_REGISTER_P (incoming))
4369 || (MEM_P (incoming)
4370 && REG_P (XEXP (incoming, 0))
4371 && HARD_REGISTER_P (XEXP (incoming, 0)))))
4373 rtx rtl = gen_rtx_ENTRY_VALUE (GET_MODE (incoming));
4375 #ifdef HAVE_window_save
4376 /* DECL_INCOMING_RTL uses the INCOMING_REGNO of parameter registers.
4377 If the target machine has an explicit window save instruction, the
4378 actual entry value is the corresponding OUTGOING_REGNO instead. */
4379 if (REG_P (incoming)
4380 && OUTGOING_REGNO (REGNO (incoming)) != REGNO (incoming))
4382 = gen_rtx_REG_offset (incoming, GET_MODE (incoming),
4383 OUTGOING_REGNO (REGNO (incoming)), 0);
4384 else if (MEM_P (incoming))
4386 rtx reg = XEXP (incoming, 0);
4387 if (OUTGOING_REGNO (REGNO (reg)) != REGNO (reg))
4389 reg = gen_raw_REG (GET_MODE (reg), OUTGOING_REGNO (REGNO (reg)));
4390 incoming = replace_equiv_address_nv (incoming, reg);
4393 incoming = copy_rtx (incoming);
4397 ENTRY_VALUE_EXP (rtl) = incoming;
4402 && GET_MODE (incoming) != BLKmode
4403 && !TREE_ADDRESSABLE (decl)
4405 && (XEXP (incoming, 0) == virtual_incoming_args_rtx
4406 || (GET_CODE (XEXP (incoming, 0)) == PLUS
4407 && XEXP (XEXP (incoming, 0), 0) == virtual_incoming_args_rtx
4408 && CONST_INT_P (XEXP (XEXP (incoming, 0), 1)))))
4409 return copy_rtx (incoming);
4414 /* Return an RTX equivalent to the value of the tree expression EXP. */
4417 expand_debug_expr (tree exp)
4419 rtx op0 = NULL_RTX, op1 = NULL_RTX, op2 = NULL_RTX;
4420 machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
4421 machine_mode inner_mode = VOIDmode;
4422 int unsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
4424 scalar_int_mode op0_mode, op1_mode, addr_mode;
4426 switch (TREE_CODE_CLASS (TREE_CODE (exp)))
4428 case tcc_expression:
4429 switch (TREE_CODE (exp))
4434 case WIDEN_MULT_PLUS_EXPR:
4435 case WIDEN_MULT_MINUS_EXPR:
4438 case TRUTH_ANDIF_EXPR:
4439 case TRUTH_ORIF_EXPR:
4440 case TRUTH_AND_EXPR:
4442 case TRUTH_XOR_EXPR:
4445 case TRUTH_NOT_EXPR:
4454 op2 = expand_debug_expr (TREE_OPERAND (exp, 2));
4461 if (mode == BLKmode)
4463 op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4466 switch (TREE_CODE (exp))
4472 case WIDEN_LSHIFT_EXPR:
4473 /* Ensure second operand isn't wider than the first one. */
4474 inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 1)));
4475 if (is_a <scalar_int_mode> (inner_mode, &op1_mode)
4476 && (GET_MODE_UNIT_PRECISION (mode)
4477 < GET_MODE_PRECISION (op1_mode)))
4478 op1 = lowpart_subreg (GET_MODE_INNER (mode), op1, op1_mode);
4487 if (mode == BLKmode)
4489 inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4490 op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4495 case tcc_comparison:
4496 unsignedp = TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)));
4504 case tcc_exceptional:
4505 case tcc_declaration:
4511 switch (TREE_CODE (exp))
4514 if (!lookup_constant_def (exp))
4516 if (strlen (TREE_STRING_POINTER (exp)) + 1
4517 != (size_t) TREE_STRING_LENGTH (exp))
4519 op0 = gen_rtx_CONST_STRING (Pmode, TREE_STRING_POINTER (exp));
4520 op0 = gen_rtx_MEM (BLKmode, op0);
4521 set_mem_attributes (op0, exp, 0);
4529 op0 = expand_expr (exp, NULL_RTX, mode, EXPAND_INITIALIZER);
4533 return immed_wide_int_const (poly_int_cst_value (exp), mode);
4536 gcc_assert (COMPLEX_MODE_P (mode));
4537 op0 = expand_debug_expr (TREE_REALPART (exp));
4538 op1 = expand_debug_expr (TREE_IMAGPART (exp));
4539 return gen_rtx_CONCAT (mode, op0, op1);
4541 case DEBUG_EXPR_DECL:
4542 op0 = DECL_RTL_IF_SET (exp);
4546 if (GET_MODE (op0) != mode)
4547 gcc_assert (VECTOR_TYPE_P (TREE_TYPE (exp)));
4552 op0 = gen_rtx_DEBUG_EXPR (mode);
4553 DEBUG_EXPR_TREE_DECL (op0) = exp;
4554 SET_DECL_RTL (exp, op0);
4564 op0 = DECL_RTL_IF_SET (exp);
4566 /* This decl was probably optimized away. */
4568 /* At least label RTXen are sometimes replaced by
4569 NOTE_INSN_DELETED_LABEL. Any notes here are not
4570 handled by copy_rtx. */
4574 || DECL_EXTERNAL (exp)
4575 || !TREE_STATIC (exp)
4577 || DECL_HARD_REGISTER (exp)
4578 || DECL_IN_CONSTANT_POOL (exp)
4580 || symtab_node::get (exp) == NULL)
4583 op0 = make_decl_rtl_for_debug (exp);
4585 || GET_CODE (XEXP (op0, 0)) != SYMBOL_REF
4586 || SYMBOL_REF_DECL (XEXP (op0, 0)) != exp)
4589 else if (VAR_P (exp)
4590 && is_global_var (exp)
4591 && symtab_node::get (exp) == NULL)
4594 op0 = copy_rtx (op0);
4596 if (GET_MODE (op0) == BLKmode
4597 /* If op0 is not BLKmode, but mode is, adjust_mode
4598 below would ICE. While it is likely a FE bug,
4599 try to be robust here. See PR43166. */
4601 || (mode == VOIDmode && GET_MODE (op0) != VOIDmode))
4603 gcc_assert (MEM_P (op0));
4604 op0 = adjust_address_nv (op0, mode, 0);
4614 inner_mode = GET_MODE (op0);
4616 if (mode == inner_mode)
4619 if (inner_mode == VOIDmode)
4621 if (TREE_CODE (exp) == SSA_NAME)
4622 inner_mode = TYPE_MODE (TREE_TYPE (exp));
4624 inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4625 if (mode == inner_mode)
4629 if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
4631 if (GET_MODE_UNIT_BITSIZE (mode)
4632 == GET_MODE_UNIT_BITSIZE (inner_mode))
4633 op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
4634 else if (GET_MODE_UNIT_BITSIZE (mode)
4635 < GET_MODE_UNIT_BITSIZE (inner_mode))
4636 op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
4638 op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
4640 else if (FLOAT_MODE_P (mode))
4642 gcc_assert (TREE_CODE (exp) != SSA_NAME);
4643 if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
4644 op0 = simplify_gen_unary (UNSIGNED_FLOAT, mode, op0, inner_mode);
4646 op0 = simplify_gen_unary (FLOAT, mode, op0, inner_mode);
4648 else if (FLOAT_MODE_P (inner_mode))
4651 op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
4653 op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
4655 else if (GET_MODE_UNIT_PRECISION (mode)
4656 == GET_MODE_UNIT_PRECISION (inner_mode))
4657 op0 = lowpart_subreg (mode, op0, inner_mode);
4658 else if (GET_MODE_UNIT_PRECISION (mode)
4659 < GET_MODE_UNIT_PRECISION (inner_mode))
4660 op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
4661 else if (UNARY_CLASS_P (exp)
4662 ? TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)))
4664 op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
4666 op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
4672 if (!is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
4674 tree newexp = fold_binary (MEM_REF, TREE_TYPE (exp),
4675 TREE_OPERAND (exp, 0),
4676 TREE_OPERAND (exp, 1));
4678 return expand_debug_expr (newexp);
4682 inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4683 op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4687 if (TREE_CODE (exp) == MEM_REF)
4689 if (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
4690 || (GET_CODE (op0) == PLUS
4691 && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR))
4692 /* (mem (debug_implicit_ptr)) might confuse aliasing.
4693 Instead just use get_inner_reference. */
4696 op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4698 if (!op1 || !poly_int_rtx_p (op1, &offset))
4701 op0 = plus_constant (inner_mode, op0, offset);
4704 as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
4706 op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
4708 if (op0 == NULL_RTX)
4711 op0 = gen_rtx_MEM (mode, op0);
4712 set_mem_attributes (op0, exp, 0);
4713 if (TREE_CODE (exp) == MEM_REF
4714 && !is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
4715 set_mem_expr (op0, NULL_TREE);
4716 set_mem_addr_space (op0, as);
4720 case TARGET_MEM_REF:
4721 if (TREE_CODE (TMR_BASE (exp)) == ADDR_EXPR
4722 && !DECL_RTL_SET_P (TREE_OPERAND (TMR_BASE (exp), 0)))
4725 op0 = expand_debug_expr
4726 (tree_mem_ref_addr (build_pointer_type (TREE_TYPE (exp)), exp));
4730 as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
4731 op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
4733 if (op0 == NULL_RTX)
4736 op0 = gen_rtx_MEM (mode, op0);
4738 set_mem_attributes (op0, exp, 0);
4739 set_mem_addr_space (op0, as);
4745 case ARRAY_RANGE_REF:
4750 case VIEW_CONVERT_EXPR:
4753 poly_int64 bitsize, bitpos;
4755 int reversep, volatilep = 0;
4757 = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
4758 &unsignedp, &reversep, &volatilep);
4761 if (known_eq (bitsize, 0))
4764 orig_op0 = op0 = expand_debug_expr (tem);
4771 machine_mode addrmode, offmode;
4776 op0 = XEXP (op0, 0);
4777 addrmode = GET_MODE (op0);
4778 if (addrmode == VOIDmode)
4781 op1 = expand_debug_expr (offset);
4785 offmode = GET_MODE (op1);
4786 if (offmode == VOIDmode)
4787 offmode = TYPE_MODE (TREE_TYPE (offset));
4789 if (addrmode != offmode)
4790 op1 = lowpart_subreg (addrmode, op1, offmode);
4792 /* Don't use offset_address here, we don't need a
4793 recognizable address, and we don't want to generate
4795 op0 = gen_rtx_MEM (mode, simplify_gen_binary (PLUS, addrmode,
4801 if (mode1 == VOIDmode)
4803 if (maybe_gt (bitsize, MAX_BITSIZE_MODE_ANY_INT))
4806 mode1 = smallest_int_mode_for_size (bitsize);
4808 poly_int64 bytepos = bits_to_bytes_round_down (bitpos);
4809 if (maybe_ne (bytepos, 0))
4811 op0 = adjust_address_nv (op0, mode1, bytepos);
4812 bitpos = num_trailing_bits (bitpos);
4814 else if (known_eq (bitpos, 0)
4815 && known_eq (bitsize, GET_MODE_BITSIZE (mode)))
4816 op0 = adjust_address_nv (op0, mode, 0);
4817 else if (GET_MODE (op0) != mode1)
4818 op0 = adjust_address_nv (op0, mode1, 0);
4820 op0 = copy_rtx (op0);
4821 if (op0 == orig_op0)
4822 op0 = shallow_copy_rtx (op0);
4823 if (TREE_CODE (tem) != SSA_NAME)
4824 set_mem_attributes (op0, exp, 0);
4827 if (known_eq (bitpos, 0) && mode == GET_MODE (op0))
4830 if (maybe_lt (bitpos, 0))
4833 if (GET_MODE (op0) == BLKmode || mode == BLKmode)
4837 if (multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
4838 && known_eq (bitsize, GET_MODE_BITSIZE (mode1)))
4840 machine_mode opmode = GET_MODE (op0);
4842 if (opmode == VOIDmode)
4843 opmode = TYPE_MODE (TREE_TYPE (tem));
4845 /* This condition may hold if we're expanding the address
4846 right past the end of an array that turned out not to
4847 be addressable (i.e., the address was only computed in
4848 debug stmts). The gen_subreg below would rightfully
4849 crash, and the address doesn't really exist, so just
4851 if (known_ge (bitpos, GET_MODE_BITSIZE (opmode)))
4854 if (multiple_p (bitpos, GET_MODE_BITSIZE (mode)))
4855 return simplify_gen_subreg (mode, op0, opmode, bytepos);
4858 return simplify_gen_ternary (SCALAR_INT_MODE_P (GET_MODE (op0))
4859 && TYPE_UNSIGNED (TREE_TYPE (exp))
4861 : ZERO_EXTRACT, mode,
4862 GET_MODE (op0) != VOIDmode
4864 : TYPE_MODE (TREE_TYPE (tem)),
4865 op0, gen_int_mode (bitsize, word_mode),
4866 gen_int_mode (bitpos, word_mode));
4871 return simplify_gen_unary (ABS, mode, op0, mode);
4874 return simplify_gen_unary (NEG, mode, op0, mode);
4877 return simplify_gen_unary (NOT, mode, op0, mode);
4880 return simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
4882 ? UNSIGNED_FLOAT : FLOAT, mode, op0,
4885 case FIX_TRUNC_EXPR:
4886 return simplify_gen_unary (unsignedp ? UNSIGNED_FIX : FIX, mode, op0,
4889 case POINTER_PLUS_EXPR:
4890 /* For the rare target where pointers are not the same size as
4891 size_t, we need to check for mis-matched modes and correct
4894 && is_a <scalar_int_mode> (GET_MODE (op0), &op0_mode)
4895 && is_a <scalar_int_mode> (GET_MODE (op1), &op1_mode)
4896 && op0_mode != op1_mode)
4898 if (GET_MODE_BITSIZE (op0_mode) < GET_MODE_BITSIZE (op1_mode)
4899 /* If OP0 is a partial mode, then we must truncate, even
4900 if it has the same bitsize as OP1 as GCC's
4901 representation of partial modes is opaque. */
4902 || (GET_MODE_CLASS (op0_mode) == MODE_PARTIAL_INT
4903 && (GET_MODE_BITSIZE (op0_mode)
4904 == GET_MODE_BITSIZE (op1_mode))))
4905 op1 = simplify_gen_unary (TRUNCATE, op0_mode, op1, op1_mode);
4907 /* We always sign-extend, regardless of the signedness of
4908 the operand, because the operand is always unsigned
4909 here even if the original C expression is signed. */
4910 op1 = simplify_gen_unary (SIGN_EXTEND, op0_mode, op1, op1_mode);
4914 return simplify_gen_binary (PLUS, mode, op0, op1);
4917 case POINTER_DIFF_EXPR:
4918 return simplify_gen_binary (MINUS, mode, op0, op1);
4921 return simplify_gen_binary (MULT, mode, op0, op1);
4924 case TRUNC_DIV_EXPR:
4925 case EXACT_DIV_EXPR:
4927 return simplify_gen_binary (UDIV, mode, op0, op1);
4929 return simplify_gen_binary (DIV, mode, op0, op1);
4931 case TRUNC_MOD_EXPR:
4932 return simplify_gen_binary (unsignedp ? UMOD : MOD, mode, op0, op1);
4934 case FLOOR_DIV_EXPR:
4936 return simplify_gen_binary (UDIV, mode, op0, op1);
4939 rtx div = simplify_gen_binary (DIV, mode, op0, op1);
4940 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
4941 rtx adj = floor_sdiv_adjust (mode, mod, op1);
4942 return simplify_gen_binary (PLUS, mode, div, adj);
4945 case FLOOR_MOD_EXPR:
4947 return simplify_gen_binary (UMOD, mode, op0, op1);
4950 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
4951 rtx adj = floor_sdiv_adjust (mode, mod, op1);
4952 adj = simplify_gen_unary (NEG, mode,
4953 simplify_gen_binary (MULT, mode, adj, op1),
4955 return simplify_gen_binary (PLUS, mode, mod, adj);
4961 rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
4962 rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
4963 rtx adj = ceil_udiv_adjust (mode, mod, op1);
4964 return simplify_gen_binary (PLUS, mode, div, adj);
4968 rtx div = simplify_gen_binary (DIV, mode, op0, op1);
4969 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
4970 rtx adj = ceil_sdiv_adjust (mode, mod, op1);
4971 return simplify_gen_binary (PLUS, mode, div, adj);
4977 rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
4978 rtx adj = ceil_udiv_adjust (mode, mod, op1);
4979 adj = simplify_gen_unary (NEG, mode,
4980 simplify_gen_binary (MULT, mode, adj, op1),
4982 return simplify_gen_binary (PLUS, mode, mod, adj);
4986 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
4987 rtx adj = ceil_sdiv_adjust (mode, mod, op1);
4988 adj = simplify_gen_unary (NEG, mode,
4989 simplify_gen_binary (MULT, mode, adj, op1),
4991 return simplify_gen_binary (PLUS, mode, mod, adj);
4994 case ROUND_DIV_EXPR:
4997 rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
4998 rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
4999 rtx adj = round_udiv_adjust (mode, mod, op1);
5000 return simplify_gen_binary (PLUS, mode, div, adj);
5004 rtx div = simplify_gen_binary (DIV, mode, op0, op1);
5005 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5006 rtx adj = round_sdiv_adjust (mode, mod, op1);
5007 return simplify_gen_binary (PLUS, mode, div, adj);
5010 case ROUND_MOD_EXPR:
5013 rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5014 rtx adj = round_udiv_adjust (mode, mod, op1);
5015 adj = simplify_gen_unary (NEG, mode,
5016 simplify_gen_binary (MULT, mode, adj, op1),
5018 return simplify_gen_binary (PLUS, mode, mod, adj);
5022 rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5023 rtx adj = round_sdiv_adjust (mode, mod, op1);
5024 adj = simplify_gen_unary (NEG, mode,
5025 simplify_gen_binary (MULT, mode, adj, op1),
5027 return simplify_gen_binary (PLUS, mode, mod, adj);
5031 return simplify_gen_binary (ASHIFT, mode, op0, op1);
5035 return simplify_gen_binary (LSHIFTRT, mode, op0, op1);
5037 return simplify_gen_binary (ASHIFTRT, mode, op0, op1);
5040 return simplify_gen_binary (ROTATE, mode, op0, op1);
5043 return simplify_gen_binary (ROTATERT, mode, op0, op1);
5046 return simplify_gen_binary (unsignedp ? UMIN : SMIN, mode, op0, op1);
5049 return simplify_gen_binary (unsignedp ? UMAX : SMAX, mode, op0, op1);
5052 case TRUTH_AND_EXPR:
5053 return simplify_gen_binary (AND, mode, op0, op1);
5057 return simplify_gen_binary (IOR, mode, op0, op1);
5060 case TRUTH_XOR_EXPR:
5061 return simplify_gen_binary (XOR, mode, op0, op1);
5063 case TRUTH_ANDIF_EXPR:
5064 return gen_rtx_IF_THEN_ELSE (mode, op0, op1, const0_rtx);
5066 case TRUTH_ORIF_EXPR:
5067 return gen_rtx_IF_THEN_ELSE (mode, op0, const_true_rtx, op1);
5069 case TRUTH_NOT_EXPR:
5070 return simplify_gen_relational (EQ, mode, inner_mode, op0, const0_rtx);
5073 return simplify_gen_relational (unsignedp ? LTU : LT, mode, inner_mode,
5077 return simplify_gen_relational (unsignedp ? LEU : LE, mode, inner_mode,
5081 return simplify_gen_relational (unsignedp ? GTU : GT, mode, inner_mode,
5085 return simplify_gen_relational (unsignedp ? GEU : GE, mode, inner_mode,
5089 return simplify_gen_relational (EQ, mode, inner_mode, op0, op1);
5092 return simplify_gen_relational (NE, mode, inner_mode, op0, op1);
5094 case UNORDERED_EXPR:
5095 return simplify_gen_relational (UNORDERED, mode, inner_mode, op0, op1);
5098 return simplify_gen_relational (ORDERED, mode, inner_mode, op0, op1);
5101 return simplify_gen_relational (UNLT, mode, inner_mode, op0, op1);
5104 return simplify_gen_relational (UNLE, mode, inner_mode, op0, op1);
5107 return simplify_gen_relational (UNGT, mode, inner_mode, op0, op1);
5110 return simplify_gen_relational (UNGE, mode, inner_mode, op0, op1);
5113 return simplify_gen_relational (UNEQ, mode, inner_mode, op0, op1);
5116 return simplify_gen_relational (LTGT, mode, inner_mode, op0, op1);
5119 return gen_rtx_IF_THEN_ELSE (mode, op0, op1, op2);
5122 gcc_assert (COMPLEX_MODE_P (mode));
5123 if (GET_MODE (op0) == VOIDmode)
5124 op0 = gen_rtx_CONST (GET_MODE_INNER (mode), op0);
5125 if (GET_MODE (op1) == VOIDmode)
5126 op1 = gen_rtx_CONST (GET_MODE_INNER (mode), op1);
5127 return gen_rtx_CONCAT (mode, op0, op1);
5130 if (GET_CODE (op0) == CONCAT)
5131 return gen_rtx_CONCAT (mode, XEXP (op0, 0),
5132 simplify_gen_unary (NEG, GET_MODE_INNER (mode),
5134 GET_MODE_INNER (mode)));
5137 scalar_mode imode = GET_MODE_INNER (mode);
5142 re = adjust_address_nv (op0, imode, 0);
5143 im = adjust_address_nv (op0, imode, GET_MODE_SIZE (imode));
5147 scalar_int_mode ifmode;
5148 scalar_int_mode ihmode;
5150 if (!int_mode_for_mode (mode).exists (&ifmode)
5151 || !int_mode_for_mode (imode).exists (&ihmode))
5153 halfsize = GEN_INT (GET_MODE_BITSIZE (ihmode));
5156 re = gen_rtx_SUBREG (ifmode, re, 0);
5157 re = gen_rtx_ZERO_EXTRACT (ihmode, re, halfsize, const0_rtx);
5158 if (imode != ihmode)
5159 re = gen_rtx_SUBREG (imode, re, 0);
5160 im = copy_rtx (op0);
5162 im = gen_rtx_SUBREG (ifmode, im, 0);
5163 im = gen_rtx_ZERO_EXTRACT (ihmode, im, halfsize, halfsize);
5164 if (imode != ihmode)
5165 im = gen_rtx_SUBREG (imode, im, 0);
5167 im = gen_rtx_NEG (imode, im);
5168 return gen_rtx_CONCAT (mode, re, im);
5172 op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
5173 if (!op0 || !MEM_P (op0))
5175 if ((TREE_CODE (TREE_OPERAND (exp, 0)) == VAR_DECL
5176 || TREE_CODE (TREE_OPERAND (exp, 0)) == PARM_DECL
5177 || TREE_CODE (TREE_OPERAND (exp, 0)) == RESULT_DECL)
5178 && (!TREE_ADDRESSABLE (TREE_OPERAND (exp, 0))
5179 || target_for_debug_bind (TREE_OPERAND (exp, 0))))
5180 return gen_rtx_DEBUG_IMPLICIT_PTR (mode, TREE_OPERAND (exp, 0));
5182 if (handled_component_p (TREE_OPERAND (exp, 0)))
5184 poly_int64 bitoffset, bitsize, maxsize, byteoffset;
5187 = get_ref_base_and_extent (TREE_OPERAND (exp, 0), &bitoffset,
5188 &bitsize, &maxsize, &reverse);
5190 || TREE_CODE (decl) == PARM_DECL
5191 || TREE_CODE (decl) == RESULT_DECL)
5192 && (!TREE_ADDRESSABLE (decl)
5193 || target_for_debug_bind (decl))
5194 && multiple_p (bitoffset, BITS_PER_UNIT, &byteoffset)
5195 && known_gt (bitsize, 0)
5196 && known_eq (bitsize, maxsize))
5198 rtx base = gen_rtx_DEBUG_IMPLICIT_PTR (mode, decl);
5199 return plus_constant (mode, base, byteoffset);
5203 if (TREE_CODE (TREE_OPERAND (exp, 0)) == MEM_REF
5204 && TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
5207 op0 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5210 && (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
5211 || (GET_CODE (op0) == PLUS
5212 && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR
5213 && CONST_INT_P (XEXP (op0, 1)))))
5215 op1 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5218 if (!op1 || !poly_int_rtx_p (op1, &offset))
5221 return plus_constant (mode, op0, offset);
5228 as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
5229 addr_mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (exp));
5230 op0 = convert_debug_memory_address (addr_mode, XEXP (op0, 0), as);
5236 unsigned HOST_WIDE_INT i, nelts;
5238 if (!VECTOR_CST_NELTS (exp).is_constant (&nelts))
5241 op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5243 for (i = 0; i < nelts; ++i)
5245 op1 = expand_debug_expr (VECTOR_CST_ELT (exp, i));
5248 XVECEXP (op0, 0, i) = op1;
5255 if (TREE_CLOBBER_P (exp))
5257 else if (TREE_CODE (TREE_TYPE (exp)) == VECTOR_TYPE)
5260 unsigned HOST_WIDE_INT nelts;
5263 if (!TYPE_VECTOR_SUBPARTS (TREE_TYPE (exp)).is_constant (&nelts))
5264 goto flag_unsupported;
5266 op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5268 FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), i, val)
5270 op1 = expand_debug_expr (val);
5273 XVECEXP (op0, 0, i) = op1;
5278 op1 = expand_debug_expr
5279 (build_zero_cst (TREE_TYPE (TREE_TYPE (exp))));
5284 for (; i < nelts; i++)
5285 XVECEXP (op0, 0, i) = op1;
5291 goto flag_unsupported;
5294 /* ??? Maybe handle some builtins? */
5299 gimple *g = get_gimple_for_ssa_name (exp);
5303 if (deep_ter_debug_map)
5305 tree *slot = deep_ter_debug_map->get (exp);
5310 t = gimple_assign_rhs_to_tree (g);
5311 op0 = expand_debug_expr (t);
5317 /* If this is a reference to an incoming value of
5318 parameter that is never used in the code or where the
5319 incoming value is never used in the code, use
5320 PARM_DECL's DECL_RTL if set. */
5321 if (SSA_NAME_IS_DEFAULT_DEF (exp)
5322 && SSA_NAME_VAR (exp)
5323 && TREE_CODE (SSA_NAME_VAR (exp)) == PARM_DECL
5324 && has_zero_uses (exp))
5326 op0 = expand_debug_parm_decl (SSA_NAME_VAR (exp));
5329 op0 = expand_debug_expr (SSA_NAME_VAR (exp));
5334 int part = var_to_partition (SA.map, exp);
5336 if (part == NO_PARTITION)
5339 gcc_assert (part >= 0 && (unsigned)part < SA.map->num_partitions);
5341 op0 = copy_rtx (SA.partition_to_pseudo[part]);
5349 /* Vector stuff. For most of the codes we don't have rtl codes. */
5350 case REALIGN_LOAD_EXPR:
5352 case VEC_PACK_FIX_TRUNC_EXPR:
5353 case VEC_PACK_FLOAT_EXPR:
5354 case VEC_PACK_SAT_EXPR:
5355 case VEC_PACK_TRUNC_EXPR:
5356 case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
5357 case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
5358 case VEC_UNPACK_FLOAT_HI_EXPR:
5359 case VEC_UNPACK_FLOAT_LO_EXPR:
5360 case VEC_UNPACK_HI_EXPR:
5361 case VEC_UNPACK_LO_EXPR:
5362 case VEC_WIDEN_MULT_HI_EXPR:
5363 case VEC_WIDEN_MULT_LO_EXPR:
5364 case VEC_WIDEN_MULT_EVEN_EXPR:
5365 case VEC_WIDEN_MULT_ODD_EXPR:
5366 case VEC_WIDEN_LSHIFT_HI_EXPR:
5367 case VEC_WIDEN_LSHIFT_LO_EXPR:
5369 case VEC_DUPLICATE_EXPR:
5370 case VEC_SERIES_EXPR:
5375 case ADDR_SPACE_CONVERT_EXPR:
5376 case FIXED_CONVERT_EXPR:
5378 case WITH_SIZE_EXPR:
5379 case BIT_INSERT_EXPR:
5383 if (SCALAR_INT_MODE_P (GET_MODE (op0))
5384 && SCALAR_INT_MODE_P (mode))
5387 = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5389 ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
5392 = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5394 ? ZERO_EXTEND : SIGN_EXTEND, mode, op1,
5396 op0 = simplify_gen_binary (MULT, mode, op0, op1);
5397 return simplify_gen_binary (PLUS, mode, op0, op2);
5401 case WIDEN_MULT_EXPR:
5402 case WIDEN_MULT_PLUS_EXPR:
5403 case WIDEN_MULT_MINUS_EXPR:
5404 if (SCALAR_INT_MODE_P (GET_MODE (op0))
5405 && SCALAR_INT_MODE_P (mode))
5407 inner_mode = GET_MODE (op0);
5408 if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
5409 op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5411 op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5412 if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 1))))
5413 op1 = simplify_gen_unary (ZERO_EXTEND, mode, op1, inner_mode);
5415 op1 = simplify_gen_unary (SIGN_EXTEND, mode, op1, inner_mode);
5416 op0 = simplify_gen_binary (MULT, mode, op0, op1);
5417 if (TREE_CODE (exp) == WIDEN_MULT_EXPR)
5419 else if (TREE_CODE (exp) == WIDEN_MULT_PLUS_EXPR)
5420 return simplify_gen_binary (PLUS, mode, op0, op2);
5422 return simplify_gen_binary (MINUS, mode, op2, op0);
5426 case MULT_HIGHPART_EXPR:
5427 /* ??? Similar to the above. */
5430 case WIDEN_SUM_EXPR:
5431 case WIDEN_LSHIFT_EXPR:
5432 if (SCALAR_INT_MODE_P (GET_MODE (op0))
5433 && SCALAR_INT_MODE_P (mode))
5436 = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5438 ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
5440 return simplify_gen_binary (TREE_CODE (exp) == WIDEN_LSHIFT_EXPR
5441 ? ASHIFT : PLUS, mode, op0, op1);
5456 /* Return an RTX equivalent to the source bind value of the tree expression
5460 expand_debug_source_expr (tree exp)
5463 machine_mode mode = VOIDmode, inner_mode;
5465 switch (TREE_CODE (exp))
5468 if (DECL_ABSTRACT_ORIGIN (exp))
5469 return expand_debug_source_expr (DECL_ABSTRACT_ORIGIN (exp));
5473 mode = DECL_MODE (exp);
5474 op0 = expand_debug_parm_decl (exp);
5477 /* See if this isn't an argument that has been completely
5479 if (!DECL_RTL_SET_P (exp)
5480 && !DECL_INCOMING_RTL (exp)
5481 && DECL_ABSTRACT_ORIGIN (current_function_decl))
5483 tree aexp = DECL_ORIGIN (exp);
5484 if (DECL_CONTEXT (aexp)
5485 == DECL_ABSTRACT_ORIGIN (current_function_decl))
5487 vec<tree, va_gc> **debug_args;
5490 debug_args = decl_debug_args_lookup (current_function_decl);
5491 if (debug_args != NULL)
5493 for (ix = 0; vec_safe_iterate (*debug_args, ix, &ddecl);
5496 return gen_rtx_DEBUG_PARAMETER_REF (mode, aexp);
5506 if (op0 == NULL_RTX)
5509 inner_mode = GET_MODE (op0);
5510 if (mode == inner_mode)
5513 if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
5515 if (GET_MODE_UNIT_BITSIZE (mode)
5516 == GET_MODE_UNIT_BITSIZE (inner_mode))
5517 op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
5518 else if (GET_MODE_UNIT_BITSIZE (mode)
5519 < GET_MODE_UNIT_BITSIZE (inner_mode))
5520 op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
5522 op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
5524 else if (FLOAT_MODE_P (mode))
5526 else if (FLOAT_MODE_P (inner_mode))
5528 if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5529 op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
5531 op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
5533 else if (GET_MODE_UNIT_PRECISION (mode)
5534 == GET_MODE_UNIT_PRECISION (inner_mode))
5535 op0 = lowpart_subreg (mode, op0, inner_mode);
5536 else if (GET_MODE_UNIT_PRECISION (mode)
5537 < GET_MODE_UNIT_PRECISION (inner_mode))
5538 op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
5539 else if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5540 op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5542 op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5547 /* Ensure INSN_VAR_LOCATION_LOC (insn) doesn't have unbound complexity.
5548 Allow 4 levels of rtl nesting for most rtl codes, and if we see anything
5549 deeper than that, create DEBUG_EXPRs and emit DEBUG_INSNs before INSN. */
5552 avoid_complex_debug_insns (rtx_insn *insn, rtx *exp_p, int depth)
5556 if (exp == NULL_RTX)
5559 if ((OBJECT_P (exp) && !MEM_P (exp)) || GET_CODE (exp) == CLOBBER)
5564 /* Create DEBUG_EXPR (and DEBUG_EXPR_DECL). */
5565 rtx dval = make_debug_expr_from_rtl (exp);
5567 /* Emit a debug bind insn before INSN. */
5568 rtx bind = gen_rtx_VAR_LOCATION (GET_MODE (exp),
5569 DEBUG_EXPR_TREE_DECL (dval), exp,
5570 VAR_INIT_STATUS_INITIALIZED);
5572 emit_debug_insn_before (bind, insn);
5577 const char *format_ptr = GET_RTX_FORMAT (GET_CODE (exp));
5579 for (i = 0; i < GET_RTX_LENGTH (GET_CODE (exp)); i++)
5580 switch (*format_ptr++)
5583 avoid_complex_debug_insns (insn, &XEXP (exp, i), depth + 1);
5588 for (j = 0; j < XVECLEN (exp, i); j++)
5589 avoid_complex_debug_insns (insn, &XVECEXP (exp, i, j), depth + 1);
5597 /* Expand the _LOCs in debug insns. We run this after expanding all
5598 regular insns, so that any variables referenced in the function
5599 will have their DECL_RTLs set. */
5602 expand_debug_locations (void)
5605 rtx_insn *last = get_last_insn ();
5606 int save_strict_alias = flag_strict_aliasing;
5608 /* New alias sets while setting up memory attributes cause
5609 -fcompare-debug failures, even though it doesn't bring about any
5611 flag_strict_aliasing = 0;
5613 for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
5614 if (DEBUG_BIND_INSN_P (insn))
5616 tree value = (tree)INSN_VAR_LOCATION_LOC (insn);
5618 rtx_insn *prev_insn, *insn2;
5621 if (value == NULL_TREE)
5625 if (INSN_VAR_LOCATION_STATUS (insn)
5626 == VAR_INIT_STATUS_UNINITIALIZED)
5627 val = expand_debug_source_expr (value);
5628 /* The avoid_deep_ter_for_debug function inserts
5629 debug bind stmts after SSA_NAME definition, with the
5630 SSA_NAME as the whole bind location. Disable temporarily
5631 expansion of that SSA_NAME into the DEBUG_EXPR_DECL
5632 being defined in this DEBUG_INSN. */
5633 else if (deep_ter_debug_map && TREE_CODE (value) == SSA_NAME)
5635 tree *slot = deep_ter_debug_map->get (value);
5638 if (*slot == INSN_VAR_LOCATION_DECL (insn))
5643 val = expand_debug_expr (value);
5645 *slot = INSN_VAR_LOCATION_DECL (insn);
5648 val = expand_debug_expr (value);
5649 gcc_assert (last == get_last_insn ());
5653 val = gen_rtx_UNKNOWN_VAR_LOC ();
5656 mode = GET_MODE (INSN_VAR_LOCATION (insn));
5658 gcc_assert (mode == GET_MODE (val)
5659 || (GET_MODE (val) == VOIDmode
5660 && (CONST_SCALAR_INT_P (val)
5661 || GET_CODE (val) == CONST_FIXED
5662 || GET_CODE (val) == LABEL_REF)));
5665 INSN_VAR_LOCATION_LOC (insn) = val;
5666 prev_insn = PREV_INSN (insn);
5667 for (insn2 = insn; insn2 != prev_insn; insn2 = PREV_INSN (insn2))
5668 avoid_complex_debug_insns (insn2, &INSN_VAR_LOCATION_LOC (insn2), 0);
5671 flag_strict_aliasing = save_strict_alias;
5674 /* Performs swapping operands of commutative operations to expand
5675 the expensive one first. */
5678 reorder_operands (basic_block bb)
5680 unsigned int *lattice; /* Hold cost of each statement. */
5681 unsigned int i = 0, n = 0;
5682 gimple_stmt_iterator gsi;
5688 use_operand_p use_p;
5689 gimple *def0, *def1;
5691 /* Compute cost of each statement using estimate_num_insns. */
5692 stmts = bb_seq (bb);
5693 for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
5695 stmt = gsi_stmt (gsi);
5696 if (!is_gimple_debug (stmt))
5697 gimple_set_uid (stmt, n++);
5699 lattice = XNEWVEC (unsigned int, n);
5700 for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
5703 stmt = gsi_stmt (gsi);
5704 if (is_gimple_debug (stmt))
5706 cost = estimate_num_insns (stmt, &eni_size_weights);
5708 FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
5710 tree use = USE_FROM_PTR (use_p);
5712 if (TREE_CODE (use) != SSA_NAME)
5714 def_stmt = get_gimple_for_ssa_name (use);
5717 lattice[i] += lattice[gimple_uid (def_stmt)];
5720 if (!is_gimple_assign (stmt)
5721 || !commutative_tree_code (gimple_assign_rhs_code (stmt)))
5723 op0 = gimple_op (stmt, 1);
5724 op1 = gimple_op (stmt, 2);
5725 if (TREE_CODE (op0) != SSA_NAME
5726 || TREE_CODE (op1) != SSA_NAME)
5728 /* Swap operands if the second one is more expensive. */
5729 def0 = get_gimple_for_ssa_name (op0);
5730 def1 = get_gimple_for_ssa_name (op1);
5734 if (!def0 || lattice[gimple_uid (def1)] > lattice[gimple_uid (def0)])
5738 if (dump_file && (dump_flags & TDF_DETAILS))
5740 fprintf (dump_file, "Swap operands in stmt:\n");
5741 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
5742 fprintf (dump_file, "Cost left opnd=%d, right opnd=%d\n",
5743 def0 ? lattice[gimple_uid (def0)] : 0,
5744 lattice[gimple_uid (def1)]);
5746 swap_ssa_operands (stmt, gimple_assign_rhs1_ptr (stmt),
5747 gimple_assign_rhs2_ptr (stmt));
5753 /* Expand basic block BB from GIMPLE trees to RTL. */
5756 expand_gimple_basic_block (basic_block bb, bool disable_tail_calls)
5758 gimple_stmt_iterator gsi;
5760 gimple *stmt = NULL;
5761 rtx_note *note = NULL;
5765 bool nondebug_stmt_seen = false;
5768 fprintf (dump_file, "\n;; Generating RTL for gimple basic block %d\n",
5771 /* Note that since we are now transitioning from GIMPLE to RTL, we
5772 cannot use the gsi_*_bb() routines because they expect the basic
5773 block to be in GIMPLE, instead of RTL. Therefore, we need to
5774 access the BB sequence directly. */
5776 reorder_operands (bb);
5777 stmts = bb_seq (bb);
5778 bb->il.gimple.seq = NULL;
5779 bb->il.gimple.phi_nodes = NULL;
5780 rtl_profile_for_bb (bb);
5781 init_rtl_bb_info (bb);
5782 bb->flags |= BB_RTL;
5784 /* Remove the RETURN_EXPR if we may fall though to the exit
5786 gsi = gsi_last (stmts);
5787 if (!gsi_end_p (gsi)
5788 && gimple_code (gsi_stmt (gsi)) == GIMPLE_RETURN)
5790 greturn *ret_stmt = as_a <greturn *> (gsi_stmt (gsi));
5792 gcc_assert (single_succ_p (bb));
5793 gcc_assert (single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun));
5795 if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
5796 && !gimple_return_retval (ret_stmt))
5798 gsi_remove (&gsi, false);
5799 single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
5803 gsi = gsi_start (stmts);
5804 if (!gsi_end_p (gsi))
5806 stmt = gsi_stmt (gsi);
5807 if (gimple_code (stmt) != GIMPLE_LABEL)
5811 rtx_code_label **elt = lab_rtx_for_bb->get (bb);
5815 gcc_checking_assert (!note);
5816 last = get_last_insn ();
5820 expand_gimple_stmt (stmt);
5827 BB_HEAD (bb) = NEXT_INSN (last);
5828 if (NOTE_P (BB_HEAD (bb)))
5829 BB_HEAD (bb) = NEXT_INSN (BB_HEAD (bb));
5830 gcc_assert (LABEL_P (BB_HEAD (bb)));
5831 note = emit_note_after (NOTE_INSN_BASIC_BLOCK, BB_HEAD (bb));
5833 maybe_dump_rtl_for_gimple_stmt (stmt, last);
5836 BB_HEAD (bb) = note = emit_note (NOTE_INSN_BASIC_BLOCK);
5839 NOTE_BASIC_BLOCK (note) = bb;
5841 for (; !gsi_end_p (gsi); gsi_next (&gsi))
5845 stmt = gsi_stmt (gsi);
5846 if (!is_gimple_debug (stmt))
5847 nondebug_stmt_seen = true;
5849 /* If this statement is a non-debug one, and we generate debug
5850 insns, then this one might be the last real use of a TERed
5851 SSA_NAME, but where there are still some debug uses further
5852 down. Expanding the current SSA name in such further debug
5853 uses by their RHS might lead to wrong debug info, as coalescing
5854 might make the operands of such RHS be placed into the same
5855 pseudo as something else. Like so:
5856 a_1 = a_0 + 1; // Assume a_1 is TERed and a_0 is dead
5860 As a_0 and a_2 don't overlap in lifetime, assume they are coalesced.
5861 If we now would expand a_1 by it's RHS (a_0 + 1) in the debug use,
5862 the write to a_2 would actually have clobbered the place which
5865 So, instead of that, we recognize the situation, and generate
5866 debug temporaries at the last real use of TERed SSA names:
5873 if (MAY_HAVE_DEBUG_BIND_INSNS
5875 && !is_gimple_debug (stmt))
5881 location_t sloc = curr_insn_location ();
5883 /* Look for SSA names that have their last use here (TERed
5884 names always have only one real use). */
5885 FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
5886 if ((def = get_gimple_for_ssa_name (op)))
5888 imm_use_iterator imm_iter;
5889 use_operand_p use_p;
5890 bool have_debug_uses = false;
5892 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, op)
5894 if (gimple_debug_bind_p (USE_STMT (use_p)))
5896 have_debug_uses = true;
5901 if (have_debug_uses)
5903 /* OP is a TERed SSA name, with DEF its defining
5904 statement, and where OP is used in further debug
5905 instructions. Generate a debug temporary, and
5906 replace all uses of OP in debug insns with that
5909 tree value = gimple_assign_rhs_to_tree (def);
5910 tree vexpr = build_debug_expr_decl (TREE_TYPE (value));
5914 set_curr_insn_location (gimple_location (def));
5917 mode = DECL_MODE (value);
5919 mode = TYPE_MODE (TREE_TYPE (value));
5920 /* FIXME: Is setting the mode really necessary? */
5921 SET_DECL_MODE (vexpr, mode);
5923 val = gen_rtx_VAR_LOCATION
5924 (mode, vexpr, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
5926 emit_debug_insn (val);
5928 FOR_EACH_IMM_USE_STMT (debugstmt, imm_iter, op)
5930 if (!gimple_debug_bind_p (debugstmt))
5933 FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
5934 SET_USE (use_p, vexpr);
5936 update_stmt (debugstmt);
5940 set_curr_insn_location (sloc);
5943 currently_expanding_gimple_stmt = stmt;
5945 /* Expand this statement, then evaluate the resulting RTL and
5946 fixup the CFG accordingly. */
5947 if (gimple_code (stmt) == GIMPLE_COND)
5949 new_bb = expand_gimple_cond (bb, as_a <gcond *> (stmt));
5952 currently_expanding_gimple_stmt = NULL;
5956 else if (is_gimple_debug (stmt))
5958 location_t sloc = curr_insn_location ();
5959 gimple_stmt_iterator nsi = gsi;
5964 tree value = NULL_TREE;
5968 if (!gimple_debug_nonbind_marker_p (stmt))
5970 if (gimple_debug_bind_p (stmt))
5972 var = gimple_debug_bind_get_var (stmt);
5974 if (TREE_CODE (var) != DEBUG_EXPR_DECL
5975 && TREE_CODE (var) != LABEL_DECL
5976 && !target_for_debug_bind (var))
5977 goto delink_debug_stmt;
5979 if (DECL_P (var) && !VECTOR_TYPE_P (TREE_TYPE (var)))
5980 mode = DECL_MODE (var);
5982 mode = TYPE_MODE (TREE_TYPE (var));
5984 if (gimple_debug_bind_has_value_p (stmt))
5985 value = gimple_debug_bind_get_value (stmt);
5987 val = gen_rtx_VAR_LOCATION
5988 (mode, var, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
5990 else if (gimple_debug_source_bind_p (stmt))
5992 var = gimple_debug_source_bind_get_var (stmt);
5994 value = gimple_debug_source_bind_get_value (stmt);
5996 if (!VECTOR_TYPE_P (TREE_TYPE (var)))
5997 mode = DECL_MODE (var);
5999 mode = TYPE_MODE (TREE_TYPE (var));
6001 val = gen_rtx_VAR_LOCATION (mode, var, (rtx)value,
6002 VAR_INIT_STATUS_UNINITIALIZED);
6007 /* If this function was first compiled with markers
6008 enabled, but they're now disable (e.g. LTO), drop
6009 them on the floor. */
6010 else if (gimple_debug_nonbind_marker_p (stmt)
6011 && !MAY_HAVE_DEBUG_MARKER_INSNS)
6012 goto delink_debug_stmt;
6013 else if (gimple_debug_begin_stmt_p (stmt))
6014 val = GEN_RTX_DEBUG_MARKER_BEGIN_STMT_PAT ();
6015 else if (gimple_debug_inline_entry_p (stmt))
6016 val = GEN_RTX_DEBUG_MARKER_INLINE_ENTRY_PAT ();
6020 last = get_last_insn ();
6022 set_curr_insn_location (gimple_location (stmt));
6024 emit_debug_insn (val);
6026 if (dump_file && (dump_flags & TDF_DETAILS))
6028 /* We can't dump the insn with a TREE where an RTX
6030 if (GET_CODE (val) == VAR_LOCATION)
6032 gcc_checking_assert (PAT_VAR_LOCATION_LOC (val) == (rtx)value);
6033 PAT_VAR_LOCATION_LOC (val) = const0_rtx;
6035 maybe_dump_rtl_for_gimple_stmt (stmt, last);
6036 if (GET_CODE (val) == VAR_LOCATION)
6037 PAT_VAR_LOCATION_LOC (val) = (rtx)value;
6041 /* In order not to generate too many debug temporaries,
6042 we delink all uses of debug statements we already expanded.
6043 Therefore debug statements between definition and real
6044 use of TERed SSA names will continue to use the SSA name,
6045 and not be replaced with debug temps. */
6046 delink_stmt_imm_use (stmt);
6050 if (gsi_end_p (nsi))
6052 stmt = gsi_stmt (nsi);
6053 if (!is_gimple_debug (stmt))
6057 set_curr_insn_location (sloc);
6061 gcall *call_stmt = dyn_cast <gcall *> (stmt);
6063 && gimple_call_tail_p (call_stmt)
6064 && disable_tail_calls)
6065 gimple_call_set_tail (call_stmt, false);
6067 if (call_stmt && gimple_call_tail_p (call_stmt))
6070 new_bb = expand_gimple_tailcall (bb, call_stmt, &can_fallthru);
6077 currently_expanding_gimple_stmt = NULL;
6084 def_operand_p def_p;
6085 def_p = SINGLE_SSA_DEF_OPERAND (stmt, SSA_OP_DEF);
6089 /* Ignore this stmt if it is in the list of
6090 replaceable expressions. */
6092 && bitmap_bit_p (SA.values,
6093 SSA_NAME_VERSION (DEF_FROM_PTR (def_p))))
6096 last = expand_gimple_stmt (stmt);
6097 maybe_dump_rtl_for_gimple_stmt (stmt, last);
6102 currently_expanding_gimple_stmt = NULL;
6104 /* Expand implicit goto and convert goto_locus. */
6105 FOR_EACH_EDGE (e, ei, bb->succs)
6107 if (e->goto_locus != UNKNOWN_LOCATION || !nondebug_stmt_seen)
6108 set_curr_insn_location (e->goto_locus);
6109 if ((e->flags & EDGE_FALLTHRU) && e->dest != bb->next_bb)
6111 emit_jump (label_rtx_for_bb (e->dest));
6112 e->flags &= ~EDGE_FALLTHRU;
6116 /* Expanded RTL can create a jump in the last instruction of block.
6117 This later might be assumed to be a jump to successor and break edge insertion.
6118 We need to insert dummy move to prevent this. PR41440. */
6119 if (single_succ_p (bb)
6120 && (single_succ_edge (bb)->flags & EDGE_FALLTHRU)
6121 && (last = get_last_insn ())
6123 || (DEBUG_INSN_P (last)
6124 && JUMP_P (prev_nondebug_insn (last)))))
6126 rtx dummy = gen_reg_rtx (SImode);
6127 emit_insn_after_noloc (gen_move_insn (dummy, dummy), last, NULL);
6130 do_pending_stack_adjust ();
6132 /* Find the block tail. The last insn in the block is the insn
6133 before a barrier and/or table jump insn. */
6134 last = get_last_insn ();
6135 if (BARRIER_P (last))
6136 last = PREV_INSN (last);
6137 if (JUMP_TABLE_DATA_P (last))
6138 last = PREV_INSN (PREV_INSN (last));
6139 if (BARRIER_P (last))
6140 last = PREV_INSN (last);
6143 update_bb_for_insn (bb);
6149 /* Create a basic block for initialization code. */
6152 construct_init_block (void)
6154 basic_block init_block, first_block;
6158 /* Multiple entry points not supported yet. */
6159 gcc_assert (EDGE_COUNT (ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs) == 1);
6160 init_rtl_bb_info (ENTRY_BLOCK_PTR_FOR_FN (cfun));
6161 init_rtl_bb_info (EXIT_BLOCK_PTR_FOR_FN (cfun));
6162 ENTRY_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6163 EXIT_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6165 e = EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun), 0);
6167 /* When entry edge points to first basic block, we don't need jump,
6168 otherwise we have to jump into proper target. */
6169 if (e && e->dest != ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb)
6171 tree label = gimple_block_label (e->dest);
6173 emit_jump (jump_target_rtx (label));
6177 flags = EDGE_FALLTHRU;
6179 init_block = create_basic_block (NEXT_INSN (get_insns ()),
6181 ENTRY_BLOCK_PTR_FOR_FN (cfun));
6182 init_block->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
6183 add_bb_to_loop (init_block, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6186 first_block = e->dest;
6187 redirect_edge_succ (e, init_block);
6188 make_single_succ_edge (init_block, first_block, flags);
6191 make_single_succ_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6194 update_bb_for_insn (init_block);
6198 /* For each lexical block, set BLOCK_NUMBER to the depth at which it is
6199 found in the block tree. */
6202 set_block_levels (tree block, int level)
6206 BLOCK_NUMBER (block) = level;
6207 set_block_levels (BLOCK_SUBBLOCKS (block), level + 1);
6208 block = BLOCK_CHAIN (block);
6212 /* Create a block containing landing pads and similar stuff. */
6215 construct_exit_block (void)
6217 rtx_insn *head = get_last_insn ();
6219 basic_block exit_block;
6223 basic_block prev_bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
6224 rtx_insn *orig_end = BB_END (prev_bb);
6226 rtl_profile_for_bb (EXIT_BLOCK_PTR_FOR_FN (cfun));
6228 /* Make sure the locus is set to the end of the function, so that
6229 epilogue line numbers and warnings are set properly. */
6230 if (LOCATION_LOCUS (cfun->function_end_locus) != UNKNOWN_LOCATION)
6231 input_location = cfun->function_end_locus;
6233 /* Generate rtl for function exit. */
6234 expand_function_end ();
6236 end = get_last_insn ();
6239 /* While emitting the function end we could move end of the last basic
6241 BB_END (prev_bb) = orig_end;
6242 while (NEXT_INSN (head) && NOTE_P (NEXT_INSN (head)))
6243 head = NEXT_INSN (head);
6244 /* But make sure exit_block starts with RETURN_LABEL, otherwise the
6245 bb count counting will be confused. Any instructions before that
6246 label are emitted for the case where PREV_BB falls through into the
6247 exit block, so append those instructions to prev_bb in that case. */
6248 if (NEXT_INSN (head) != return_label)
6250 while (NEXT_INSN (head) != return_label)
6252 if (!NOTE_P (NEXT_INSN (head)))
6253 BB_END (prev_bb) = NEXT_INSN (head);
6254 head = NEXT_INSN (head);
6257 exit_block = create_basic_block (NEXT_INSN (head), end, prev_bb);
6258 exit_block->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count;
6259 add_bb_to_loop (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6262 while (ix < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
6264 e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), ix);
6265 if (!(e->flags & EDGE_ABNORMAL))
6266 redirect_edge_succ (e, exit_block);
6271 e = make_single_succ_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6273 FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
6276 exit_block->count -= e2->count ();
6278 update_bb_for_insn (exit_block);
6281 /* Helper function for discover_nonconstant_array_refs.
6282 Look for ARRAY_REF nodes with non-constant indexes and mark them
6286 discover_nonconstant_array_refs_r (tree * tp, int *walk_subtrees,
6290 bitmap forced_stack_vars = (bitmap)((walk_stmt_info *)data)->info;
6292 if (IS_TYPE_OR_DECL_P (t))
6294 else if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6296 while (((TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6297 && is_gimple_min_invariant (TREE_OPERAND (t, 1))
6298 && (!TREE_OPERAND (t, 2)
6299 || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6300 || (TREE_CODE (t) == COMPONENT_REF
6301 && (!TREE_OPERAND (t,2)
6302 || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6303 || TREE_CODE (t) == BIT_FIELD_REF
6304 || TREE_CODE (t) == REALPART_EXPR
6305 || TREE_CODE (t) == IMAGPART_EXPR
6306 || TREE_CODE (t) == VIEW_CONVERT_EXPR
6307 || CONVERT_EXPR_P (t))
6308 t = TREE_OPERAND (t, 0);
6310 if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6312 t = get_base_address (t);
6314 && DECL_MODE (t) != BLKmode
6315 && !TREE_ADDRESSABLE (t))
6316 bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6321 /* References of size POLY_INT_CST to a fixed-size object must go
6322 through memory. It's more efficient to force that here than
6323 to create temporary slots on the fly.
6324 RTL expansion expectes TARGET_MEM_REF to always address actual memory. */
6325 else if (TREE_CODE (t) == TARGET_MEM_REF
6326 || (TREE_CODE (t) == MEM_REF
6327 && TYPE_SIZE (TREE_TYPE (t))
6328 && POLY_INT_CST_P (TYPE_SIZE (TREE_TYPE (t)))))
6330 tree base = get_base_address (t);
6333 && !TREE_ADDRESSABLE (base)
6334 && DECL_MODE (base) != BLKmode
6335 && GET_MODE_SIZE (DECL_MODE (base)).is_constant ())
6336 bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6343 /* If there's a chance to get a pseudo for t then if it would be of float mode
6344 and the actual access is via an integer mode (lowered memcpy or similar
6345 access) then avoid the register expansion if the mode likely is not storage
6346 suitable for raw bits processing (like XFmode on i?86). */
6349 avoid_type_punning_on_regs (tree t, bitmap forced_stack_vars)
6351 machine_mode access_mode = TYPE_MODE (TREE_TYPE (t));
6352 if (access_mode != BLKmode
6353 && !SCALAR_INT_MODE_P (access_mode))
6355 tree base = get_base_address (t);
6357 && !TREE_ADDRESSABLE (base)
6358 && FLOAT_MODE_P (DECL_MODE (base))
6359 && maybe_lt (GET_MODE_PRECISION (DECL_MODE (base)),
6360 GET_MODE_BITSIZE (GET_MODE_INNER (DECL_MODE (base))))
6361 /* Double check in the expensive way we really would get a pseudo. */
6362 && use_register_for_decl (base))
6363 bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6366 /* RTL expansion is not able to compile array references with variable
6367 offsets for arrays stored in single register. Discover such
6368 expressions and mark variables as addressable to avoid this
6372 discover_nonconstant_array_refs (bitmap forced_stack_vars)
6375 gimple_stmt_iterator gsi;
6377 walk_stmt_info wi = {};
6378 wi.info = forced_stack_vars;
6379 FOR_EACH_BB_FN (bb, cfun)
6380 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
6382 gimple *stmt = gsi_stmt (gsi);
6383 if (!is_gimple_debug (stmt))
6385 walk_gimple_op (stmt, discover_nonconstant_array_refs_r, &wi);
6386 gcall *call = dyn_cast <gcall *> (stmt);
6387 if (call && gimple_call_internal_p (call))
6389 tree cand = NULL_TREE;
6390 switch (gimple_call_internal_fn (call))
6392 case IFN_LOAD_LANES:
6393 /* The source must be a MEM. */
6394 cand = gimple_call_arg (call, 0);
6396 case IFN_STORE_LANES:
6397 /* The destination must be a MEM. */
6398 cand = gimple_call_lhs (call);
6404 cand = get_base_address (cand);
6407 && use_register_for_decl (cand))
6408 bitmap_set_bit (forced_stack_vars, DECL_UID (cand));
6410 if (gimple_vdef (stmt))
6412 tree t = gimple_get_lhs (stmt);
6413 if (t && REFERENCE_CLASS_P (t))
6414 avoid_type_punning_on_regs (t, forced_stack_vars);
6420 /* This function sets crtl->args.internal_arg_pointer to a virtual
6421 register if DRAP is needed. Local register allocator will replace
6422 virtual_incoming_args_rtx with the virtual register. */
6425 expand_stack_alignment (void)
6428 unsigned int preferred_stack_boundary;
6430 if (! SUPPORTS_STACK_ALIGNMENT)
6433 if (cfun->calls_alloca
6434 || cfun->has_nonlocal_label
6435 || crtl->has_nonlocal_goto)
6436 crtl->need_drap = true;
6438 /* Call update_stack_boundary here again to update incoming stack
6439 boundary. It may set incoming stack alignment to a different
6440 value after RTL expansion. TARGET_FUNCTION_OK_FOR_SIBCALL may
6441 use the minimum incoming stack alignment to check if it is OK
6442 to perform sibcall optimization since sibcall optimization will
6443 only align the outgoing stack to incoming stack boundary. */
6444 if (targetm.calls.update_stack_boundary)
6445 targetm.calls.update_stack_boundary ();
6447 /* The incoming stack frame has to be aligned at least at
6448 parm_stack_boundary. */
6449 gcc_assert (crtl->parm_stack_boundary <= INCOMING_STACK_BOUNDARY);
6451 /* Update crtl->stack_alignment_estimated and use it later to align
6452 stack. We check PREFERRED_STACK_BOUNDARY if there may be non-call
6453 exceptions since callgraph doesn't collect incoming stack alignment
6455 if (cfun->can_throw_non_call_exceptions
6456 && PREFERRED_STACK_BOUNDARY > crtl->preferred_stack_boundary)
6457 preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
6459 preferred_stack_boundary = crtl->preferred_stack_boundary;
6460 if (preferred_stack_boundary > crtl->stack_alignment_estimated)
6461 crtl->stack_alignment_estimated = preferred_stack_boundary;
6462 if (preferred_stack_boundary > crtl->stack_alignment_needed)
6463 crtl->stack_alignment_needed = preferred_stack_boundary;
6465 gcc_assert (crtl->stack_alignment_needed
6466 <= crtl->stack_alignment_estimated);
6468 crtl->stack_realign_needed
6469 = INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
6470 crtl->stack_realign_tried = crtl->stack_realign_needed;
6472 crtl->stack_realign_processed = true;
6474 /* Target has to redefine TARGET_GET_DRAP_RTX to support stack
6476 gcc_assert (targetm.calls.get_drap_rtx != NULL);
6477 drap_rtx = targetm.calls.get_drap_rtx ();
6479 /* stack_realign_drap and drap_rtx must match. */
6480 gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
6482 /* Do nothing if NULL is returned, which means DRAP is not needed. */
6483 if (drap_rtx != NULL)
6485 crtl->args.internal_arg_pointer = drap_rtx;
6487 /* Call fixup_tail_calls to clean up REG_EQUIV note if DRAP is
6489 fixup_tail_calls ();
6495 expand_main_function (void)
6497 #if (defined(INVOKE__main) \
6498 || (!defined(HAS_INIT_SECTION) \
6499 && !defined(INIT_SECTION_ASM_OP) \
6500 && !defined(INIT_ARRAY_SECTION_ASM_OP)))
6501 emit_library_call (init_one_libfunc (NAME__MAIN), LCT_NORMAL, VOIDmode);
6506 /* Expand code to initialize the stack_protect_guard. This is invoked at
6507 the beginning of a function to be protected. */
6510 stack_protect_prologue (void)
6512 tree guard_decl = targetm.stack_protect_guard ();
6515 crtl->stack_protect_guard_decl = guard_decl;
6516 x = expand_normal (crtl->stack_protect_guard);
6518 if (targetm.have_stack_protect_combined_set () && guard_decl)
6520 gcc_assert (DECL_P (guard_decl));
6521 y = DECL_RTL (guard_decl);
6523 /* Allow the target to compute address of Y and copy it to X without
6524 leaking Y into a register. This combined address + copy pattern
6525 allows the target to prevent spilling of any intermediate results by
6526 splitting it after register allocator. */
6527 if (rtx_insn *insn = targetm.gen_stack_protect_combined_set (x, y))
6535 y = expand_normal (guard_decl);
6539 /* Allow the target to copy from Y to X without leaking Y into a
6541 if (targetm.have_stack_protect_set ())
6542 if (rtx_insn *insn = targetm.gen_stack_protect_set (x, y))
6548 /* Otherwise do a straight move. */
6549 emit_move_insn (x, y);
6552 /* Translate the intermediate representation contained in the CFG
6553 from GIMPLE trees to RTL.
6555 We do conversion per basic block and preserve/update the tree CFG.
6556 This implies we have to do some magic as the CFG can simultaneously
6557 consist of basic blocks containing RTL and GIMPLE trees. This can
6558 confuse the CFG hooks, so be careful to not manipulate CFG during
6563 const pass_data pass_data_expand =
6565 RTL_PASS, /* type */
6566 "expand", /* name */
6567 OPTGROUP_NONE, /* optinfo_flags */
6568 TV_EXPAND, /* tv_id */
6569 ( PROP_ssa | PROP_gimple_leh | PROP_cfg
6572 | PROP_gimple_lva), /* properties_required */
6573 PROP_rtl, /* properties_provided */
6574 ( PROP_ssa | PROP_gimple ), /* properties_destroyed */
6575 0, /* todo_flags_start */
6576 0, /* todo_flags_finish */
6579 class pass_expand : public rtl_opt_pass
6582 pass_expand (gcc::context *ctxt)
6583 : rtl_opt_pass (pass_data_expand, ctxt)
6586 /* opt_pass methods: */
6587 unsigned int execute (function *) final override;
6589 }; // class pass_expand
6592 pass_expand::execute (function *fun)
6594 basic_block bb, init_block;
6597 rtx_insn *var_seq, *var_ret_seq;
6600 timevar_push (TV_OUT_OF_SSA);
6601 rewrite_out_of_ssa (&SA);
6602 timevar_pop (TV_OUT_OF_SSA);
6603 SA.partition_to_pseudo = XCNEWVEC (rtx, SA.map->num_partitions);
6605 if (MAY_HAVE_DEBUG_BIND_STMTS && flag_tree_ter)
6607 gimple_stmt_iterator gsi;
6608 FOR_EACH_BB_FN (bb, cfun)
6609 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
6610 if (gimple_debug_bind_p (gsi_stmt (gsi)))
6611 avoid_deep_ter_for_debug (gsi_stmt (gsi), 0);
6614 /* Mark arrays indexed with non-constant indices with TREE_ADDRESSABLE. */
6615 auto_bitmap forced_stack_vars;
6616 discover_nonconstant_array_refs (forced_stack_vars);
6618 /* Make sure all values used by the optimization passes have sane
6622 /* Some backends want to know that we are expanding to RTL. */
6623 currently_expanding_to_rtl = 1;
6624 /* Dominators are not kept up-to-date as we may create new basic-blocks. */
6625 free_dominance_info (CDI_DOMINATORS);
6627 rtl_profile_for_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
6629 insn_locations_init ();
6630 if (!DECL_IS_UNDECLARED_BUILTIN (current_function_decl))
6632 /* Eventually, all FEs should explicitly set function_start_locus. */
6633 if (LOCATION_LOCUS (fun->function_start_locus) == UNKNOWN_LOCATION)
6634 set_curr_insn_location
6635 (DECL_SOURCE_LOCATION (current_function_decl));
6637 set_curr_insn_location (fun->function_start_locus);
6640 set_curr_insn_location (UNKNOWN_LOCATION);
6641 prologue_location = curr_insn_location ();
6643 #ifdef INSN_SCHEDULING
6644 init_sched_attrs ();
6647 /* Make sure first insn is a note even if we don't want linenums.
6648 This makes sure the first insn will never be deleted.
6649 Also, final expects a note to appear there. */
6650 emit_note (NOTE_INSN_DELETED);
6652 targetm.expand_to_rtl_hook ();
6653 crtl->init_stack_alignment ();
6654 fun->cfg->max_jumptable_ents = 0;
6656 /* Resovle the function section. Some targets, like ARM EABI rely on knowledge
6657 of the function section at exapnsion time to predict distance of calls. */
6658 resolve_unique_section (current_function_decl, 0, flag_function_sections);
6660 /* Expand the variables recorded during gimple lowering. */
6661 timevar_push (TV_VAR_EXPAND);
6664 var_ret_seq = expand_used_vars (forced_stack_vars);
6666 var_seq = get_insns ();
6668 timevar_pop (TV_VAR_EXPAND);
6670 /* Honor stack protection warnings. */
6671 if (warn_stack_protect)
6673 if (fun->calls_alloca)
6674 warning (OPT_Wstack_protector,
6675 "stack protector not protecting local variables: "
6676 "variable length buffer");
6677 if (has_short_buffer && !crtl->stack_protect_guard)
6678 warning (OPT_Wstack_protector,
6679 "stack protector not protecting function: "
6680 "all local arrays are less than %d bytes long",
6681 (int) param_ssp_buffer_size);
6684 /* Temporarily mark PARM_DECLs and RESULT_DECLs we need to expand to
6685 memory addressable so expand_function_start can emit the required
6687 auto_vec<tree, 16> marked_parms;
6688 for (tree parm = DECL_ARGUMENTS (current_function_decl); parm;
6689 parm = DECL_CHAIN (parm))
6690 if (!TREE_ADDRESSABLE (parm)
6691 && bitmap_bit_p (forced_stack_vars, DECL_UID (parm)))
6693 TREE_ADDRESSABLE (parm) = 1;
6694 marked_parms.safe_push (parm);
6696 if (DECL_RESULT (current_function_decl)
6697 && !TREE_ADDRESSABLE (DECL_RESULT (current_function_decl))
6698 && bitmap_bit_p (forced_stack_vars,
6699 DECL_UID (DECL_RESULT (current_function_decl))))
6701 TREE_ADDRESSABLE (DECL_RESULT (current_function_decl)) = 1;
6702 marked_parms.safe_push (DECL_RESULT (current_function_decl));
6705 /* Set up parameters and prepare for return, for the function. */
6706 expand_function_start (current_function_decl);
6708 /* Clear TREE_ADDRESSABLE again. */
6709 while (!marked_parms.is_empty ())
6710 TREE_ADDRESSABLE (marked_parms.pop ()) = 0;
6712 /* If we emitted any instructions for setting up the variables,
6713 emit them before the FUNCTION_START note. */
6716 emit_insn_before (var_seq, parm_birth_insn);
6718 /* In expand_function_end we'll insert the alloca save/restore
6719 before parm_birth_insn. We've just insertted an alloca call.
6720 Adjust the pointer to match. */
6721 parm_birth_insn = var_seq;
6724 /* Now propagate the RTL assignment of each partition to the
6725 underlying var of each SSA_NAME. */
6728 FOR_EACH_SSA_NAME (i, name, cfun)
6730 /* We might have generated new SSA names in
6731 update_alias_info_with_stack_vars. They will have a NULL
6732 defining statements, and won't be part of the partitioning,
6734 if (!SSA_NAME_DEF_STMT (name))
6737 adjust_one_expanded_partition_var (name);
6740 /* Clean up RTL of variables that straddle across multiple
6741 partitions, and check that the rtl of any PARM_DECLs that are not
6742 cleaned up is that of their default defs. */
6743 FOR_EACH_SSA_NAME (i, name, cfun)
6747 /* We might have generated new SSA names in
6748 update_alias_info_with_stack_vars. They will have a NULL
6749 defining statements, and won't be part of the partitioning,
6751 if (!SSA_NAME_DEF_STMT (name))
6753 part = var_to_partition (SA.map, name);
6754 if (part == NO_PARTITION)
6757 /* If this decl was marked as living in multiple places, reset
6758 this now to NULL. */
6759 tree var = SSA_NAME_VAR (name);
6760 if (var && DECL_RTL_IF_SET (var) == pc_rtx)
6761 SET_DECL_RTL (var, NULL);
6762 /* Check that the pseudos chosen by assign_parms are those of
6763 the corresponding default defs. */
6764 else if (SSA_NAME_IS_DEFAULT_DEF (name)
6765 && (TREE_CODE (var) == PARM_DECL
6766 || TREE_CODE (var) == RESULT_DECL))
6768 rtx in = DECL_RTL_IF_SET (var);
6770 rtx out = SA.partition_to_pseudo[part];
6771 gcc_assert (in == out);
6773 /* Now reset VAR's RTL to IN, so that the _EXPR attrs match
6774 those expected by debug backends for each parm and for
6775 the result. This is particularly important for stabs,
6776 whose register elimination from parm's DECL_RTL may cause
6777 -fcompare-debug differences as SET_DECL_RTL changes reg's
6778 attrs. So, make sure the RTL already has the parm as the
6779 EXPR, so that it won't change. */
6780 SET_DECL_RTL (var, NULL_RTX);
6782 set_mem_attributes (in, var, true);
6783 SET_DECL_RTL (var, in);
6787 /* If this function is `main', emit a call to `__main'
6788 to run global initializers, etc. */
6789 if (DECL_NAME (current_function_decl)
6790 && MAIN_NAME_P (DECL_NAME (current_function_decl))
6791 && DECL_FILE_SCOPE_P (current_function_decl))
6792 expand_main_function ();
6794 /* Initialize the stack_protect_guard field. This must happen after the
6795 call to __main (if any) so that the external decl is initialized. */
6796 if (crtl->stack_protect_guard && targetm.stack_protect_runtime_enabled_p ())
6797 stack_protect_prologue ();
6799 expand_phi_nodes (&SA);
6801 /* Release any stale SSA redirection data. */
6802 redirect_edge_var_map_empty ();
6804 /* Register rtl specific functions for cfg. */
6805 rtl_register_cfg_hooks ();
6807 init_block = construct_init_block ();
6809 /* Clear EDGE_EXECUTABLE on the entry edge(s). It is cleaned from the
6810 remaining edges later. */
6811 FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (fun)->succs)
6812 e->flags &= ~EDGE_EXECUTABLE;
6814 /* If the function has too many markers, drop them while expanding. */
6815 if (cfun->debug_marker_count
6816 >= param_max_debug_marker_count)
6817 cfun->debug_nonbind_markers = false;
6819 lab_rtx_for_bb = new hash_map<basic_block, rtx_code_label *>;
6820 FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
6822 bb = expand_gimple_basic_block (bb, var_ret_seq != NULL_RTX);
6824 if (MAY_HAVE_DEBUG_BIND_INSNS)
6825 expand_debug_locations ();
6827 if (deep_ter_debug_map)
6829 delete deep_ter_debug_map;
6830 deep_ter_debug_map = NULL;
6833 /* Free stuff we no longer need after GIMPLE optimizations. */
6834 free_dominance_info (CDI_DOMINATORS);
6835 free_dominance_info (CDI_POST_DOMINATORS);
6836 delete_tree_cfg_annotations (fun);
6838 timevar_push (TV_OUT_OF_SSA);
6839 finish_out_of_ssa (&SA);
6840 timevar_pop (TV_OUT_OF_SSA);
6842 timevar_push (TV_POST_EXPAND);
6843 /* We are no longer in SSA form. */
6844 fun->gimple_df->in_ssa_p = false;
6845 loops_state_clear (LOOP_CLOSED_SSA);
6847 /* Expansion is used by optimization passes too, set maybe_hot_insn_p
6848 conservatively to true until they are all profile aware. */
6849 delete lab_rtx_for_bb;
6850 free_histograms (fun);
6852 construct_exit_block ();
6853 insn_locations_finalize ();
6857 rtx_insn *after = return_label;
6858 rtx_insn *next = NEXT_INSN (after);
6859 if (next && NOTE_INSN_BASIC_BLOCK_P (next))
6861 emit_insn_after (var_ret_seq, after);
6864 if (hwasan_sanitize_stack_p ())
6865 hwasan_maybe_emit_frame_base_init ();
6867 /* Zap the tree EH table. */
6868 set_eh_throw_stmt_table (fun, NULL);
6870 /* We need JUMP_LABEL be set in order to redirect jumps, and hence
6871 split edges which edge insertions might do. */
6872 rebuild_jump_labels (get_insns ());
6874 /* If we have a single successor to the entry block, put the pending insns
6875 after parm birth, but before NOTE_INSNS_FUNCTION_BEG. */
6876 if (single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (fun)))
6878 edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
6881 rtx_insn *insns = e->insns.r;
6883 rebuild_jump_labels_chain (insns);
6884 if (NOTE_P (parm_birth_insn)
6885 && NOTE_KIND (parm_birth_insn) == NOTE_INSN_FUNCTION_BEG)
6886 emit_insn_before_noloc (insns, parm_birth_insn, e->dest);
6888 emit_insn_after_noloc (insns, parm_birth_insn, e->dest);
6892 /* Otherwise, as well as for other edges, take the usual way. */
6893 commit_edge_insertions ();
6895 /* We're done expanding trees to RTL. */
6896 currently_expanding_to_rtl = 0;
6898 flush_mark_addressable_queue ();
6900 FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun)->next_bb,
6901 EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
6905 for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
6907 /* Clear EDGE_EXECUTABLE. This flag is never used in the backend. */
6908 e->flags &= ~EDGE_EXECUTABLE;
6910 /* At the moment not all abnormal edges match the RTL
6911 representation. It is safe to remove them here as
6912 find_many_sub_basic_blocks will rediscover them.
6913 In the future we should get this fixed properly. */
6914 if ((e->flags & EDGE_ABNORMAL)
6915 && !(e->flags & EDGE_SIBCALL))
6922 auto_sbitmap blocks (last_basic_block_for_fn (fun));
6923 bitmap_ones (blocks);
6924 find_many_sub_basic_blocks (blocks);
6925 purge_all_dead_edges ();
6927 /* After initial rtl generation, call back to finish generating
6928 exception support code. We need to do this before cleaning up
6929 the CFG as the code does not expect dead landing pads. */
6930 if (fun->eh->region_tree != NULL)
6931 finish_eh_generation ();
6933 /* Call expand_stack_alignment after finishing all
6934 updates to crtl->preferred_stack_boundary. */
6935 expand_stack_alignment ();
6937 /* Fixup REG_EQUIV notes in the prologue if there are tailcalls in this
6939 if (crtl->tail_call_emit)
6940 fixup_tail_calls ();
6942 HOST_WIDE_INT patch_area_size, patch_area_entry;
6943 parse_and_check_patch_area (flag_patchable_function_entry, false,
6944 &patch_area_size, &patch_area_entry);
6946 tree patchable_function_entry_attr
6947 = lookup_attribute ("patchable_function_entry",
6948 DECL_ATTRIBUTES (cfun->decl));
6949 if (patchable_function_entry_attr)
6951 tree pp_val = TREE_VALUE (patchable_function_entry_attr);
6952 tree patchable_function_entry_value1 = TREE_VALUE (pp_val);
6954 patch_area_size = tree_to_uhwi (patchable_function_entry_value1);
6955 patch_area_entry = 0;
6956 if (TREE_CHAIN (pp_val) != NULL_TREE)
6958 tree patchable_function_entry_value2
6959 = TREE_VALUE (TREE_CHAIN (pp_val));
6960 patch_area_entry = tree_to_uhwi (patchable_function_entry_value2);
6964 if (patch_area_entry > patch_area_size)
6966 if (patch_area_size > 0)
6967 warning (OPT_Wattributes,
6968 "patchable function entry %wu exceeds size %wu",
6969 patch_area_entry, patch_area_size);
6970 patch_area_entry = 0;
6973 crtl->patch_area_size = patch_area_size;
6974 crtl->patch_area_entry = patch_area_entry;
6976 /* BB subdivision may have created basic blocks that are only reachable
6977 from unlikely bbs but not marked as such in the profile. */
6979 propagate_unlikely_bbs_forward ();
6981 /* Remove unreachable blocks, otherwise we cannot compute dominators
6982 which are needed for loop state verification. As a side-effect
6983 this also compacts blocks.
6984 ??? We cannot remove trivially dead insns here as for example
6985 the DRAP reg on i?86 is not magically live at this point.
6986 gcc.c-torture/execute/ipa-sra-2.c execution, -Os -m32 fails otherwise. */
6987 cleanup_cfg (CLEANUP_NO_INSN_DEL);
6989 checking_verify_flow_info ();
6991 /* Initialize pseudos allocated for hard registers. */
6992 emit_initial_value_sets ();
6994 /* And finally unshare all RTL. */
6997 /* There's no need to defer outputting this function any more; we
6998 know we want to output it. */
6999 DECL_DEFER_OUTPUT (current_function_decl) = 0;
7001 /* Now that we're done expanding trees to RTL, we shouldn't have any
7002 more CONCATs anywhere. */
7003 generating_concat_p = 0;
7008 "\n\n;;\n;; Full RTL generated for this function:\n;;\n");
7009 /* And the pass manager will dump RTL for us. */
7012 /* If we're emitting a nested function, make sure its parent gets
7013 emitted as well. Doing otherwise confuses debug info. */
7016 for (parent = DECL_CONTEXT (current_function_decl);
7017 parent != NULL_TREE;
7018 parent = get_containing_scope (parent))
7019 if (TREE_CODE (parent) == FUNCTION_DECL)
7020 TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (parent)) = 1;
7023 TREE_ASM_WRITTEN (current_function_decl) = 1;
7025 /* After expanding, the return labels are no longer needed. */
7026 return_label = NULL;
7027 naked_return_label = NULL;
7029 /* After expanding, the tm_restart map is no longer needed. */
7030 if (fun->gimple_df->tm_restart)
7031 fun->gimple_df->tm_restart = NULL;
7033 /* Tag the blocks with a depth number so that change_scope can find
7034 the common parent easily. */
7035 set_block_levels (DECL_INITIAL (fun->decl), 0);
7036 default_rtl_profile ();
7038 /* For -dx discard loops now, otherwise IL verify in clean_state will
7040 if (rtl_dump_and_exit)
7042 cfun->curr_properties &= ~PROP_loops;
7043 loop_optimizer_finalize ();
7046 timevar_pop (TV_POST_EXPAND);
7054 make_pass_expand (gcc::context *ctxt)
7056 return new pass_expand (ctxt);