1 /* Callgraph based intraprocedural optimizations.
2 Copyright (C) 2003 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING. If not, write to the Free
19 Software Foundation, 59 Temple Place - Suite 330, Boston, MA
24 #include "coretypes.h"
27 #include "tree-inline.h"
28 #include "langhooks.h"
36 #include "diagnostic.h"
42 #define INSNS_PER_CALL 10
44 static void cgraph_expand_all_functions (void);
45 static void cgraph_mark_functions_to_output (void);
46 static void cgraph_expand_function (struct cgraph_node *);
47 static tree record_call_1 (tree *, int *, void *);
48 static void cgraph_mark_local_functions (void);
49 static void cgraph_optimize_function (struct cgraph_node *);
50 static bool cgraph_default_inline_p (struct cgraph_node *n);
51 static void cgraph_analyze_function (struct cgraph_node *node);
52 static void cgraph_decide_inlining_incrementally (struct cgraph_node *);
54 /* Statistics we collect about inlining algorithm. */
55 static int ncalls_inlined;
56 static int nfunctions_inlined;
57 static int initial_insns;
58 static int overall_insns;
60 /* Records tree nodes seen in cgraph_create_edges. Simply using
61 walk_tree_without_duplicates doesn't guarantee each node is visited
62 once because it gets a new htab upon each recursive call from
64 static htab_t visited_nodes;
66 /* Determine if function DECL is needed. That is, visible to something
67 either outside this translation unit, something magic in the system
68 configury, or (if not doing unit-at-a-time) to something we havn't
72 decide_is_function_needed (struct cgraph_node *node, tree decl)
74 /* If we decided it was needed before, but at the time we didn't have
75 the body of the function available, then it's still needed. We have
76 to go back and re-check its dependencies now. */
80 /* Externally visible functions must be output. The exception is
81 COMDAT functions that must be output only when they are needed. */
82 if (TREE_PUBLIC (decl) && !DECL_COMDAT (decl) && !DECL_EXTERNAL (decl))
85 /* Constructors and destructors are reachable from the runtime by
87 if (DECL_STATIC_CONSTRUCTOR (decl) || DECL_STATIC_DESTRUCTOR (decl))
90 /* If the user told us it is used, then it must be so. */
91 if (lookup_attribute ("used", DECL_ATTRIBUTES (decl)))
94 /* ??? If the assembler name is set by hand, it is possible to assemble
95 the name later after finalizing the function and the fact is noticed
96 in assemble_name then. This is arguably a bug. */
97 if (DECL_ASSEMBLER_NAME_SET_P (decl)
98 && TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (decl)))
101 if (flag_unit_at_a_time)
104 /* If not doing unit at a time, then we'll only defer this function
105 if its marked for inlining. Otherwise we want to emit it now. */
107 /* "extern inline" functions are never output locally. */
108 if (DECL_EXTERNAL (decl))
110 /* We want to emit COMDAT functions only when absolutely necessary. */
111 if (DECL_COMDAT (decl))
113 if (!DECL_INLINE (decl)
114 || (!node->local.disregard_inline_limits
115 /* When declared inline, defer even the uninlinable functions.
116 This allows them to be eliminated when unused. */
117 && !DECL_DECLARED_INLINE_P (decl)
118 && (!node->local.inlinable || !cgraph_default_inline_p (node))))
124 /* When not doing unit-at-a-time, output all functions enqueued.
125 Return true when such a functions were found. */
128 cgraph_assemble_pending_functions (void)
132 if (flag_unit_at_a_time)
135 while (cgraph_nodes_queue)
137 struct cgraph_node *n = cgraph_nodes_queue;
139 cgraph_nodes_queue = cgraph_nodes_queue->next_needed;
140 if (!n->origin && !DECL_EXTERNAL (n->decl))
142 cgraph_expand_function (n);
150 /* DECL has been parsed. Take it, queue it, compile it at the whim of the
151 logic in effect. If NESTED is true, then our caller cannot stand to have
152 the garbage collector run at the moment. We would need to either create
153 a new GC context, or just not compile right now. */
156 cgraph_finalize_function (tree decl, bool nested)
158 struct cgraph_node *node = cgraph_node (decl);
160 if (node->local.finalized)
162 /* As an GCC extension we allow redefinition of the function. The
163 semantics when both copies of bodies differ is not well defined.
164 We replace the old body with new body so in unit at a time mode
165 we always use new body, while in normal mode we may end up with
166 old body inlined into some functions and new body expanded and
169 ??? It may make more sense to use one body for inlining and other
170 body for expanding the function but this is difficult to do. */
172 /* If node->output is set, then this is a unit-at-a-time compilation
173 and we have already begun whole-unit analysis. This is *not*
174 testing for whether we've already emitted the function. That
175 case can be sort-of legitimately seen with real function
176 redefinition errors. I would argue that the front end should
177 never present us with such a case, but don't enforce that for now. */
181 /* Reset our datastructures so we can analyze the function again. */
182 memset (&node->local, 0, sizeof (node->local));
183 memset (&node->global, 0, sizeof (node->global));
184 memset (&node->rtl, 0, sizeof (node->rtl));
185 node->analyzed = false;
186 while (node->callees)
187 cgraph_remove_edge (node, node->callees->callee);
189 /* We may need to re-queue the node for assembling in case
190 we already proceeded it and ignored as not needed. */
191 if (node->reachable && !flag_unit_at_a_time)
193 struct cgraph_node *n;
195 for (n = cgraph_nodes_queue; n; n = n->next_needed)
203 notice_global_symbol (decl);
205 node->local.finalized = true;
207 /* If not unit at a time, then we need to create the call graph
208 now, so that called functions can be queued and emitted now. */
209 if (!flag_unit_at_a_time)
211 cgraph_analyze_function (node);
212 cgraph_decide_inlining_incrementally (node);
215 if (decide_is_function_needed (node, decl))
216 cgraph_mark_needed_node (node);
218 /* If not unit at a time, go ahead and emit everything we've found
219 to be reachable at this time. */
221 cgraph_assemble_pending_functions ();
223 /* If we've not yet emitted decl, tell the debug info about it. */
224 if (!TREE_ASM_WRITTEN (decl))
225 (*debug_hooks->deferred_inline_function) (decl);
228 /* Walk tree and record all calls. Called via walk_tree. */
230 record_call_1 (tree *tp, int *walk_subtrees, void *data)
234 switch (TREE_CODE (t))
237 /* ??? Really, we should mark this decl as *potentially* referenced
238 by this function and re-examine whether the decl is actually used
239 after rtl has been generated. */
241 cgraph_varpool_mark_needed_node (cgraph_varpool_node (t));
245 if (flag_unit_at_a_time)
247 /* Record dereferences to the functions. This makes the
248 functions reachable unconditionally. */
249 tree decl = TREE_OPERAND (*tp, 0);
250 if (TREE_CODE (decl) == FUNCTION_DECL)
251 cgraph_mark_needed_node (cgraph_node (decl));
257 tree decl = get_callee_fndecl (*tp);
258 if (decl && TREE_CODE (decl) == FUNCTION_DECL)
260 if (DECL_BUILT_IN (decl))
262 cgraph_record_call (data, decl);
264 /* When we see a function call, we don't want to look at the
265 function reference in the ADDR_EXPR that is hanging from
266 the CALL_EXPR we're examining here, because we would
267 conclude incorrectly that the function's address could be
268 taken by something that is not a function call. So only
269 walk the function parameter list, skip the other subtrees. */
271 walk_tree (&TREE_OPERAND (*tp, 1), record_call_1, data,
279 /* Save some cycles by not walking types and declaration as we
280 won't find anything useful there anyway. */
281 if (DECL_P (*tp) || TYPE_P (*tp))
287 if ((unsigned int) TREE_CODE (t) >= LAST_AND_UNUSED_TREE_CODE)
288 return (*lang_hooks.callgraph.analyze_expr) (tp, walk_subtrees, data);
295 /* Create cgraph edges for function calls inside BODY from DECL. */
298 cgraph_create_edges (tree decl, tree body)
300 /* The nodes we're interested in are never shared, so walk
301 the tree ignoring duplicates. */
302 visited_nodes = htab_create (37, htab_hash_pointer,
303 htab_eq_pointer, NULL);
304 walk_tree (&body, record_call_1, decl, visited_nodes);
305 htab_delete (visited_nodes);
306 visited_nodes = NULL;
309 /* Analyze the function scheduled to be output. */
311 cgraph_analyze_function (struct cgraph_node *node)
313 tree decl = node->decl;
315 current_function_decl = decl;
317 /* First kill forward declaration so reverse inlining works properly. */
318 cgraph_create_edges (decl, DECL_SAVED_TREE (decl));
320 node->local.inlinable = tree_inlinable_function_p (decl);
321 if (!DECL_ESTIMATED_INSNS (decl))
322 DECL_ESTIMATED_INSNS (decl)
323 = (*lang_hooks.tree_inlining.estimate_num_insns) (decl);
324 node->local.self_insns = DECL_ESTIMATED_INSNS (decl);
325 if (node->local.inlinable)
326 node->local.disregard_inline_limits
327 = (*lang_hooks.tree_inlining.disregard_inline_limits) (decl);
329 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
330 node->global.insns = node->local.self_insns;
331 if (!DECL_EXTERNAL (decl))
333 node->global.cloned_times = 1;
334 node->global.will_be_output = true;
337 node->analyzed = true;
338 current_function_decl = NULL;
341 /* Analyze the whole compilation unit once it is parsed completely. */
344 cgraph_finalize_compilation_unit (void)
346 struct cgraph_node *node;
348 if (!flag_unit_at_a_time)
350 cgraph_assemble_pending_functions ();
354 cgraph_varpool_assemble_pending_decls ();
356 fprintf (stderr, "\nAnalyzing compilation unit\n");
358 timevar_push (TV_CGRAPH);
359 if (cgraph_dump_file)
361 fprintf (cgraph_dump_file, "Initial entry points:");
362 for (node = cgraph_nodes; node; node = node->next)
363 if (node->needed && DECL_SAVED_TREE (node->decl))
364 fprintf (cgraph_dump_file, " %s", cgraph_node_name (node));
365 fprintf (cgraph_dump_file, "\n");
368 /* Propagate reachability flag and lower representation of all reachable
369 functions. In the future, lowering will introduce new functions and
370 new entry points on the way (by template instantiation and virtual
371 method table generation for instance). */
372 while (cgraph_nodes_queue)
374 struct cgraph_edge *edge;
375 tree decl = cgraph_nodes_queue->decl;
377 node = cgraph_nodes_queue;
378 cgraph_nodes_queue = cgraph_nodes_queue->next_needed;
380 /* ??? It is possible to create extern inline function and later using
381 weak alas attribute to kill it's body. See
382 gcc.c-torture/compile/20011119-1.c */
383 if (!DECL_SAVED_TREE (decl))
386 if (node->analyzed || !node->reachable || !DECL_SAVED_TREE (decl))
389 cgraph_analyze_function (node);
391 for (edge = node->callees; edge; edge = edge->next_callee)
392 if (!edge->callee->reachable)
393 cgraph_mark_reachable_node (edge->callee);
395 cgraph_varpool_assemble_pending_decls ();
398 /* Collect entry points to the unit. */
400 if (cgraph_dump_file)
402 fprintf (cgraph_dump_file, "Unit entry points:");
403 for (node = cgraph_nodes; node; node = node->next)
404 if (node->needed && DECL_SAVED_TREE (node->decl))
405 fprintf (cgraph_dump_file, " %s", cgraph_node_name (node));
406 fprintf (cgraph_dump_file, "\n\nInitial ");
407 dump_cgraph (cgraph_dump_file);
410 if (cgraph_dump_file)
411 fprintf (cgraph_dump_file, "\nReclaiming functions:");
413 for (node = cgraph_nodes; node; node = node->next)
415 tree decl = node->decl;
417 if (!node->reachable && DECL_SAVED_TREE (decl))
419 cgraph_remove_node (node);
420 if (cgraph_dump_file)
421 fprintf (cgraph_dump_file, " %s", cgraph_node_name (node));
424 if (cgraph_dump_file)
426 fprintf (cgraph_dump_file, "\n\nReclaimed ");
427 dump_cgraph (cgraph_dump_file);
430 timevar_pop (TV_CGRAPH);
433 /* Figure out what functions we want to assemble. */
436 cgraph_mark_functions_to_output (void)
438 struct cgraph_node *node;
440 for (node = cgraph_nodes; node; node = node->next)
442 tree decl = node->decl;
443 struct cgraph_edge *e;
447 for (e = node->callers; e; e = e->next_caller)
451 /* We need to output all local functions that are used and not
452 always inlined, as well as those that are reachable from
453 outside the current compilation unit. */
454 if (DECL_SAVED_TREE (decl)
456 || (e && node->reachable))
457 && !TREE_ASM_WRITTEN (decl) && !node->origin
458 && !DECL_EXTERNAL (decl))
463 /* Optimize the function before expansion. */
466 cgraph_optimize_function (struct cgraph_node *node)
468 tree decl = node->decl;
470 timevar_push (TV_INTEGRATION);
471 /* optimize_inline_calls avoids inlining of current_function_decl. */
472 current_function_decl = decl;
473 if (flag_inline_trees)
474 optimize_inline_calls (decl);
477 for (node = node->nested; node; node = node->next_nested)
478 cgraph_optimize_function (node);
480 timevar_pop (TV_INTEGRATION);
483 /* Expand function specified by NODE. */
486 cgraph_expand_function (struct cgraph_node *node)
488 tree decl = node->decl;
490 if (flag_unit_at_a_time)
491 announce_function (decl);
493 cgraph_optimize_function (node);
495 /* Generate RTL for the body of DECL. Nested functions are expanded
496 via lang_expand_decl_stmt. */
497 (*lang_hooks.callgraph.expand_function) (decl);
499 if (!cgraph_function_possibly_inlined_p (decl))
500 DECL_SAVED_TREE (decl) = NULL;
501 current_function_decl = NULL;
504 /* Fill array order with all nodes with output flag set in the reverse
505 topological order. */
507 cgraph_postorder (struct cgraph_node **order)
509 struct cgraph_node *node, *node2;
512 struct cgraph_edge *edge, last;
514 struct cgraph_node **stack =
515 xcalloc (cgraph_n_nodes, sizeof (struct cgraph_node *));
517 /* We have to deal with cycles nicely, so use a depth first traversal
518 output algorithm. Ignore the fact that some functions won't need
519 to be output and put them into order as well, so we get dependencies
520 right through intline functions. */
521 for (node = cgraph_nodes; node; node = node->next)
523 for (node = cgraph_nodes; node; node = node->next)
530 node->aux = node->callers;
533 while (node2->aux != &last)
536 if (edge->next_caller)
537 node2->aux = edge->next_caller;
540 if (!edge->caller->aux)
542 if (!edge->caller->callers)
543 edge->caller->aux = &last;
545 edge->caller->aux = edge->caller->callers;
546 stack[stack_size++] = node2;
547 node2 = edge->caller;
551 if (node2->aux == &last)
553 order[order_pos++] = node2;
555 node2 = stack[--stack_size];
565 #define INLINED_TIMES(node) ((size_t)(node)->aux)
566 #define SET_INLINED_TIMES(node,times) ((node)->aux = (void *)(times))
568 /* Return list of nodes we decided to inline NODE into, set their output
569 flag and compute INLINED_TIMES.
571 We do simple backtracing to get INLINED_TIMES right. This should not be
572 expensive as we limit the amount of inlining. Alternatively we may first
573 discover set of nodes, topologically sort these and propagate
577 cgraph_inlined_into (struct cgraph_node *node, struct cgraph_node **array)
580 struct cgraph_edge **stack;
581 struct cgraph_edge *e, *e1;
585 /* Fast path: since we traverse in mostly topological order, we will likely
587 for (e = node->callers; e; e = e->next_caller)
594 /* Allocate stack for back-tracking up callgraph. */
595 stack = xmalloc ((cgraph_n_nodes + 1) * sizeof (struct cgraph_edge));
598 /* Push the first edge on to the stack. */
603 struct cgraph_node *caller;
605 /* Look at the edge on the top of the stack. */
609 /* Check if the caller destination has been visited yet. */
612 array[nfound++] = e->caller;
613 /* Mark that we have visited the destination. */
614 caller->output = true;
615 SET_INLINED_TIMES (caller, 0);
617 SET_INLINED_TIMES (caller, INLINED_TIMES (caller) + 1);
619 for (e1 = caller->callers; e1; e1 = e1->next_caller)
628 for (e1 = e->next_caller; e1; e1 = e1->next_caller)
651 if (cgraph_dump_file)
653 fprintf (cgraph_dump_file, " Found inline predecesors of %s:",
654 cgraph_node_name (node));
655 for (i = 0; i < nfound; i++)
657 fprintf (cgraph_dump_file, " %s", cgraph_node_name (array[i]));
658 if (INLINED_TIMES (array[i]) != 1)
659 fprintf (cgraph_dump_file, " (%i times)",
660 (int)INLINED_TIMES (array[i]));
662 fprintf (cgraph_dump_file, "\n");
668 /* Return list of nodes we decided to inline into NODE, set their output
669 flag and compute INLINED_TIMES.
671 This function is identical to cgraph_inlined_into with callers and callees
675 cgraph_inlined_callees (struct cgraph_node *node, struct cgraph_node **array)
678 struct cgraph_edge **stack;
679 struct cgraph_edge *e, *e1;
683 /* Fast path: since we traverse in mostly topological order, we will likely
685 for (e = node->callees; e; e = e->next_callee)
692 /* Allocate stack for back-tracking up callgraph. */
693 stack = xmalloc ((cgraph_n_nodes + 1) * sizeof (struct cgraph_edge));
696 /* Push the first edge on to the stack. */
701 struct cgraph_node *callee;
703 /* Look at the edge on the top of the stack. */
707 /* Check if the callee destination has been visited yet. */
710 array[nfound++] = e->callee;
711 /* Mark that we have visited the destination. */
712 callee->output = true;
713 SET_INLINED_TIMES (callee, 0);
715 SET_INLINED_TIMES (callee, INLINED_TIMES (callee) + 1);
717 for (e1 = callee->callees; e1; e1 = e1->next_callee)
726 for (e1 = e->next_callee; e1; e1 = e1->next_callee)
748 if (cgraph_dump_file)
750 fprintf (cgraph_dump_file, " Found inline successors of %s:",
751 cgraph_node_name (node));
752 for (i = 0; i < nfound; i++)
754 fprintf (cgraph_dump_file, " %s", cgraph_node_name (array[i]));
755 if (INLINED_TIMES (array[i]) != 1)
756 fprintf (cgraph_dump_file, " (%i times)",
757 (int)INLINED_TIMES (array[i]));
759 fprintf (cgraph_dump_file, "\n");
765 /* Estimate size of the function after inlining WHAT into TO. */
768 cgraph_estimate_size_after_inlining (int times, struct cgraph_node *to,
769 struct cgraph_node *what)
771 return (what->global.insns - INSNS_PER_CALL) * times + to->global.insns;
774 /* Estimate the growth caused by inlining NODE into all callees. */
777 cgraph_estimate_growth (struct cgraph_node *node)
781 int clones_added = 0;
782 struct cgraph_edge *e;
784 for (e = node->callers; e; e = e->next_caller)
787 growth += ((cgraph_estimate_size_after_inlining (1, e->caller, node)
789 e->caller->global.insns) *e->caller->global.cloned_times);
790 calls_saved += e->caller->global.cloned_times;
791 clones_added += e->caller->global.cloned_times;
794 /* ??? Wrong for self recursive functions or cases where we decide to not
795 inline for different reasons, but it is not big deal as in that case
796 we will keep the body around, but we will also avoid some inlining. */
797 if (!node->needed && !node->origin && !DECL_EXTERNAL (node->decl))
798 growth -= node->global.insns, clones_added--;
806 /* Update insn sizes after inlining WHAT into TO that is already inlined into
807 all nodes in INLINED array. */
810 cgraph_mark_inline (struct cgraph_node *to, struct cgraph_node *what,
811 struct cgraph_node **inlined, int ninlined,
812 struct cgraph_node **inlined_callees,
813 int ninlined_callees)
818 struct cgraph_edge *e;
822 what->global.inlined = 1;
823 for (e = what->callers; e; e = e->next_caller)
829 e->inline_call = true;
831 clones += e->caller->global.cloned_times;
833 else if (!e->inline_call)
838 ncalls_inlined += times;
840 new_insns = cgraph_estimate_size_after_inlining (times, to, what);
841 if (to->global.will_be_output)
842 overall_insns += new_insns - to->global.insns;
843 to->global.insns = new_insns;
845 if (!called && !what->needed && !what->origin
846 && flag_unit_at_a_time
847 && !DECL_EXTERNAL (what->decl))
849 if (!what->global.will_be_output)
852 nfunctions_inlined++;
853 what->global.will_be_output = 0;
854 overall_insns -= what->global.insns;
856 what->global.cloned_times += clones;
857 for (i = 0; i < ninlined; i++)
860 cgraph_estimate_size_after_inlining (INLINED_TIMES (inlined[i]) *
861 times, inlined[i], what);
862 if (inlined[i]->global.will_be_output)
863 overall_insns += new_insns - inlined[i]->global.insns;
864 inlined[i]->global.insns = new_insns;
866 for (i = 0; i < ninlined_callees; i++)
868 inlined_callees[i]->global.cloned_times +=
869 INLINED_TIMES (inlined_callees[i]) * clones;
873 /* Return false when inlining WHAT into TO is not good idea as it would cause
874 too large growth of function bodies. */
877 cgraph_check_inline_limits (struct cgraph_node *to, struct cgraph_node *what,
878 struct cgraph_node **inlined, int ninlined)
882 struct cgraph_edge *e;
886 for (e = to->callees; e; e = e->next_callee)
887 if (e->callee == what)
890 /* When inlining large function body called once into small function,
891 take the inlined function as base for limiting the growth. */
892 if (to->local.self_insns > what->local.self_insns)
893 limit = to->local.self_insns;
895 limit = what->local.self_insns;
897 limit += limit * PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH) / 100;
899 newsize = cgraph_estimate_size_after_inlining (times, to, what);
900 if (newsize > PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS)
903 for (i = 0; i < ninlined; i++)
906 cgraph_estimate_size_after_inlining (INLINED_TIMES (inlined[i]) *
907 times, inlined[i], what);
908 if (newsize > PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS)
910 inlined[i]->local.self_insns *
911 (100 + PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH)) / 100)
917 /* Return true when function N is small enough to be inlined. */
920 cgraph_default_inline_p (struct cgraph_node *n)
922 if (!DECL_INLINE (n->decl) || !DECL_SAVED_TREE (n->decl))
924 if (DECL_DECLARED_INLINE_P (n->decl))
925 return n->global.insns < MAX_INLINE_INSNS_SINGLE;
927 return n->global.insns < MAX_INLINE_INSNS_AUTO;
930 /* We use greedy algorithm for inlining of small functions:
931 All inline candidates are put into prioritized heap based on estimated
932 growth of the overall number of instructions and then update the estimates.
934 INLINED and INLINED_CALEES are just pointers to arrays large enough
935 to be passed to cgraph_inlined_into and cgraph_inlined_callees. */
938 cgraph_decide_inlining_of_small_functions (struct cgraph_node **inlined,
939 struct cgraph_node **inlined_callees)
942 struct cgraph_node *node;
943 fibheap_t heap = fibheap_new ();
944 struct fibnode **heap_node =
945 xcalloc (cgraph_max_uid, sizeof (struct fibnode *));
946 int ninlined, ninlined_callees;
947 int max_insns = ((HOST_WIDEST_INT) initial_insns
948 * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH)) / 100);
950 /* Put all inline candidates into the heap. */
952 for (node = cgraph_nodes; node; node = node->next)
954 struct cgraph_edge *e;
956 if (!node->local.inlinable || !node->callers
957 || !cgraph_default_inline_p (node))
960 /* Rule out always_inline functions we dealt with earlier. */
961 for (e = node->callers; e; e = e->next_caller)
966 heap_node[node->uid] =
967 fibheap_insert (heap, cgraph_estimate_growth (node), node);
970 if (cgraph_dump_file)
971 fprintf (cgraph_dump_file, "\nDeciding on smaller functions:\n");
972 while ((node = fibheap_extract_min (heap)) && overall_insns <= max_insns)
974 struct cgraph_edge *e;
975 int old_insns = overall_insns;
977 heap_node[node->uid] = NULL;
978 if (cgraph_dump_file)
979 fprintf (cgraph_dump_file,
980 "\nConsidering %s with %i insns\n"
981 " Estimated growth is %+i insns.\n",
982 cgraph_node_name (node), node->global.insns,
983 cgraph_estimate_growth (node));
984 if (!cgraph_default_inline_p (node))
986 if (cgraph_dump_file)
987 fprintf (cgraph_dump_file, " Function too large.\n");
990 ninlined_callees = cgraph_inlined_callees (node, inlined_callees);
991 for (e = node->callers; e; e = e->next_caller)
992 if (!e->inline_call && e->caller != node)
994 ninlined = cgraph_inlined_into (e->caller, inlined);
995 if (e->callee->output
996 || !cgraph_check_inline_limits (e->caller, node, inlined,
999 for (i = 0; i < ninlined; i++)
1000 inlined[i]->output = 0, node->aux = 0;
1001 if (cgraph_dump_file)
1002 fprintf (cgraph_dump_file, " Not inlining into %s.\n",
1003 cgraph_node_name (e->caller));
1006 cgraph_mark_inline (e->caller, node, inlined, ninlined,
1007 inlined_callees, ninlined_callees);
1008 if (heap_node[e->caller->uid])
1009 fibheap_replace_key (heap, heap_node[e->caller->uid],
1010 cgraph_estimate_growth (e->caller));
1012 /* Size of the functions we updated into has changed, so update
1014 for (i = 0; i < ninlined; i++)
1016 inlined[i]->output = 0, node->aux = 0;
1017 if (heap_node[inlined[i]->uid])
1018 fibheap_replace_key (heap, heap_node[inlined[i]->uid],
1019 cgraph_estimate_growth (inlined[i]));
1021 if (cgraph_dump_file)
1022 fprintf (cgraph_dump_file,
1023 " Inlined into %s which now has %i insns.\n",
1024 cgraph_node_name (e->caller),
1025 e->caller->global.insns);
1029 /* Similarly all functions called by the function we just inlined
1030 are now called more times; update keys. */
1032 for (e = node->callees; e; e = e->next_callee)
1033 if (!e->inline_call && heap_node[e->callee->uid])
1034 fibheap_replace_key (heap, heap_node[e->callee->uid],
1035 cgraph_estimate_growth (e->callee));
1037 for (i = 0; i < ninlined_callees; i++)
1039 struct cgraph_edge *e;
1041 for (e = inlined_callees[i]->callees; e; e = e->next_callee)
1042 if (!e->inline_call && heap_node[e->callee->uid])
1043 fibheap_replace_key (heap, heap_node[e->callee->uid],
1044 cgraph_estimate_growth (e->callee));
1046 inlined_callees[i]->output = 0, node->aux = 0;
1048 if (cgraph_dump_file)
1049 fprintf (cgraph_dump_file,
1050 " Inlined %i times for a net change of %+i insns.\n",
1051 node->global.cloned_times, overall_insns - old_insns);
1053 if (cgraph_dump_file && !fibheap_empty (heap))
1054 fprintf (cgraph_dump_file, "\nReached the inline-unit-growth limit.\n");
1055 fibheap_delete (heap);
1059 /* Decide on the inlining. We do so in the topological order to avoid
1060 expenses on updating datastructures. */
1063 cgraph_decide_inlining (void)
1065 struct cgraph_node *node;
1067 struct cgraph_node **order =
1068 xcalloc (cgraph_n_nodes, sizeof (struct cgraph_node *));
1069 struct cgraph_node **inlined =
1070 xcalloc (cgraph_n_nodes, sizeof (struct cgraph_node *));
1071 struct cgraph_node **inlined_callees =
1072 xcalloc (cgraph_n_nodes, sizeof (struct cgraph_node *));
1074 int ninlined_callees;
1078 for (node = cgraph_nodes; node; node = node->next)
1079 initial_insns += node->local.self_insns;
1080 overall_insns = initial_insns;
1082 nnodes = cgraph_postorder (order);
1084 if (cgraph_dump_file)
1085 fprintf (cgraph_dump_file,
1086 "\nDeciding on inlining. Starting with %i insns.\n",
1089 for (node = cgraph_nodes; node; node = node->next)
1092 if (cgraph_dump_file)
1093 fprintf (cgraph_dump_file, "\nInlining always_inline functions:\n");
1095 /* In the first pass mark all always_inline edges. Do this with a priority
1096 so none of our later choices will make this impossible. */
1097 for (i = nnodes - 1; i >= 0; i--)
1099 struct cgraph_edge *e;
1103 for (e = node->callees; e; e = e->next_callee)
1104 if (e->callee->local.disregard_inline_limits)
1108 if (cgraph_dump_file)
1109 fprintf (cgraph_dump_file,
1110 "\nConsidering %s %i insns (always inline)\n",
1111 cgraph_node_name (e->callee), e->callee->global.insns);
1112 ninlined = cgraph_inlined_into (order[i], inlined);
1113 for (; e; e = e->next_callee)
1115 old_insns = overall_insns;
1116 if (e->inline_call || !e->callee->local.disregard_inline_limits)
1118 if (e->callee->output || e->callee == node)
1121 cgraph_inlined_callees (e->callee, inlined_callees);
1122 cgraph_mark_inline (node, e->callee, inlined, ninlined,
1123 inlined_callees, ninlined_callees);
1124 for (y = 0; y < ninlined_callees; y++)
1125 inlined_callees[y]->output = 0, node->aux = 0;
1126 if (cgraph_dump_file)
1127 fprintf (cgraph_dump_file,
1128 " Inlined into %s which now has %i insns.\n",
1129 cgraph_node_name (node->callees->caller),
1130 node->callees->caller->global.insns);
1132 if (cgraph_dump_file && node->global.cloned_times > 0)
1133 fprintf (cgraph_dump_file,
1134 " Inlined %i times for a net change of %+i insns.\n",
1135 node->global.cloned_times, overall_insns - old_insns);
1136 for (y = 0; y < ninlined; y++)
1137 inlined[y]->output = 0, node->aux = 0;
1140 cgraph_decide_inlining_of_small_functions (inlined, inlined_callees);
1142 if (cgraph_dump_file)
1143 fprintf (cgraph_dump_file, "\nDeciding on functions called once:\n");
1145 /* And finally decide what functions are called once. */
1147 for (i = nnodes - 1; i >= 0; i--)
1151 if (node->callers && !node->callers->next_caller && !node->needed
1152 && node->local.inlinable && !node->callers->inline_call
1153 && !DECL_EXTERNAL (node->decl) && !DECL_COMDAT (node->decl))
1156 struct cgraph_node *node1;
1158 /* Verify that we won't duplicate the caller. */
1159 for (node1 = node->callers->caller;
1160 node1->callers && node1->callers->inline_call
1161 && ok; node1 = node1->callers->caller)
1162 if (node1->callers->next_caller || node1->needed)
1166 if (cgraph_dump_file)
1167 fprintf (cgraph_dump_file,
1168 "\nConsidering %s %i insns.\n"
1169 " Called once from %s %i insns.\n",
1170 cgraph_node_name (node), node->global.insns,
1171 cgraph_node_name (node->callers->caller),
1172 node->callers->caller->global.insns);
1173 ninlined = cgraph_inlined_into (node->callers->caller, inlined);
1174 old_insns = overall_insns;
1175 if (cgraph_check_inline_limits
1176 (node->callers->caller, node, inlined, ninlined))
1179 cgraph_inlined_callees (node, inlined_callees);
1180 cgraph_mark_inline (node->callers->caller, node, inlined,
1181 ninlined, inlined_callees,
1183 for (y = 0; y < ninlined_callees; y++)
1184 inlined_callees[y]->output = 0, node->aux = 0;
1185 if (cgraph_dump_file)
1186 fprintf (cgraph_dump_file,
1187 " Inlined into %s which now has %i insns"
1188 " for a net change of %+i insns.\n",
1189 cgraph_node_name (node->callers->caller),
1190 node->callers->caller->global.insns,
1191 overall_insns - old_insns);
1195 if (cgraph_dump_file)
1196 fprintf (cgraph_dump_file,
1197 " Inline limit reached, not inlined.\n");
1199 for (y = 0; y < ninlined; y++)
1200 inlined[y]->output = 0, node->aux = 0;
1205 if (cgraph_dump_file)
1206 fprintf (cgraph_dump_file,
1207 "\nInlined %i calls, eliminated %i functions, "
1208 "%i insns turned to %i insns.\n\n",
1209 ncalls_inlined, nfunctions_inlined, initial_insns,
1213 free (inlined_callees);
1216 /* Decide on the inlining. We do so in the topological order to avoid
1217 expenses on updating datastructures. */
1220 cgraph_decide_inlining_incrementally (struct cgraph_node *node)
1222 struct cgraph_edge *e;
1223 struct cgraph_node **inlined =
1224 xmalloc (sizeof (struct cgraph_node *) * cgraph_n_nodes);
1225 struct cgraph_node **inlined_callees =
1226 xmalloc (sizeof (struct cgraph_node *) * cgraph_n_nodes);
1228 int ninlined_callees;
1231 ninlined = cgraph_inlined_into (node, inlined);
1233 /* First of all look for always inline functions. */
1234 for (e = node->callees; e; e = e->next_callee)
1235 if (e->callee->local.disregard_inline_limits && !e->callee->output
1236 && e->callee != node && !e->inline_call)
1238 ninlined_callees = cgraph_inlined_callees (e->callee, inlined_callees);
1239 cgraph_mark_inline (node, e->callee, inlined, ninlined,
1240 inlined_callees, ninlined_callees);
1241 for (y = 0; y < ninlined_callees; y++)
1242 inlined_callees[y]->output = 0, node->aux = 0;
1245 /* Now do the automatic inlining. */
1246 for (e = node->callees; e; e = e->next_callee)
1247 if (e->callee->local.inlinable && !e->callee->output
1248 && e->callee != node && !e->inline_call
1249 && cgraph_default_inline_p (e->callee)
1250 && cgraph_check_inline_limits (node, e->callee, inlined,
1253 ninlined_callees = cgraph_inlined_callees (e->callee, inlined_callees);
1254 cgraph_mark_inline (node, e->callee, inlined, ninlined,
1255 inlined_callees, ninlined_callees);
1256 for (y = 0; y < ninlined_callees; y++)
1257 inlined_callees[y]->output = 0, node->aux = 0;
1260 /* Clear the flags set by cgraph_inlined_into. */
1261 for (y = 0; y < ninlined; y++)
1262 inlined[y]->output = 0, node->aux = 0;
1265 free (inlined_callees);
1269 /* Return true when CALLER_DECL should be inlined into CALLEE_DECL. */
1272 cgraph_inline_p (tree caller_decl, tree callee_decl)
1274 struct cgraph_node *caller = cgraph_node (caller_decl);
1275 struct cgraph_node *callee = cgraph_node (callee_decl);
1276 struct cgraph_edge *e;
1278 for (e = caller->callees; e; e = e->next_callee)
1279 if (e->callee == callee)
1280 return e->inline_call;
1281 /* We do not record builtins in the callgraph. Perhaps it would make more
1282 sense to do so and then prune out those not overwritten by explicit
1286 /* Expand all functions that must be output.
1288 Attempt to topologically sort the nodes so function is output when
1289 all called functions are already assembled to allow data to be
1290 propagated across the callgraph. Use a stack to get smaller distance
1291 between a function and it's callees (later we may choose to use a more
1292 sophisticated algorithm for function reordering; we will likely want
1293 to use subsections to make the output functions appear in top-down
1297 cgraph_expand_all_functions (void)
1299 struct cgraph_node *node;
1300 struct cgraph_node **order =
1301 xcalloc (cgraph_n_nodes, sizeof (struct cgraph_node *));
1305 cgraph_mark_functions_to_output ();
1307 order_pos = cgraph_postorder (order);
1309 for (i = order_pos - 1; i >= 0; i--)
1314 if (!node->reachable)
1317 cgraph_expand_function (node);
1323 /* Mark all local functions.
1325 A local function is one whose calls can occur only in the
1326 current compilation unit, so we change its calling convention.
1327 We simply mark all static functions whose address is not taken
1331 cgraph_mark_local_functions (void)
1333 struct cgraph_node *node;
1335 if (cgraph_dump_file)
1336 fprintf (cgraph_dump_file, "\nMarking local functions:");
1338 /* Figure out functions we want to assemble. */
1339 for (node = cgraph_nodes; node; node = node->next)
1341 node->local.local = (!node->needed
1342 && DECL_SAVED_TREE (node->decl)
1343 && !TREE_PUBLIC (node->decl));
1344 if (cgraph_dump_file && node->local.local)
1345 fprintf (cgraph_dump_file, " %s", cgraph_node_name (node));
1347 if (cgraph_dump_file)
1348 fprintf (cgraph_dump_file, "\n\n");
1351 /* Perform simple optimizations based on callgraph. */
1354 cgraph_optimize (void)
1356 if (!flag_unit_at_a_time)
1358 timevar_push (TV_CGRAPHOPT);
1360 fprintf (stderr, "Performing intraprocedural optimizations\n");
1362 cgraph_mark_local_functions ();
1363 if (cgraph_dump_file)
1365 fprintf (cgraph_dump_file, "Marked ");
1366 dump_cgraph (cgraph_dump_file);
1369 cgraph_decide_inlining ();
1370 cgraph_global_info_ready = true;
1371 if (cgraph_dump_file)
1373 fprintf (cgraph_dump_file, "Optimized ");
1374 dump_cgraph (cgraph_dump_file);
1376 timevar_pop (TV_CGRAPHOPT);
1378 /* Output everything. */
1380 fprintf (stderr, "Assembling functions:\n");
1381 cgraph_expand_all_functions ();
1382 if (cgraph_dump_file)
1384 fprintf (cgraph_dump_file, "\nFinal ");
1385 dump_cgraph (cgraph_dump_file);