1 """Class for printing reports on profiled python code."""
3 # Written by James Roskind
4 # Based on prior profile module by Sjoerd Mullender...
5 # which was hacked somewhat by: Guido van Rossum
7 # Copyright Disney Enterprises, Inc. All Rights Reserved.
8 # Licensed to PSF under a Contributor Agreement
10 # Licensed under the Apache License, Version 2.0 (the "License");
11 # you may not use this file except in compliance with the License.
12 # You may obtain a copy of the License at
14 # http://www.apache.org/licenses/LICENSE-2.0
16 # Unless required by applicable law or agreed to in writing, software
17 # distributed under the License is distributed on an "AS IS" BASIS,
18 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
19 # either express or implied. See the License for the specific language
20 # governing permissions and limitations under the License.
28 from functools import cmp_to_key
33 """This class is used for creating reports from data generated by the
34 Profile class. It is a "friend" of that class, and imports data either
35 by direct access to members of Profile class, or by reading in a dictionary
36 that was emitted (via marshal) from the Profile class.
38 The big change from the previous Profiler (in terms of raw functionality)
39 is that an "add()" method has been provided to combine Stats from
40 several distinct profile runs. Both the constructor and the add()
41 method now take arbitrarily many file names as arguments.
43 All the print methods now take an argument that indicates how many lines
44 to print. If the arg is a floating point number between 0 and 1.0, then
45 it is taken as a decimal percentage of the available lines to be printed
46 (e.g., .1 means print 10% of all available lines). If it is an integer,
47 it is taken to mean the number of lines of data that you wish to have
50 The sort_stats() method now processes some additional options (i.e., in
51 addition to the old -1, 0, 1, or 2). It takes an arbitrary number of
52 quoted strings to select the sort order. For example sort_stats('time',
53 'name') sorts on the major key of 'internal function time', and on the
54 minor key of 'the name of the function'. Look at the two tables in
55 sort_stats() and get_sort_arg_defs(self) for more examples.
57 All methods return self, so you can string together commands like:
58 Stats('foo', 'goo').strip_dirs().sort_stats('calls').\
59 print_stats(5).print_callers(5)
62 def __init__(self, *args, **kwds):
63 # I can't figure out how to explictly specify a stream keyword arg
65 # def __init__(self, *args, stream=sys.stdout): ...
66 # so I use **kwds and sqauwk if something unexpected is passed in.
67 self.stream = sys.stdout
69 self.stream = kwds["stream"]
74 extras = ", ".join(["%s=%s" % (k, kwds[k]) for k in keys])
75 raise ValueError, "unrecognized keyword args: %s" % extras
85 self.all_callees = None # calc only if needed
94 self.sort_arg_dict = {}
98 self.get_top_level_stats()
102 print >> self.stream, "Invalid timing data",
103 if self.files: print >> self.stream, self.files[-1],
106 def load_stats(self, arg):
107 if not arg: self.stats = {}
108 elif isinstance(arg, basestring):
110 self.stats = marshal.load(f)
113 file_stats = os.stat(arg)
114 arg = time.ctime(file_stats.st_mtime) + " " + arg
115 except: # in case this is not unix
118 elif hasattr(arg, 'create_stats'):
120 self.stats = arg.stats
123 raise TypeError, "Cannot create or construct a %r object from '%r''" % (
127 def get_top_level_stats(self):
128 for func, (cc, nc, tt, ct, callers) in self.stats.items():
129 self.total_calls += nc
130 self.prim_calls += cc
132 if ("jprofile", 0, "profiler") in callers:
133 self.top_level[func] = None
134 if len(func_std_string(func)) > self.max_name_len:
135 self.max_name_len = len(func_std_string(func))
137 def add(self, *arg_list):
138 if not arg_list: return self
139 if len(arg_list) > 1: self.add(*arg_list[1:])
141 if type(self) != type(other) or self.__class__ != other.__class__:
143 self.files += other.files
144 self.total_calls += other.total_calls
145 self.prim_calls += other.prim_calls
146 self.total_tt += other.total_tt
147 for func in other.top_level:
148 self.top_level[func] = None
150 if self.max_name_len < other.max_name_len:
151 self.max_name_len = other.max_name_len
155 for func, stat in other.stats.iteritems():
156 if func in self.stats:
157 old_func_stat = self.stats[func]
159 old_func_stat = (0, 0, 0, 0, {},)
160 self.stats[func] = add_func_stats(old_func_stat, stat)
163 def dump_stats(self, filename):
164 """Write the profile data to a file we know how to load back."""
165 f = file(filename, 'wb')
167 marshal.dump(self.stats, f)
171 # list the tuple indices and directions for sorting,
172 # along with some printable description
173 sort_arg_dict_default = {
174 "calls" : (((1,-1), ), "call count"),
175 "cumulative": (((3,-1), ), "cumulative time"),
176 "file" : (((4, 1), ), "file name"),
177 "line" : (((5, 1), ), "line number"),
178 "module" : (((4, 1), ), "file name"),
179 "name" : (((6, 1), ), "function name"),
180 "nfl" : (((6, 1),(4, 1),(5, 1),), "name/file/line"),
181 "pcalls" : (((0,-1), ), "call count"),
182 "stdname" : (((7, 1), ), "standard name"),
183 "time" : (((2,-1), ), "internal time"),
186 def get_sort_arg_defs(self):
187 """Expand all abbreviations that are unique."""
188 if not self.sort_arg_dict:
189 self.sort_arg_dict = dict = {}
191 for word, tup in self.sort_arg_dict_default.iteritems():
197 bad_list[fragment] = 0
200 fragment = fragment[:-1]
201 for word in bad_list:
203 return self.sort_arg_dict
205 def sort_stats(self, *field):
209 if len(field) == 1 and isinstance(field[0], (int, long)):
210 # Be compatible with old profiler
211 field = [ {-1: "stdname",
214 2: "cumulative"}[field[0]] ]
216 sort_arg_defs = self.get_sort_arg_defs()
221 sort_tuple = sort_tuple + sort_arg_defs[word][0]
222 self.sort_type += connector + sort_arg_defs[word][1]
226 for func, (cc, nc, tt, ct, callers) in self.stats.iteritems():
227 stats_list.append((cc, nc, tt, ct) + func +
228 (func_std_string(func), func))
230 stats_list.sort(key=cmp_to_key(TupleComp(sort_tuple).compare))
232 self.fcn_list = fcn_list = []
233 for tuple in stats_list:
234 fcn_list.append(tuple[-1])
237 def reverse_order(self):
239 self.fcn_list.reverse()
242 def strip_dirs(self):
243 oldstats = self.stats
244 self.stats = newstats = {}
246 for func, (cc, nc, tt, ct, callers) in oldstats.iteritems():
247 newfunc = func_strip_path(func)
248 if len(func_std_string(newfunc)) > max_name_len:
249 max_name_len = len(func_std_string(newfunc))
251 for func2, caller in callers.iteritems():
252 newcallers[func_strip_path(func2)] = caller
254 if newfunc in newstats:
255 newstats[newfunc] = add_func_stats(
257 (cc, nc, tt, ct, newcallers))
259 newstats[newfunc] = (cc, nc, tt, ct, newcallers)
260 old_top = self.top_level
261 self.top_level = new_top = {}
263 new_top[func_strip_path(func)] = None
265 self.max_name_len = max_name_len
268 self.all_callees = None
271 def calc_callees(self):
272 if self.all_callees: return
273 self.all_callees = all_callees = {}
274 for func, (cc, nc, tt, ct, callers) in self.stats.iteritems():
275 if not func in all_callees:
276 all_callees[func] = {}
277 for func2, caller in callers.iteritems():
278 if not func2 in all_callees:
279 all_callees[func2] = {}
280 all_callees[func2][func] = caller
283 #******************************************************************
284 # The following functions support actual printing of reports
285 #******************************************************************
287 # Optional "amount" is either a line count, or a percentage of lines.
289 def eval_print_amount(self, sel, list, msg):
291 if isinstance(sel, basestring):
293 rex = re.compile(sel)
295 msg += " <Invalid regular expression %r>\n" % sel
299 if rex.search(func_std_string(func)):
300 new_list.append(func)
303 if isinstance(sel, float) and 0.0 <= sel < 1.0:
304 count = int(count * sel + .5)
305 new_list = list[:count]
306 elif isinstance(sel, (int, long)) and 0 <= sel < count:
308 new_list = list[:count]
309 if len(list) != len(new_list):
310 msg += " List reduced from %r to %r due to restriction <%r>\n" % (
311 len(list), len(new_list), sel)
315 def get_print_list(self, sel_list):
316 width = self.max_name_len
318 stat_list = self.fcn_list[:]
319 msg = " Ordered by: " + self.sort_type + '\n'
321 stat_list = self.stats.keys()
322 msg = " Random listing order was used\n"
324 for selection in sel_list:
325 stat_list, msg = self.eval_print_amount(selection, stat_list, msg)
327 count = len(stat_list)
331 print >> self.stream, msg
332 if count < len(self.stats):
334 for func in stat_list:
335 if len(func_std_string(func)) > width:
336 width = len(func_std_string(func))
337 return width+2, stat_list
339 def print_stats(self, *amount):
340 for filename in self.files:
341 print >> self.stream, filename
342 if self.files: print >> self.stream
344 for func in self.top_level:
345 print >> self.stream, indent, func_get_function_name(func)
347 print >> self.stream, indent, self.total_calls, "function calls",
348 if self.total_calls != self.prim_calls:
349 print >> self.stream, "(%d primitive calls)" % self.prim_calls,
350 print >> self.stream, "in %.3f seconds" % self.total_tt
352 width, list = self.get_print_list(amount)
356 self.print_line(func)
361 def print_callees(self, *amount):
362 width, list = self.get_print_list(amount)
366 self.print_call_heading(width, "called...")
368 if func in self.all_callees:
369 self.print_call_line(width, func, self.all_callees[func])
371 self.print_call_line(width, func, {})
376 def print_callers(self, *amount):
377 width, list = self.get_print_list(amount)
379 self.print_call_heading(width, "was called by...")
381 cc, nc, tt, ct, callers = self.stats[func]
382 self.print_call_line(width, func, callers, "<-")
387 def print_call_heading(self, name_size, column_title):
388 print >> self.stream, "Function ".ljust(name_size) + column_title
389 # print sub-header only if we have new-style callers
391 for cc, nc, tt, ct, callers in self.stats.itervalues():
393 value = callers.itervalues().next()
394 subheader = isinstance(value, tuple)
397 print >> self.stream, " "*name_size + " ncalls tottime cumtime"
399 def print_call_line(self, name_size, source, call_dict, arrow="->"):
400 print >> self.stream, func_std_string(source).ljust(name_size) + arrow,
404 clist = call_dict.keys()
408 name = func_std_string(func)
409 value = call_dict[func]
410 if isinstance(value, tuple):
411 nc, cc, tt, ct = value
413 substats = '%d/%d' % (nc, cc)
415 substats = '%d' % (nc,)
416 substats = '%s %s %s %s' % (substats.rjust(7+2*len(indent)),
417 f8(tt), f8(ct), name)
418 left_width = name_size + 1
420 substats = '%s(%r) %s' % (name, value, f8(self.stats[func][3]))
421 left_width = name_size + 3
422 print >> self.stream, indent*left_width + substats
425 def print_title(self):
426 print >> self.stream, ' ncalls tottime percall cumtime percall',
427 print >> self.stream, 'filename:lineno(function)'
429 def print_line(self, func): # hack : should print percentages
430 cc, nc, tt, ct, callers = self.stats[func]
433 c = c + '/' + str(cc)
434 print >> self.stream, c.rjust(9),
435 print >> self.stream, f8(tt),
437 print >> self.stream, ' '*8,
439 print >> self.stream, f8(float(tt)/nc),
440 print >> self.stream, f8(ct),
442 print >> self.stream, ' '*8,
444 print >> self.stream, f8(float(ct)/cc),
445 print >> self.stream, func_std_string(func)
448 """This class provides a generic function for comparing any two tuples.
449 Each instance records a list of tuple-indices (from most significant
450 to least significant), and sort direction (ascending or decending) for
451 each tuple-index. The compare functions can then be used as the function
452 argument to the system sort() function when a list of tuples need to be
453 sorted in the instances order."""
455 def __init__(self, comp_select_list):
456 self.comp_select_list = comp_select_list
458 def compare (self, left, right):
459 for index, direction in self.comp_select_list:
468 #**************************************************************************
469 # func_name is a triple (file:string, line:int, name:string)
471 def func_strip_path(func_name):
472 filename, line, name = func_name
473 return os.path.basename(filename), line, name
475 def func_get_function_name(func):
478 def func_std_string(func_name): # match what old profile produced
479 if func_name[:2] == ('~', 0):
480 # special case for built-in functions
482 if name.startswith('<') and name.endswith('>'):
483 return '{%s}' % name[1:-1]
487 return "%s:%d(%s)" % func_name
489 #**************************************************************************
490 # The following functions combine statists for pairs functions.
491 # The bulk of the processing involves correctly handling "call" lists,
492 # such as callers and callees.
493 #**************************************************************************
495 def add_func_stats(target, source):
496 """Add together all the stats for two profile entries."""
497 cc, nc, tt, ct, callers = source
498 t_cc, t_nc, t_tt, t_ct, t_callers = target
499 return (cc+t_cc, nc+t_nc, tt+t_tt, ct+t_ct,
500 add_callers(t_callers, callers))
502 def add_callers(target, source):
503 """Combine two caller lists in a single list."""
505 for func, caller in target.iteritems():
506 new_callers[func] = caller
507 for func, caller in source.iteritems():
508 if func in new_callers:
509 if isinstance(caller, tuple):
510 # format used by cProfile
511 new_callers[func] = tuple([i[0] + i[1] for i in
512 zip(caller, new_callers[func])])
514 # format used by profile
515 new_callers[func] += caller
517 new_callers[func] = caller
520 def count_calls(callers):
521 """Sum the caller statistics to get total number of calls received."""
523 for calls in callers.itervalues():
527 #**************************************************************************
528 # The following functions support printing of reports
529 #**************************************************************************
534 #**************************************************************************
535 # Statistics browser added by ESR, April 2001
536 #**************************************************************************
538 if __name__ == '__main__':
545 class ProfileBrowser(cmd.Cmd):
546 def __init__(self, profile=None):
547 cmd.Cmd.__init__(self)
550 self.stream = sys.stdout
551 if profile is not None:
552 self.do_read(profile)
554 def generic(self, fn, line):
559 processed.append(int(term))
565 if frac > 1 or frac < 0:
566 print >> self.stream, "Fraction argument must be in [0, 1]"
568 processed.append(frac)
572 processed.append(term)
574 getattr(self.stats, fn)(*processed)
576 print >> self.stream, "No statistics object is loaded."
578 def generic_help(self):
579 print >> self.stream, "Arguments may be:"
580 print >> self.stream, "* An integer maximum number of entries to print."
581 print >> self.stream, "* A decimal fractional number between 0 and 1, controlling"
582 print >> self.stream, " what fraction of selected entries to print."
583 print >> self.stream, "* A regular expression; only entries with function names"
584 print >> self.stream, " that match it are printed."
586 def do_add(self, line):
590 print >> self.stream, "No statistics object is loaded."
593 print >> self.stream, "Add profile info from given file to current statistics object."
595 def do_callees(self, line):
596 return self.generic('print_callees', line)
597 def help_callees(self):
598 print >> self.stream, "Print callees statistics from the current stat object."
601 def do_callers(self, line):
602 return self.generic('print_callers', line)
603 def help_callers(self):
604 print >> self.stream, "Print callers statistics from the current stat object."
607 def do_EOF(self, line):
608 print >> self.stream, ""
611 print >> self.stream, "Leave the profile brower."
613 def do_quit(self, line):
616 print >> self.stream, "Leave the profile brower."
618 def do_read(self, line):
621 self.stats = Stats(line)
622 except IOError, args:
623 print >> self.stream, args[1]
625 except Exception as err:
626 print >> self.stream, err.__class__.__name__ + ':', err
628 self.prompt = line + "% "
629 elif len(self.prompt) > 2:
630 line = self.prompt[:-2]
633 print >> self.stream, "No statistics object is current -- cannot reload."
636 print >> self.stream, "Read in profile data from a specified file."
637 print >> self.stream, "Without argument, reload the current file."
639 def do_reverse(self, line):
641 self.stats.reverse_order()
643 print >> self.stream, "No statistics object is loaded."
645 def help_reverse(self):
646 print >> self.stream, "Reverse the sort order of the profiling report."
648 def do_sort(self, line):
650 print >> self.stream, "No statistics object is loaded."
652 abbrevs = self.stats.get_sort_arg_defs()
653 if line and all((x in abbrevs) for x in line.split()):
654 self.stats.sort_stats(*line.split())
656 print >> self.stream, "Valid sort keys (unique prefixes are accepted):"
657 for (key, value) in Stats.sort_arg_dict_default.iteritems():
658 print >> self.stream, "%s -- %s" % (key, value[1])
661 print >> self.stream, "Sort profile data according to specified keys."
662 print >> self.stream, "(Typing `sort' without arguments lists valid keys.)"
663 def complete_sort(self, text, *args):
664 return [a for a in Stats.sort_arg_dict_default if a.startswith(text)]
666 def do_stats(self, line):
667 return self.generic('print_stats', line)
668 def help_stats(self):
669 print >> self.stream, "Print statistics from the current stat object."
672 def do_strip(self, line):
674 self.stats.strip_dirs()
676 print >> self.stream, "No statistics object is loaded."
677 def help_strip(self):
678 print >> self.stream, "Strip leading path information from filenames in the report."
681 print >> self.stream, "Show help for a given command."
683 def postcmd(self, stop, line):
689 if len(sys.argv) > 1:
690 initprofile = sys.argv[1]
694 browser = ProfileBrowser(initprofile)
695 print >> browser.stream, "Welcome to the profile statistics browser."
697 print >> browser.stream, "Goodbye."
698 except KeyboardInterrupt: