From 4725a72664032b4e7c233079aa073a427d2958e2 Mon Sep 17 00:00:00 2001 From: Sasha Goldshtein Date: Tue, 18 Oct 2016 20:54:47 +0300 Subject: [PATCH] trace, argdist: -I switch for trace and miscellaneous fixes (#761) * trace: Additional include files support Similarly to `argdist`, `trace` now has a `-I` option for adding include files that can be used in filter and print expressions. This also required a slight modification to `argdist`'s syntax for consistency: where previously we would allow `-I header1 header2`, we now require `-I header1 -I header2` to avoid any mixups with which argument is a header file and which is a probe for `trace`. This is very unlikely to break anyone, because I haven't seen the `-I` option used at all, not to mention extensively with multiple headers. Also made sure the man and example pages are up to date. * argdist: Update -C and -H switches for consistency This commit updates `argdist`'s `-H` and `-C` switches for consistency with the `-I` switch and `trace`'s switches. Specifically, each probe needs an explicit `-C` or `-H` specifier in front of it. This also allows safe and understandable mixing of histogram and counting probes, for example: ``` argdist -C 'p:c:write()' -H 'p::vfs__write(int fd, const void *buf, size_t size):size_t:size#write sizes' ``` * trace: Fix stack trace support for tracepoints Tracepoint probes don't have a `ctx` argument, it's called `args` instead. The recently-added stack trace support code didn't take this into account, and consequently didn't work for tracepoints. This commit fixes the issue, so we can now do things like `trace -K t:block:block_rq_complete`. --- man/man8/argdist.8 | 8 ++++---- man/man8/trace.8 | 13 ++++++++++++- tools/argdist.py | 18 +++++++----------- tools/argdist_example.txt | 14 ++++++-------- tools/trace.py | 32 +++++++++++++++++++------------- tools/trace_example.txt | 4 ++++ 6 files changed, 52 insertions(+), 37 deletions(-) diff --git a/man/man8/argdist.8 b/man/man8/argdist.8 index d1c9404..a24302f 100644 --- a/man/man8/argdist.8 +++ b/man/man8/argdist.8 @@ -2,7 +2,7 @@ .SH NAME argdist \- Trace a function and display a histogram or frequency count of its parameter values. Uses Linux eBPF/bcc. .SH SYNOPSIS -.B argdist [-h] [-p PID] [-z STRING_SIZE] [-i INTERVAL] [-n COUNT] [-v] [-T TOP] [-H specifier [specifier ...]] [-C specifier [specifier ...]] [-I header [header ...]] +.B argdist [-h] [-p PID] [-z STRING_SIZE] [-i INTERVAL] [-n COUNT] [-v] [-T TOP] [-H specifier] [-C specifier] [-I header] .SH DESCRIPTION argdist attaches to function entry and exit points, collects specified parameter values, and stores them in a histogram or a frequency collection that counts @@ -36,12 +36,12 @@ Display the generated BPF program, for debugging purposes. \-T TOP When collecting frequency counts, display only the top TOP entries. .TP -\-H SPECIFIER, \-C SPECIFIER +\-H specifiers, \-C specifiers One or more probe specifications that instruct argdist which functions to probe, which parameters to collect, how to aggregate them, and whether to perform any filtering. See SPECIFIER SYNTAX below. .TP -\-I HEADER +\-I header One or more header files that should be included in the BPF program. This enables the use of structure definitions, enumerations, and constants that are available in these headers. You should provide the same path you would @@ -163,7 +163,7 @@ Print the functions used as thread entry points and how common they are: .TP Print histograms of sleep() and nanosleep() parameter values: # -.B argdist -H 'p:c:sleep(u32 seconds):u32:seconds' 'p:c:nanosleep(struct timespec *req):long:req->tv_nsec' +.B argdist -H 'p:c:sleep(u32 seconds):u32:seconds' -H 'p:c:nanosleep(struct timespec *req):long:req->tv_nsec' .TP Spy on writes to STDOUT performed by process 2780, up to a string size of 120 characters: # diff --git a/man/man8/trace.8 b/man/man8/trace.8 index f33d5e4..ee37924 100644 --- a/man/man8/trace.8 +++ b/man/man8/trace.8 @@ -2,7 +2,7 @@ .SH NAME trace \- Trace a function and print its arguments or return value, optionally evaluating a filter. Uses Linux eBPF/bcc. .SH SYNOPSIS -.B trace [-h] [-p PID] [-v] [-Z STRING_SIZE] [-S] [-M MAX_EVENTS] [-o] probe [probe ...] +.B trace [-h] [-p PID] [-v] [-Z STRING_SIZE] [-S] [-M MAX_EVENTS] [-o] [-K] [-U] [-I header] probe [probe ...] .SH DESCRIPTION trace probes functions you specify and displays trace messages if a particular condition is met. You can control the message format to display function @@ -38,6 +38,17 @@ Print up to MAX_EVENTS trace messages and then exit. Print times relative to the beginning of the trace (offsets), in seconds. The default is to print absolute time. .TP +\-K +Print the kernel stack for each event. +.TP +\-U +Print the user stack for each event. +.TP +\-I header +Additional header files to include in the BPF program. This is needed if your +filter or print expressions use types or data structures that are not available +in the standard headers. For example: 'linux/mm.h' +.TP probe [probe ...] One or more probes that attach to functions, filter conditions, and print information. See PROBE SYNTAX below. diff --git a/tools/argdist.py b/tools/argdist.py index a4aab70..2bb44c0 100755 --- a/tools/argdist.py +++ b/tools/argdist.py @@ -3,11 +3,8 @@ # argdist Trace a function and display a distribution of its # parameter values as a histogram or frequency count. # -# USAGE: argdist [-h] [-p PID] [-z STRING_SIZE] [-i INTERVAL] -# [-n COUNT] [-v] [-c] [-T TOP] -# [-C specifier [specifier ...]] -# [-H specifier [specifier ...]] -# [-I header [header ...]] +# USAGE: argdist [-h] [-p PID] [-z STRING_SIZE] [-i INTERVAL] [-n COUNT] [-v] +# [-c] [-T TOP] [-C specifier] [-H specifier] [-I header] # # Licensed under the Apache License, Version 2.0 (the "License") # Copyright (C) 2016 Sasha Goldshtein. @@ -545,9 +542,8 @@ argdist -C 'u:pthread:pthread_start():u64:arg2' -p 1337 Print frequency of function addresses used as a pthread start function, relying on the USDT pthread_start probe in process 1337 -argdist -H \\ - 'p:c:sleep(u32 seconds):u32:seconds' \\ - 'p:c:nanosleep(struct timespec *req):long:req->tv_nsec' +argdist -H 'p:c:sleep(u32 seconds):u32:seconds' \\ + -H 'p:c:nanosleep(struct timespec *req):long:req->tv_nsec' Print histograms of sleep() and nanosleep() parameter values argdist -p 2780 -z 120 \\ @@ -577,15 +573,15 @@ argdist -p 2780 -z 120 \\ parser.add_argument("-T", "--top", type=int, help="number of top results to show (not applicable to " + "histograms)") - parser.add_argument("-H", "--histogram", nargs="*", + parser.add_argument("-H", "--histogram", action="append", dest="histspecifier", metavar="specifier", help="probe specifier to capture histogram of " + "(see examples below)") - parser.add_argument("-C", "--count", nargs="*", + parser.add_argument("-C", "--count", action="append", dest="countspecifier", metavar="specifier", help="probe specifier to capture count of " + "(see examples below)") - parser.add_argument("-I", "--include", nargs="*", + parser.add_argument("-I", "--include", action="append", metavar="header", help="additional header files to include in the BPF program") self.args = parser.parse_args() diff --git a/tools/argdist_example.txt b/tools/argdist_example.txt index 55fdc05..71ee238 100644 --- a/tools/argdist_example.txt +++ b/tools/argdist_example.txt @@ -299,8 +299,7 @@ USAGE message: # argdist -h usage: argdist [-h] [-p PID] [-z STRING_SIZE] [-i INTERVAL] [-n COUNT] [-v] - [-c] [-T TOP] [-H [specifier [specifier ...]]] - [-C [specifier [specifier ...]]] [-I [header [header ...]]] + [-c] [-T TOP] [-H specifier] [-C[specifier] [-I header] Trace a function and display a summary of its parameter values. @@ -317,13 +316,13 @@ optional arguments: -c, --cumulative do not clear histograms and freq counts at each interval -T TOP, --top TOP number of top results to show (not applicable to histograms) - -H [specifier [specifier ...]], --histogram [specifier [specifier ...]] + -H specifier, --histogram specifier probe specifier to capture histogram of (see examples below) - -C [specifier [specifier ...]], --count [specifier [specifier ...]] + -C specifier, --count specifier probe specifier to capture count of (see examples below) - -I [header [header ...]], --include [header [header ...]] + -I header, --include header additional header files to include in the BPF program Probe specifier syntax: @@ -392,9 +391,8 @@ argdist -C 'u:pthread:pthread_start():u64:arg2' -p 1337 Print frequency of function addresses used as a pthread start function, relying on the USDT pthread_start probe in process 1337 -argdist -H \ - 'p:c:sleep(u32 seconds):u32:seconds' \ - 'p:c:nanosleep(struct timespec *req):long:req->tv_nsec' +argdist -H 'p:c:sleep(u32 seconds):u32:seconds' \ + -H 'p:c:nanosleep(struct timespec *req):long:req->tv_nsec' Print histograms of sleep() and nanosleep() parameter values argdist -p 2780 -z 120 \ diff --git a/tools/trace.py b/tools/trace.py index 6915fc0..d6aef8d 100755 --- a/tools/trace.py +++ b/tools/trace.py @@ -4,7 +4,7 @@ # parameters, with an optional filter. # # USAGE: trace [-h] [-p PID] [-v] [-Z STRING_SIZE] [-S] [-M MAX_EVENTS] [-o] -# probe [probe ...] +# [-K] [-U] [-I header] probe [probe ...] # # Licensed under the Apache License, Version 2.0 (the "License") # Copyright (C) 2016 Sasha Goldshtein. @@ -362,25 +362,26 @@ BPF_PERF_OUTPUT(%s); for i, expr in enumerate(self.values): data_fields += self._generate_field_assign(i) + if self.probe_type == "t": + heading = "TRACEPOINT_PROBE(%s, %s)" % \ + (self.tp_category, self.tp_event) + ctx_name = "args" + else: + heading = "int %s(%s)" % (self.probe_name, signature) + ctx_name = "ctx" + stack_trace = "" if self.user_stack: stack_trace += """ __data.user_stack_id = %s.get_stackid( - ctx, BPF_F_REUSE_STACKID | BPF_F_USER_STACK - );""" % self.stacks_name + %s, BPF_F_REUSE_STACKID | BPF_F_USER_STACK + );""" % (self.stacks_name, ctx_name) if self.kernel_stack: stack_trace += """ __data.kernel_stack_id = %s.get_stackid( - ctx, BPF_F_REUSE_STACKID - );""" % self.stacks_name + %s, BPF_F_REUSE_STACKID + );""" % (self.stacks_name, ctx_name) - if self.probe_type == "t": - heading = "TRACEPOINT_PROBE(%s, %s)" % \ - (self.tp_category, self.tp_event) - ctx_name = "args" - else: - heading = "int %s(%s)" % (self.probe_name, signature) - ctx_name = "ctx" text = heading + """ { %s @@ -551,10 +552,13 @@ trace 'u:pthread:pthread_create (arg4 != 0)' help="use relative time from first traced message") parser.add_argument("-K", "--kernel-stack", action="store_true", help="output kernel stack trace") - parser.add_argument("-U", "--user_stack", action="store_true", + parser.add_argument("-U", "--user-stack", action="store_true", help="output user stack trace") parser.add_argument(metavar="probe", dest="probes", nargs="+", help="probe specifier (see examples)") + parser.add_argument("-I", "--include", action="append", + metavar="header", + help="additional header files to include in the BPF program") self.args = parser.parse_args() def _create_probes(self): @@ -571,6 +575,8 @@ trace 'u:pthread:pthread_create (arg4 != 0)' #include /* For TASK_COMM_LEN */ """ + for include in (self.args.include or []): + self.program += "#include <%s>\n" % include self.program += BPF.generate_auto_includes( map(lambda p: p.raw_probe, self.probes)) for probe in self.probes: diff --git a/tools/trace_example.txt b/tools/trace_example.txt index 20d61c5..e9d7a95 100644 --- a/tools/trace_example.txt +++ b/tools/trace_example.txt @@ -179,6 +179,10 @@ optional arguments: -M MAX_EVENTS, --max-events MAX_EVENTS number of events to print before quitting -o, --offset use relative time from first traced message + -K, --kernel-stack output kernel stack trace + -U, --user-stack output user stack trace + -I header, --include header + additional header files to include in the BPF program EXAMPLES: -- 2.7.4