From 66bf2e8e0bd7b7367f9a01502dbcac994a063be2 Mon Sep 17 00:00:00 2001 From: Mark Drayton Date: Sun, 31 Jul 2016 22:47:07 +0100 Subject: [PATCH] offcputime: one symbol cache per process, improve pid/tid handling --- man/man8/offcputime.8 | 12 ++++++++++++ tools/offcputime.py | 34 +++++++++++++++++++++++++--------- tools/offcputime_example.txt | 5 ++++- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/man/man8/offcputime.8 b/man/man8/offcputime.8 index e61617d..5d17d3e 100644 --- a/man/man8/offcputime.8 +++ b/man/man8/offcputime.8 @@ -40,6 +40,9 @@ Print output in folded stack format. \-p PID Trace this process ID only (filtered in-kernel). .TP +\-t TID +Trace this thread ID only (filtered in-kernel). +.TP \-u Only trace user threads (no kernel threads). .TP @@ -52,6 +55,15 @@ Show stacks from user space only (no kernel space stacks). \-K Show stacks from kernel space only (no user space stacks). .TP +\-d +Insert delimiter between kernel/user stacks. +.TP +\-f +Output folded format. +.TP +\-\-stack-storage-size STACK_STORAGE_SIZE +Change the number of unique stack traces that can be stored and displayed. +.TP duration Duration to trace, in seconds. .SH EXAMPLES diff --git a/tools/offcputime.py b/tools/offcputime.py index 1b1feea..89f5d70 100755 --- a/tools/offcputime.py +++ b/tools/offcputime.py @@ -41,6 +41,7 @@ examples = """examples: ./offcputime 5 # trace for 5 seconds only ./offcputime -f 5 # 5 seconds, and output in folded format ./offcputime -p 185 # only trace threads for PID 185 + ./offcputime -t 188 # only trace thread 188 ./offcputime -u # only trace user threads (no kernel) ./offcputime -k # only trace kernel threads (no user) ./offcputime -U # only show user space stacks (no kernel) @@ -51,8 +52,12 @@ parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, epilog=examples) thread_group = parser.add_mutually_exclusive_group() -thread_group.add_argument("-p", "--pid", type=positive_int, - help="trace this PID only") +# Note: this script provides --pid and --tid flags but their arguments are +# referred to internally using kernel nomenclature: TGID and PID. +thread_group.add_argument("-p", "--pid", metavar="PID", dest="tgid", + help="trace this PID only", type=positive_int) +thread_group.add_argument("-t", "--tid", metavar="TID", dest="pid", + help="trace this TID only", type=positive_int) thread_group.add_argument("-u", "--user-threads-only", action="store_true", help="user threads only (no kernel threads)") thread_group.add_argument("-k", "--kernel-threads-only", action="store_true", @@ -68,12 +73,14 @@ parser.add_argument("-f", "--folded", action="store_true", help="output folded format") parser.add_argument("--stack-storage-size", default=1024, type=positive_nonzero_int, - help="the number of unique stack traces that can be stored and " \ - "displayed (default 1024)") + help="the number of unique stack traces that can be stored and " + "displayed (default 1024)") parser.add_argument("duration", nargs="?", default=99999999, type=positive_nonzero_int, help="duration of trace, in seconds") args = parser.parse_args() +if args.pid and args.tgid: + parser.error("specify only one of -p and -t") folded = args.folded duration = int(args.duration) @@ -90,6 +97,7 @@ bpf_text = """ struct key_t { u32 pid; + u32 tgid; int user_stack_id; int kernel_stack_id; char name[TASK_COMM_LEN]; @@ -100,6 +108,7 @@ BPF_STACK_TRACE(stack_traces, STACK_STORAGE_SIZE) int oncpu(struct pt_regs *ctx, struct task_struct *prev) { u32 pid = prev->pid; + u32 tgid = prev->tgid; u64 ts, *tsp; // record previous thread sleep time @@ -110,6 +119,7 @@ int oncpu(struct pt_regs *ctx, struct task_struct *prev) { // get the current thread's start time pid = bpf_get_current_pid_tgid(); + tgid = bpf_get_current_pid_tgid() >> 32; tsp = start.lookup(&pid); if (tsp == 0) { return 0; // missed start or filtered @@ -128,6 +138,7 @@ int oncpu(struct pt_regs *ctx, struct task_struct *prev) { struct key_t key = {}; key.pid = pid; + key.tgid = tgid; key.user_stack_id = USER_STACK_GET; key.kernel_stack_id = KERNEL_STACK_GET; bpf_get_current_comm(&key.name, sizeof(key.name)); @@ -140,9 +151,12 @@ int oncpu(struct pt_regs *ctx, struct task_struct *prev) { # set thread filter thread_context = "" -if args.pid is not None: - thread_context = "PID %s" % args.pid - thread_filter = 'pid == %s' % args.pid +if args.tgid is not None: + thread_context = "PID %d" % args.tgid + thread_filter = 'tgid == %d' % args.tgid +elif args.pid is not None: + thread_context = "TID %d" % args.pid + thread_filter = 'pid == %d' % args.pid elif args.user_threads_only: thread_context = "user threads" thread_filter = '!(prev->flags & PF_KTHREAD)' @@ -224,6 +238,8 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value): has_enomem = True continue + # user stacks will be symbolized by tgid, not pid, to avoid the overhead + # of one symbol resolver per thread user_stack = [] if k.user_stack_id < 0 else \ stack_traces.walk(k.user_stack_id) kernel_stack = [] if k.kernel_stack_id < 0 else \ @@ -234,7 +250,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value): user_stack = list(user_stack) kernel_stack = list(kernel_stack) line = [k.name.decode()] + \ - [b.sym(addr, k.pid) for addr in reversed(user_stack)] + \ + [b.sym(addr, k.tgid) for addr in reversed(user_stack)] + \ (need_delimiter and ["-"] or []) + \ [b.ksym(addr) for addr in reversed(kernel_stack)] print("%s %d" % (";".join(line), v.value)) @@ -245,7 +261,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value): if need_delimiter: print(" --") for addr in user_stack: - print(" %016x %s" % (addr, b.sym(addr, k.pid))) + print(" %016x %s" % (addr, b.sym(addr, k.tgid))) print(" %-16s %s (%d)" % ("-", k.name, k.pid)) print(" %d\n" % v.value) diff --git a/tools/offcputime_example.txt b/tools/offcputime_example.txt index ee0c6ed..f903458 100644 --- a/tools/offcputime_example.txt +++ b/tools/offcputime_example.txt @@ -719,7 +719,7 @@ creating your "off-CPU time flame graphs". USAGE message: # ./offcputime -h -usage: offcputime.py [-h] [-p PID | -k | -u] [-K | -U] [-f] +usage: offcputime.py [-h] [-p PID | -t TID | -u | -k] [-U | -K] [-d] [-f] [--stack-storage-size STACK_STORAGE_SIZE] [duration] @@ -731,6 +731,7 @@ positional arguments: optional arguments: -h, --help show this help message and exit -p PID, --pid PID trace this PID only + -t TID, --tid TID trace this TID only -u, --user-threads-only user threads only (no kernel threads) -k, --kernel-threads-only @@ -741,6 +742,7 @@ optional arguments: -K, --kernel-stacks-only show stacks from kernel space only (no user space stacks) + -d, --delimited insert delimiter between kernel/user stacks -f, --folded output folded format --stack-storage-size STACK_STORAGE_SIZE the number of unique stack traces that can be stored @@ -751,6 +753,7 @@ examples: ./offcputime 5 # trace for 5 seconds only ./offcputime -f 5 # 5 seconds, and output in folded format ./offcputime -p 185 # only trace threads for PID 185 + ./offcputime -t 188 # only trace thread 188 ./offcputime -u # only trace user threads (no kernel) ./offcputime -k # only trace kernel threads (no user) ./offcputime -U # only show user space stacks (no kernel) -- 2.7.4