From d2241f4f3c1c7edd045c9a96f32ee8de77da0356 Mon Sep 17 00:00:00 2001 From: Sasha Goldshtein Date: Tue, 9 Feb 2016 06:23:10 -0800 Subject: [PATCH] Added --stack-depth switch to control the number of stack frames captured for each allocation --- man/man8/memleak.8 | 27 ++++++++++++++++++--------- tools/memleak.c | 13 +------------ tools/memleak.py | 11 ++++++++--- tools/memleak_examples.txt | 31 ++++++++++++++++++++++++++++++- 4 files changed, 57 insertions(+), 25 deletions(-) diff --git a/man/man8/memleak.8 b/man/man8/memleak.8 index 37951d9..835aafb 100644 --- a/man/man8/memleak.8 +++ b/man/man8/memleak.8 @@ -2,7 +2,7 @@ .SH NAME memleak \- Print a summary of outstanding allocations and their call stacks to detect memory leaks. Uses Linux eBPF/bcc. .SH SYNOPSIS -.B memleak [-h] [-p PID] [-t] [-a] [-o OLDER] [-c COMMAND] [-s SAMPLE_RATE] [INTERVAL] [COUNT] +.B memleak [-h] [-p PID] [-t] [-a] [-o OLDER] [-c COMMAND] [-s SAMPLE_RATE] [-d STACK_DEPTH] [INTERVAL] [COUNT] .SH DESCRIPTION memleak traces and matches memory allocation and deallocation requests, and collects call stacks for each allocation. memleak can then print a summary @@ -11,7 +11,8 @@ of which call stacks performed allocations that weren't subsequently freed. When tracing a specific process, memleak instruments malloc and free from libc. When tracing all processes, memleak instruments kmalloc and kfree. -The stack depth is currently limited to 10 (+1 for the current instruction pointer). +The stack depth is limited to 10 by default (+1 for the current instruction pointer), +but it can be controlled using the \-d switch if deeper stacks are required. This currently only works on x86_64. Check for future versions. .SH REQUIREMENTS @@ -27,13 +28,6 @@ Trace this process ID only (filtered in-kernel). This traces malloc and free fro \-t Print a trace of all allocation and free requests and results. .TP -INTERVAL -Print a summary of oustanding allocations and their call stacks every INTERVAL seconds. -The default interval is 5 seconds. -.TP -COUNT -Print the outstanding allocations summary COUNT times and then exit. -.TP \-a Print a list of allocations that weren't freed (and their sizes) in addition to their call stacks. .TP @@ -46,6 +40,17 @@ Run the specified command and trace its allocations only. This traces malloc and .TP \-s SAMPLE_RATE Record roughly every SAMPLE_RATE-th allocation to reduce overhead. +.TP +\-d STACK_DEPTH +Capture STACK_DEPTH frames (or less) when obtaining allocation call stacks. +The default value is 10. +.TP +INTERVAL +Print a summary of oustanding allocations and their call stacks every INTERVAL seconds. +The default interval is 5 seconds. +.TP +COUNT +Print the outstanding allocations summary COUNT times and then exit. .SH EXAMPLES .TP Print outstanding kernel allocation stacks every 3 seconds: @@ -76,6 +81,10 @@ placed in a typical period of 10 seconds: # .B perf stat -a -e 'probe:__kmalloc' -- sleep 10 + +Another setting that may help reduce overhead is lowering the number of stack +frames captured and parsed by memleak for each allocation, using the \-d switch. + .SH SOURCE This is from bcc. .IP diff --git a/tools/memleak.c b/tools/memleak.c index c00c398..5c1ce42 100644 --- a/tools/memleak.c +++ b/tools/memleak.c @@ -1,7 +1,5 @@ #include -#define MAX_STACK_SIZE 10 - struct alloc_info_t { u64 size; u64 timestamp_ns; @@ -29,16 +27,7 @@ static int grab_stack(struct pt_regs *ctx, struct alloc_info_t *info) { int depth = 0; u64 bp = ctx->bp; - if (!(info->callstack[depth++] = get_frame(&bp))) return depth; - if (!(info->callstack[depth++] = get_frame(&bp))) return depth; - if (!(info->callstack[depth++] = get_frame(&bp))) return depth; - if (!(info->callstack[depth++] = get_frame(&bp))) return depth; - if (!(info->callstack[depth++] = get_frame(&bp))) return depth; - if (!(info->callstack[depth++] = get_frame(&bp))) return depth; - if (!(info->callstack[depth++] = get_frame(&bp))) return depth; - if (!(info->callstack[depth++] = get_frame(&bp))) return depth; - if (!(info->callstack[depth++] = get_frame(&bp))) return depth; - if (!(info->callstack[depth++] = get_frame(&bp))) return depth; + GRAB_ONE_FRAME return depth; } diff --git a/tools/memleak.py b/tools/memleak.py index 7fe707d..063cf31 100755 --- a/tools/memleak.py +++ b/tools/memleak.py @@ -174,7 +174,7 @@ allocations made with kmalloc/kfree. parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawDescriptionHelpFormatter, epilog=examples) -parser.add_argument("-p", "--pid", type=int, +parser.add_argument("-p", "--pid", type=int, default=-1, help="the PID to trace; if not specified, trace kernel allocs") parser.add_argument("-t", "--trace", action="store_true", help="print trace messages for each alloc/free call") @@ -190,10 +190,12 @@ parser.add_argument("-c", "--command", help="execute and trace the specified command") parser.add_argument("-s", "--sample-rate", default=1, type=int, help="sample every N-th allocation to decrease the overhead") +parser.add_argument("-d", "--stack_depth", default=10, type=int, + help="maximum stack depth to capture") args = parser.parse_args() -pid = -1 if args.pid is None else args.pid +pid = args.pid command = args.command kernel_trace = (pid == -1 and command is None) trace_all = args.trace @@ -201,6 +203,7 @@ interval = args.interval min_age_ns = 1e6 * args.older sample_every_n = args.sample_rate num_prints = args.count +max_stack_size = args.stack_depth + 2 if command is not None: print("Executing '%s' and tracing the resulting process." % command) @@ -209,7 +212,9 @@ if command is not None: bpf_source = open("memleak.c").read() bpf_source = bpf_source.replace("SHOULD_PRINT", "1" if trace_all else "0") bpf_source = bpf_source.replace("SAMPLE_EVERY_N", str(sample_every_n)) - +bpf_source = bpf_source.replace("GRAB_ONE_FRAME", max_stack_size * + "\tif (!(info->callstack[depth++] = get_frame(&bp))) return depth;\n") +bpf_source = bpf_source.replace("MAX_STACK_SIZE", str(max_stack_size)) bpf_program = BPF(text=bpf_source) if not kernel_trace: diff --git a/tools/memleak_examples.txt b/tools/memleak_examples.txt index f046506..2948606 100644 --- a/tools/memleak_examples.txt +++ b/tools/memleak_examples.txt @@ -119,12 +119,39 @@ For example: ... will print the outstanding allocation statistics every second, for ten times, and then exit. +memleak may introduce considerable overhead if your application or kernel is +allocating and freeing memory at a very high rate. In that case, you can +control the overhead by sampling every N-th allocation. For example, to sample +roughly 10% of the allocations and print the outstanding allocations every 5 +seconds, 3 times before quitting: + +# ./memleak.py -p $(pidof allocs) -s 10 5 3 +Attaching to malloc and free in pid 2614, Ctrl+C to quit. +*** Outstanding allocations: + 16 bytes in 1 allocations from stack + main+0x6d [/home/vagrant/allocs] (400862) + __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fdc11ce8790) + +*** Outstanding allocations: + 16 bytes in 1 allocations from stack + main+0x6d [/home/vagrant/allocs] (400862) + __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fdc11ce8790) + +*** Outstanding allocations: + 32 bytes in 2 allocations from stack + main+0x6d [/home/vagrant/allocs] (400862) + __libc_start_main+0xf0 [/usr/lib64/libc-2.21.so] (7fdc11ce8790) + +Note that even though the application leaks 16 bytes of memory every second, +the report (printed every 5 seconds) doesn't "see" all the allocations because +of the sampling rate applied. + USAGE message: # ./memleak.py -h usage: memleak.py [-h] [-p PID] [-t] [-a] [-o OLDER] [-c COMMAND] - [-s SAMPLE_RATE] + [-s SAMPLE_RATE] [-d STACK_DEPTH] [interval] [count] Trace outstanding memory allocations that weren't freed. @@ -148,6 +175,8 @@ optional arguments: execute and trace the specified command -s SAMPLE_RATE, --sample-rate SAMPLE_RATE sample every N-th allocation to decrease the overhead + -d STACK_DEPTH, --stack_depth STACK_DEPTH + maximum stack depth to capture EXAMPLES: -- 2.7.4