From 521ab4f137aa75f651f50b47c54950c091d1b529 Mon Sep 17 00:00:00 2001 From: Sasha Goldshtein Date: Mon, 8 Feb 2016 05:48:31 -0800 Subject: [PATCH] Added -s switch to perform allocation sampling --- man/man8/memleak.8 | 8 ++++++-- tools/memleak.c | 9 +++++++++ tools/memleak.py | 8 +++++++- tools/memleak_examples.txt | 6 +++++- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/man/man8/memleak.8 b/man/man8/memleak.8 index 0f873fd..612d751 100644 --- a/man/man8/memleak.8 +++ b/man/man8/memleak.8 @@ -2,7 +2,7 @@ .SH NAME memleak \- Print a summary of outstanding allocations and their call stacks to detect memory leaks. Uses Linux eBPF/bcc. .SH SYNOPSIS -.B memleak [-h] [-p PID] [-t] [-i INTERVAL] [-a] [-o OLDER] [-c COMMAND] +.B memleak [-h] [-p PID] [-t] [-i INTERVAL] [-a] [-o OLDER] [-c COMMAND] [-s SAMPLE_RATE] .SH DESCRIPTION memleak traces and matches memory allocation and deallocation requests, and collects call stacks for each allocation. memleak can then print a summary @@ -40,6 +40,9 @@ The default value is 500 milliseconds. .TP \-c COMMAND Run the specified command and trace its allocations only. This traces malloc and free from libc. +.TP +\-s SAMPLE_RATE +Record roughly every SAMPLE_RATE-th allocation to reduce overhead. .SH EXAMPLES .TP Print outstanding kernel allocation stacks every 3 seconds: @@ -57,7 +60,8 @@ Run ./allocs and print outstanding allocation stacks for that process: memleak can have significant overhead if the target process or kernel performs allocations at a very high rate. Pathological cases may exhibit up to 100x degradation in running time. Most of the time, however, memleak shouldn't cause -a significant slowdown. +a significant slowdown. You can also use the \-s switch to reduce the overhead +further by capturing only every N-th allocation. To determine the rate at which your application is calling malloc/free, or the rate at which your kernel is calling kmalloc/kfree, place a probe with perf and diff --git a/tools/memleak.c b/tools/memleak.c index 03fa242..c00c398 100644 --- a/tools/memleak.c +++ b/tools/memleak.c @@ -44,6 +44,15 @@ static int grab_stack(struct pt_regs *ctx, struct alloc_info_t *info) int alloc_enter(struct pt_regs *ctx, size_t size) { + // Ideally, this should use a random number source, such as + // BPF_FUNC_get_prandom_u32, but that's currently not supported + // by the bcc front-end. + if (SAMPLE_EVERY_N > 1) { + u64 ts = bpf_ktime_get_ns(); + if (ts % SAMPLE_EVERY_N != 0) + return 0; + } + u64 pid = bpf_get_current_pid_tgid(); u64 size64 = size; sizes.update(&pid, &size64); diff --git a/tools/memleak.py b/tools/memleak.py index b966ff0..0180d72 100755 --- a/tools/memleak.py +++ b/tools/memleak.py @@ -47,7 +47,7 @@ class StackDecoder(object): @staticmethod def _is_binary_segment(parts): return len(parts) == 6 and \ - parts[5][0] == '[' and 'x' in parts[1] + parts[5][0] != '[' and 'x' in parts[1] def _get_code_ranges(self): ranges = {} @@ -161,6 +161,8 @@ EXAMPLES: ./memleak.py -o 60000 Trace allocations in kernel mode and display a summary of outstanding allocations that are at least one minute (60 seconds) old +./memleak.py -s 5 + Trace roughly every 5th allocation, to reduce overhead """ description = """ @@ -184,6 +186,8 @@ parser.add_argument("-o", "--older", default=500, help="prune allocations younger than this age in milliseconds") parser.add_argument("-c", "--command", help="execute and trace the specified command") +parser.add_argument("-s", "--sample-rate", default=1, + help="sample every N-th allocation to decrease the overhead") args = parser.parse_args() @@ -193,6 +197,7 @@ kernel_trace = (pid == -1 and command is None) trace_all = args.trace interval = int(args.interval) min_age_ns = 1e6 * int(args.older) +sample_every_n = args.sample_rate if command is not None: print("Executing '%s' and tracing the resulting process." % command) @@ -200,6 +205,7 @@ if command is not None: bpf_source = open("memleak.c").read() bpf_source = bpf_source.replace("SHOULD_PRINT", "1" if trace_all else "0") +bpf_source = bpf_source.replace("SAMPLE_EVERY_N", str(sample_every_n)) bpf_program = BPF(text=bpf_source) diff --git a/tools/memleak_examples.txt b/tools/memleak_examples.txt index 4217775..9861b18 100644 --- a/tools/memleak_examples.txt +++ b/tools/memleak_examples.txt @@ -117,7 +117,7 @@ USAGE message: # ./memleak.py -h usage: memleak.py [-h] [-p PID] [-t] [-i INTERVAL] [-a] [-o OLDER] - [-c COMMAND] + [-c COMMAND] [-s SAMPLE_RATE] Trace outstanding memory allocations that weren't freed. Supports both user-mode allocations made with malloc/free and kernel-mode @@ -137,6 +137,8 @@ optional arguments: milliseconds -c COMMAND, --command COMMAND execute and trace the specified command + -s SAMPLE_RATE, --sample-rate SAMPLE_RATE + sample every N-th allocation to decrease the overhead EXAMPLES: @@ -146,4 +148,6 @@ EXAMPLES: ./memleak.py -p $(pidof allocs) -t Trace allocations and display each individual call to malloc/free ./memleak.py -p $(pidof allocs) -a -i 10 Trace allocations and display allocated addresses, sizes, and stacks every 10 seconds for outstanding allocations ./memleak.py -c "./allocs" Run the specified command and trace its allocations ./memleak.py Trace allocations in kernel mode and display a summary of outstanding allocations every 5 seconds ./memleak.py -o 60000 Trace allocations in kernel mode and display a summary of outstanding allocations that are at least one minute (60 seconds) old +./memleak.py -s 5 + Trace roughly every 5th allocation, to reduce overhead -- 2.7.4