of I/O latency (time), printing this as a histogram when Ctrl-C is hit.
For example:
-# ./biolatency
+# ./biolatency
Tracing block device I/O... Hit Ctrl-C to end.
^C
usecs : count distribution
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# funccount Count kernel function calls.
-# For Linux, uses BCC, eBPF. See .c file.
+# funccount Count kernel function calls.
+# For Linux, uses BCC, eBPF. See .c file.
#
# USAGE: funccount [-h] [-p PID] [-i INTERVAL] [-T] [-r] pattern
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 09-Sep-2015 Brendan Gregg Created this.
+# 09-Sep-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
./funccount -p 185 'vfs_*' # count vfs calls for PID 181 only
"""
parser = argparse.ArgumentParser(
- description="Count kernel function calls",
- formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog=examples)
+ description="Count kernel function calls",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=examples)
parser.add_argument("-p", "--pid",
- help="trace this PID only")
+ help="trace this PID only")
parser.add_argument("-i", "--interval", default=99999999,
- help="summary interval, seconds")
+ help="summary interval, seconds")
parser.add_argument("-T", "--timestamp", action="store_true",
- help="include timestamp on output")
+ help="include timestamp on output")
parser.add_argument("-r", "--regexp", action="store_true",
- help="use regular expressions. Default is \"*\" wildcards only.")
+ help="use regular expressions. Default is \"*\" wildcards only.")
parser.add_argument("pattern",
- help="search expression for kernel functions")
+ help="search expression for kernel functions")
args = parser.parse_args()
pattern = args.pattern
if not args.regexp:
- pattern = pattern.replace('*', '.*')
- pattern = '^' + pattern + '$'
+ pattern = pattern.replace('*', '.*')
+ pattern = '^' + pattern + '$'
debug = 0
# signal handler
def signal_ignore(signal, frame):
- print()
+ print()
# load BPF program
bpf_text = """
#include <uapi/linux/ptrace.h>
struct key_t {
- u64 ip;
+ u64 ip;
};
BPF_HASH(counts, struct key_t);
int trace_count(struct pt_regs *ctx) {
- FILTER
- struct key_t key = {};
- u64 zero = 0, *val;
- key.ip = ctx->ip;
- val = counts.lookup_or_init(&key, &zero);
- (*val)++;
- return 0;
+ FILTER
+ struct key_t key = {};
+ u64 zero = 0, *val;
+ key.ip = ctx->ip;
+ val = counts.lookup_or_init(&key, &zero);
+ (*val)++;
+ return 0;
}
"""
if args.pid:
- bpf_text = bpf_text.replace('FILTER',
- ('u32 pid; pid = bpf_get_current_pid_tgid(); ' +
- 'if (pid != %s) { return 0; }') % (args.pid))
+ bpf_text = bpf_text.replace('FILTER',
+ ('u32 pid; pid = bpf_get_current_pid_tgid(); ' +
+ 'if (pid != %s) { return 0; }') % (args.pid))
else:
- bpf_text = bpf_text.replace('FILTER', '')
+ bpf_text = bpf_text.replace('FILTER', '')
if debug:
- print(bpf_text)
+ print(bpf_text)
b = BPF(text=bpf_text)
b.attach_kprobe(event_re=pattern, fn_name="trace_count")
matched = b.num_open_kprobes()
if matched == 0:
- print("0 functions matched by \"%s\". Exiting." % args.pattern)
- exit()
+ print("0 functions matched by \"%s\". Exiting." % args.pattern)
+ exit()
# header
print("Tracing %d functions for \"%s\"... Hit Ctrl-C to end." %
# output
exiting = 0 if args.interval else 1
while (1):
- try:
- sleep(int(args.interval))
- except KeyboardInterrupt:
- exiting=1
- # as cleanup can take many seconds, trap Ctrl-C:
- signal.signal(signal.SIGINT, signal_ignore)
+ try:
+ sleep(int(args.interval))
+ except KeyboardInterrupt:
+ exiting = 1
+ # as cleanup can take many seconds, trap Ctrl-C:
+ signal.signal(signal.SIGINT, signal_ignore)
- print()
- if args.timestamp:
- print("%-8s\n" % strftime("%H:%M:%S"), end="")
+ print()
+ if args.timestamp:
+ print("%-8s\n" % strftime("%H:%M:%S"), end="")
- print("%-16s %-26s %8s" % ("ADDR", "FUNC", "COUNT"))
- counts = b.get_table("counts")
- for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
- print("%-16x %-26s %8d" % (k.ip, b.ksym(k.ip), v.value))
- counts.clear()
+ print("%-16s %-26s %8s" % ("ADDR", "FUNC", "COUNT"))
+ counts = b.get_table("counts")
+ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
+ print("%-16x %-26s %8d" % (k.ip, b.ksym(k.ip), v.value))
+ counts.clear()
- if exiting:
- print("Detaching...")
- exit()
+ if exiting:
+ print("Detaching...")
+ exit()
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# funclatency Time kernel funcitons and print latency as a histogram.
-# For Linux, uses BCC, eBPF.
+# funclatency Time kernel funcitons and print latency as a histogram.
+# For Linux, uses BCC, eBPF.
#
# USAGE: funclatency [-h] [-p PID] [-i INTERVAL] [-T] [-u] [-m] [-r] pattern
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 20-Sep-2015 Brendan Gregg Created this.
+# 20-Sep-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
./funclatency -F 'vfs_r*' # show one histogram per matched function
"""
parser = argparse.ArgumentParser(
- description="Time kernel funcitons and print latency as a histogram",
- formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog=examples)
+ description="Time kernel funcitons and print latency as a histogram",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=examples)
parser.add_argument("-p", "--pid",
- help="trace this PID only")
+ help="trace this PID only")
parser.add_argument("-i", "--interval", default=99999999,
- help="summary interval, seconds")
+ help="summary interval, seconds")
parser.add_argument("-T", "--timestamp", action="store_true",
- help="include timestamp on output")
+ help="include timestamp on output")
parser.add_argument("-u", "--microseconds", action="store_true",
- help="microsecond histogram")
+ help="microsecond histogram")
parser.add_argument("-m", "--milliseconds", action="store_true",
- help="millisecond histogram")
+ help="millisecond histogram")
parser.add_argument("-F", "--function", action="store_true",
- help="show a separate histogram per function")
+ help="show a separate histogram per function")
parser.add_argument("-r", "--regexp", action="store_true",
- help="use regular expressions. Default is \"*\" wildcards only.")
+ help="use regular expressions. Default is \"*\" wildcards only.")
parser.add_argument("pattern",
- help="search expression for kernel functions")
+ help="search expression for kernel functions")
args = parser.parse_args()
pattern = args.pattern
if not args.regexp:
- pattern = pattern.replace('*', '.*')
- pattern = '^' + pattern + '$'
+ pattern = pattern.replace('*', '.*')
+ pattern = '^' + pattern + '$'
debug = 0
# define BPF program
#include <linux/blkdev.h>
typedef struct ip_key {
- u64 ip;
- u64 slot;
+ u64 ip;
+ u64 slot;
} ip_key_t;
BPF_HASH(start, u32);
int trace_func_entry(struct pt_regs *ctx)
{
- u32 pid = bpf_get_current_pid_tgid();
- u64 ts = bpf_ktime_get_ns();
+ u32 pid = bpf_get_current_pid_tgid();
+ u64 ts = bpf_ktime_get_ns();
- FILTER
- ENTRYSTORE
- start.update(&pid, &ts);
+ FILTER
+ ENTRYSTORE
+ start.update(&pid, &ts);
- return 0;
+ return 0;
}
int trace_func_return(struct pt_regs *ctx)
{
- u64 *tsp, delta;
- u32 pid = bpf_get_current_pid_tgid();
-
- // calculate delta time
- tsp = start.lookup(&pid);
- if (tsp == 0) {
- return 0; // missed start
- }
- delta = bpf_ktime_get_ns() - *tsp;
- start.delete(&pid);
- FACTOR
-
- // store as histogram
- STORE
-
- return 0;
+ u64 *tsp, delta;
+ u32 pid = bpf_get_current_pid_tgid();
+
+ // calculate delta time
+ tsp = start.lookup(&pid);
+ if (tsp == 0) {
+ return 0; // missed start
+ }
+ delta = bpf_ktime_get_ns() - *tsp;
+ start.delete(&pid);
+ FACTOR
+
+ // store as histogram
+ STORE
+
+ return 0;
}
"""
# code substitutions
if args.pid:
- bpf_text = bpf_text.replace('FILTER',
- 'if (pid != %s) { return 0; }' % args.pid)
+ bpf_text = bpf_text.replace('FILTER',
+ 'if (pid != %s) { return 0; }' % args.pid)
else:
- bpf_text = bpf_text.replace('FILTER', '')
+ bpf_text = bpf_text.replace('FILTER', '')
if args.milliseconds:
- bpf_text = bpf_text.replace('FACTOR', 'delta /= 1000000;')
- label = "msecs"
+ bpf_text = bpf_text.replace('FACTOR', 'delta /= 1000000;')
+ label = "msecs"
elif args.microseconds:
- bpf_text = bpf_text.replace('FACTOR', 'delta /= 1000;')
- label = "usecs"
+ bpf_text = bpf_text.replace('FACTOR', 'delta /= 1000;')
+ label = "usecs"
else:
- bpf_text = bpf_text.replace('FACTOR', '')
- label = "nsecs"
+ bpf_text = bpf_text.replace('FACTOR', '')
+ label = "nsecs"
if args.function:
- bpf_text = bpf_text.replace('STORAGE', 'BPF_HASH(ipaddr, u32);\n' +
- 'BPF_HISTOGRAM(dist, ip_key_t);')
- # stash the IP on entry, as on return it's kretprobe_trampoline:
- bpf_text = bpf_text.replace('ENTRYSTORE',
- 'u64 ip = ctx->ip; ipaddr.update(&pid, &ip);')
- bpf_text = bpf_text.replace('STORE',
- 'u64 ip, *ipp = ipaddr.lookup(&pid); if (ipp) { ip = *ipp; ' +
- 'dist.increment((ip_key_t){ip, bpf_log2l(delta)}); ' +
- 'ipaddr.delete(&pid); }')
+ bpf_text = bpf_text.replace('STORAGE', 'BPF_HASH(ipaddr, u32);\n' +
+ 'BPF_HISTOGRAM(dist, ip_key_t);')
+ # stash the IP on entry, as on return it's kretprobe_trampoline:
+ bpf_text = bpf_text.replace('ENTRYSTORE',
+ 'u64 ip = ctx->ip; ipaddr.update(&pid, &ip);')
+ bpf_text = bpf_text.replace('STORE',
+ 'u64 ip, *ipp = ipaddr.lookup(&pid); if (ipp) { ip = *ipp; ' +
+ 'dist.increment((ip_key_t){ip, bpf_log2l(delta)}); ' +
+ 'ipaddr.delete(&pid); }')
else:
- bpf_text = bpf_text.replace('STORAGE', 'BPF_HISTOGRAM(dist);')
- bpf_text = bpf_text.replace('ENTRYSTORE', '')
- bpf_text = bpf_text.replace('STORE',
- 'dist.increment(bpf_log2l(delta));')
+ bpf_text = bpf_text.replace('STORAGE', 'BPF_HISTOGRAM(dist);')
+ bpf_text = bpf_text.replace('ENTRYSTORE', '')
+ bpf_text = bpf_text.replace('STORE',
+ 'dist.increment(bpf_log2l(delta));')
if debug:
- print(bpf_text)
+ print(bpf_text)
# signal handler
def signal_ignore(signal, frame):
- print()
+ print()
# load BPF program
b = BPF(text=bpf_text)
b.attach_kretprobe(event_re=pattern, fn_name="trace_func_return")
matched = b.num_open_kprobes()
if matched == 0:
- print("0 functions matched by \"%s\". Exiting." % args.pattern)
- exit()
+ print("0 functions matched by \"%s\". Exiting." % args.pattern)
+ exit()
# header
print("Tracing %d functions for \"%s\"... Hit Ctrl-C to end." %
exiting = 0 if args.interval else 1
dist = b.get_table("dist")
while (1):
- try:
- sleep(int(args.interval))
- except KeyboardInterrupt:
- exiting=1
- # as cleanup can take many seconds, trap Ctrl-C:
- signal.signal(signal.SIGINT, signal_ignore)
-
- print()
- if args.timestamp:
- print("%-8s\n" % strftime("%H:%M:%S"), end="")
-
- if args.function:
- dist.print_log2_hist(label, "Function", BPF.ksym)
- else:
- dist.print_log2_hist(label)
- dist.clear()
-
- if exiting:
- print("Detaching...")
- exit()
+ try:
+ sleep(int(args.interval))
+ except KeyboardInterrupt:
+ exiting = 1
+ # as cleanup can take many seconds, trap Ctrl-C:
+ signal.signal(signal.SIGINT, signal_ignore)
+
+ print()
+ if args.timestamp:
+ print("%-8s\n" % strftime("%H:%M:%S"), end="")
+
+ if args.function:
+ dist.print_log2_hist(label, "Function", BPF.ksym)
+ else:
+ dist.print_log2_hist(label)
+ dist.clear()
+
+ if exiting:
+ print("Detaching...")
+ exit()
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# hardirqs Summarize hard IRQ (interrupt) event time.
-# For Linux, uses BCC, eBPF.
+# hardirqs Summarize hard IRQ (interrupt) event time.
+# For Linux, uses BCC, eBPF.
#
# USAGE: hardirqs [-h] [-T] [-Q] [-m] [-D] [interval] [count]
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 19-Oct-2015 Brendan Gregg Created this.
+# 19-Oct-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
./hardirqs -NT 1 # 1s summaries, nanoseconds, and timestamps
"""
parser = argparse.ArgumentParser(
- description="Summarize hard irq event time as histograms",
- formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog=examples)
+ description="Summarize hard irq event time as histograms",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=examples)
parser.add_argument("-T", "--timestamp", action="store_true",
- help="include timestamp on output")
+ help="include timestamp on output")
parser.add_argument("-N", "--nanoseconds", action="store_true",
- help="output in nanoseconds")
+ help="output in nanoseconds")
parser.add_argument("-d", "--dist", action="store_true",
- help="show distributions as histograms")
+ help="show distributions as histograms")
parser.add_argument("interval", nargs="?", default=99999999,
- help="output interval, in seconds")
+ help="output interval, in seconds")
parser.add_argument("count", nargs="?", default=99999999,
- help="number of outputs")
+ help="number of outputs")
args = parser.parse_args()
countdown = int(args.count)
if args.nanoseconds:
- factor = 1
- label = "nsecs"
+ factor = 1
+ label = "nsecs"
else:
- factor = 1000
- label = "usecs"
+ factor = 1000
+ label = "usecs"
debug = 0
### define BPF program
#include <linux/interrupt.h>
typedef struct irq_key {
- char name[32];
- u64 slot;
+ char name[32];
+ u64 slot;
} irq_key_t;
BPF_HASH(start, u32);
BPF_HASH(irqdesc, u32, struct irq_desc *);
// time IRQ
int trace_start(struct pt_regs *ctx, struct irq_desc *desc)
{
- u32 pid = bpf_get_current_pid_tgid();
- u64 ts = bpf_ktime_get_ns();
- start.update(&pid, &ts);
- irqdesc.update(&pid, &desc);
- return 0;
+ u32 pid = bpf_get_current_pid_tgid();
+ u64 ts = bpf_ktime_get_ns();
+ start.update(&pid, &ts);
+ irqdesc.update(&pid, &desc);
+ return 0;
}
int trace_completion(struct pt_regs *ctx)
{
- u64 *tsp, delta;
- struct irq_desc **descp;
- u32 pid = bpf_get_current_pid_tgid();
-
- // fetch timestamp and calculate delta
- tsp = start.lookup(&pid);
- descp = irqdesc.lookup(&pid);
- if (tsp == 0 || descp == 0) {
- return 0; // missed start
- }
- // Note: descp is a value from map, so '&' can be done without
- // probe_read, but the next level irqaction * needs a probe read.
- // Do these steps first after reading the map, otherwise some of these
- // pointers may get pushed onto the stack and verifier will fail.
- struct irqaction *action = 0;
- bpf_probe_read(&action, sizeof(action), &(*descp)->action);
- const char **namep = &action->name;
- char *name = 0;
- bpf_probe_read(&name, sizeof(name), namep);
- delta = bpf_ktime_get_ns() - *tsp;
-
- // store as sum or histogram
- STORE
-
- start.delete(&pid);
- irqdesc.delete(&pid);
- return 0;
+ u64 *tsp, delta;
+ struct irq_desc **descp;
+ u32 pid = bpf_get_current_pid_tgid();
+
+ // fetch timestamp and calculate delta
+ tsp = start.lookup(&pid);
+ descp = irqdesc.lookup(&pid);
+ if (tsp == 0 || descp == 0) {
+ return 0; // missed start
+ }
+ // Note: descp is a value from map, so '&' can be done without
+ // probe_read, but the next level irqaction * needs a probe read.
+ // Do these steps first after reading the map, otherwise some of these
+ // pointers may get pushed onto the stack and verifier will fail.
+ struct irqaction *action = 0;
+ bpf_probe_read(&action, sizeof(action), &(*descp)->action);
+ const char **namep = &action->name;
+ char *name = 0;
+ bpf_probe_read(&name, sizeof(name), namep);
+ delta = bpf_ktime_get_ns() - *tsp;
+
+ // store as sum or histogram
+ STORE
+
+ start.delete(&pid);
+ irqdesc.delete(&pid);
+ return 0;
}
"""
### code substitutions
if args.dist:
- bpf_text = bpf_text.replace('STORE',
- 'irq_key_t key = {.slot = bpf_log2l(delta)};' +
- 'bpf_probe_read(&key.name, sizeof(key.name), name);' +
- 'dist.increment(key);')
+ bpf_text = bpf_text.replace('STORE',
+ 'irq_key_t key = {.slot = bpf_log2l(delta)};' +
+ 'bpf_probe_read(&key.name, sizeof(key.name), name);' +
+ 'dist.increment(key);')
else:
- bpf_text = bpf_text.replace('STORE',
- 'irq_key_t key = {.slot = 0 /* ignore */};' +
- 'bpf_probe_read(&key.name, sizeof(key.name), name);' +
- 'u64 zero = 0, *vp = dist.lookup_or_init(&key, &zero);' +
- '(*vp) += delta;')
+ bpf_text = bpf_text.replace('STORE',
+ 'irq_key_t key = {.slot = 0 /* ignore */};' +
+ 'bpf_probe_read(&key.name, sizeof(key.name), name);' +
+ 'u64 zero = 0, *vp = dist.lookup_or_init(&key, &zero);' +
+ '(*vp) += delta;')
if debug:
- print(bpf_text)
+ print(bpf_text)
### load BPF program
b = BPF(text=bpf_text)
exiting = 0 if args.interval else 1
dist = b.get_table("dist")
while (1):
- try:
- sleep(int(args.interval))
- except KeyboardInterrupt:
- exiting=1
-
- print()
- if args.timestamp:
- print("%-8s\n" % strftime("%H:%M:%S"), end="")
-
- if args.dist:
- dist.print_log2_hist(label, "hardirq")
- else:
- print("%-26s %11s" % ("HARDIRQ", "TOTAL_" + label))
- for k, v in sorted(dist.items(), key=lambda dist: dist[1].value):
- print("%-26s %11d" % (k.name, v.value / factor))
- dist.clear()
-
- countdown -= 1
- if exiting or countdown == 0:
- exit()
+ try:
+ sleep(int(args.interval))
+ except KeyboardInterrupt:
+ exiting = 1
+
+ print()
+ if args.timestamp:
+ print("%-8s\n" % strftime("%H:%M:%S"), end="")
+
+ if args.dist:
+ dist.print_log2_hist(label, "hardirq")
+ else:
+ print("%-26s %11s" % ("HARDIRQ", "TOTAL_" + label))
+ for k, v in sorted(dist.items(), key=lambda dist: dist[1].value):
+ print("%-26s %11d" % (k.name, v.value / factor))
+ dist.clear()
+
+ countdown -= 1
+ if exiting or countdown == 0:
+ exit()
This program traces hard interrupts (irqs), and stores timing statistics
in-kernel for efficiency. For example:
-# ./hardirqs
+# ./hardirqs
Tracing hard irq event time... Hit Ctrl-C to end.
^C
HARDIRQ TOTAL_usecs
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# killsnoop Trace signals issued by the kill() syscall.
-# For Linux, uses BCC, eBPF. Embedded C.
+# killsnoop Trace signals issued by the kill() syscall.
+# For Linux, uses BCC, eBPF. Embedded C.
#
# USAGE: killsnoop [-h] [-t] [-x] [-p PID]
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 20-Sep-2015 Brendan Gregg Created this.
+# 20-Sep-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
./killsnoop -p 181 # only trace PID 181
"""
parser = argparse.ArgumentParser(
- description="Trace signals issued by the kill() syscall",
- formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog=examples)
+ description="Trace signals issued by the kill() syscall",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=examples)
parser.add_argument("-t", "--timestamp", action="store_true",
- help="include timestamp on output")
+ help="include timestamp on output")
parser.add_argument("-x", "--failed", action="store_true",
- help="only show failed opens")
+ help="only show failed opens")
parser.add_argument("-p", "--pid",
- help="trace this PID only")
+ help="trace this PID only")
args = parser.parse_args()
debug = 0
int kprobe__sys_kill(struct pt_regs *ctx, int tpid, int sig)
{
- u32 pid = bpf_get_current_pid_tgid();
+ u32 pid = bpf_get_current_pid_tgid();
- FILTER
- args_pid.update(&pid, &tpid);
- args_sig.update(&pid, &sig);
+ FILTER
+ args_pid.update(&pid, &tpid);
+ args_sig.update(&pid, &sig);
- return 0;
+ return 0;
};
int kretprobe__sys_kill(struct pt_regs *ctx)
{
- int *tpidp, *sigp, ret = ctx->ax;
- u32 pid = bpf_get_current_pid_tgid();
+ int *tpidp, *sigp, ret = ctx->ax;
+ u32 pid = bpf_get_current_pid_tgid();
- tpidp = args_pid.lookup(&pid);
- sigp = args_sig.lookup(&pid);
- if (tpidp == 0 || sigp == 0) {
- return 0; // missed entry
- }
+ tpidp = args_pid.lookup(&pid);
+ sigp = args_sig.lookup(&pid);
+ if (tpidp == 0 || sigp == 0) {
+ return 0; // missed entry
+ }
- bpf_trace_printk("%d %d %d\\n", *tpidp, *sigp, ret);
- args_pid.delete(&pid);
- args_sig.delete(&pid);
+ bpf_trace_printk("%d %d %d\\n", *tpidp, *sigp, ret);
+ args_pid.delete(&pid);
+ args_sig.delete(&pid);
- return 0;
+ return 0;
}
"""
if args.pid:
- bpf_text = bpf_text.replace('FILTER',
- 'if (pid != %s) { return 0; }' % args.pid)
+ bpf_text = bpf_text.replace('FILTER',
+ 'if (pid != %s) { return 0; }' % args.pid)
else:
- bpf_text = bpf_text.replace('FILTER', '')
+ bpf_text = bpf_text.replace('FILTER', '')
if debug:
- print(bpf_text)
+ print(bpf_text)
# initialize BPF
b = BPF(text=bpf_text)
# header
if args.timestamp:
- print("%-14s" % ("TIME(s)"), end="")
+ print("%-14s" % ("TIME(s)"), end="")
print("%-6s %-16s %-4s %-6s %s" % ("PID", "COMM", "SIG", "TPID", "RESULT"))
start_ts = 0
# format output
while 1:
- (task, pid, cpu, flags, ts, msg) = b.trace_fields()
- (tpid_s, sig_s, ret_s) = msg.split(" ")
-
- ret = int(ret_s)
- if (args.failed and (ret >= 0)):
- continue
-
- # print columns
- if args.timestamp:
- if start_ts == 0:
- start_ts = ts
- print("%-14.9f" % (ts - start_ts), end="")
- print("%-6d %-16s %-4s %-6s %s" % (pid, task, sig_s, tpid_s, ret_s))
+ (task, pid, cpu, flags, ts, msg) = b.trace_fields()
+ (tpid_s, sig_s, ret_s) = msg.split(" ")
+
+ ret = int(ret_s)
+ if (args.failed and (ret >= 0)):
+ continue
+
+ # print columns
+ if args.timestamp:
+ if start_ts == 0:
+ start_ts = ts
+ print("%-14.9f" % (ts - start_ts), end="")
+ print("%-6d %-16s %-4s %-6s %s" % (pid, task, sig_s, tpid_s, ret_s))
This traces signals sent via the kill() syscall. For example:
-# ./killsnoop
+# ./killsnoop
PID COMM SIG TPID RESULT
17064 bash 9 27682 0
17064 bash 9 27682 -3
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# opensnoop Trace open() syscalls.
-# For Linux, uses BCC, eBPF. Embedded C.
+# opensnoop Trace open() syscalls.
+# For Linux, uses BCC, eBPF. Embedded C.
#
# USAGE: opensnoop [-h] [-t] [-x] [-p PID]
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 17-Sep-2015 Brendan Gregg Created this.
+# 17-Sep-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
./opensnoop -p 181 # only trace PID 181
"""
parser = argparse.ArgumentParser(
- description="Trace open() syscalls",
- formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog=examples)
+ description="Trace open() syscalls",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=examples)
parser.add_argument("-t", "--timestamp", action="store_true",
- help="include timestamp on output")
+ help="include timestamp on output")
parser.add_argument("-x", "--failed", action="store_true",
- help="only show failed opens")
+ help="only show failed opens")
parser.add_argument("-p", "--pid",
- help="trace this PID only")
+ help="trace this PID only")
args = parser.parse_args()
debug = 0
int kprobe__sys_open(struct pt_regs *ctx, const char __user *filename)
{
- u32 pid = bpf_get_current_pid_tgid();
+ u32 pid = bpf_get_current_pid_tgid();
- FILTER
- args_filename.update(&pid, &filename);
+ FILTER
+ args_filename.update(&pid, &filename);
- return 0;
+ return 0;
};
int kretprobe__sys_open(struct pt_regs *ctx)
{
- const char **filenamep;
- int ret = ctx->ax;
- u32 pid = bpf_get_current_pid_tgid();
+ const char **filenamep;
+ int ret = ctx->ax;
+ u32 pid = bpf_get_current_pid_tgid();
- filenamep = args_filename.lookup(&pid);
- if (filenamep == 0) {
- // missed entry
- return 0;
- }
+ filenamep = args_filename.lookup(&pid);
+ if (filenamep == 0) {
+ // missed entry
+ return 0;
+ }
- bpf_trace_printk("%s %d\\n", *filenamep, ret);
- args_filename.delete(&pid);
+ bpf_trace_printk("%s %d\\n", *filenamep, ret);
+ args_filename.delete(&pid);
- return 0;
+ return 0;
}
"""
if args.pid:
- bpf_text = bpf_text.replace('FILTER',
- 'if (pid != %s) { return 0; }' % args.pid)
+ bpf_text = bpf_text.replace('FILTER',
+ 'if (pid != %s) { return 0; }' % args.pid)
else:
- bpf_text = bpf_text.replace('FILTER', '')
+ bpf_text = bpf_text.replace('FILTER', '')
if debug:
- print(bpf_text)
+ print(bpf_text)
# initialize BPF
b = BPF(text=bpf_text)
# header
if args.timestamp:
- print("%-14s" % ("TIME(s)"), end="")
+ print("%-14s" % ("TIME(s)"), end="")
print("%-6s %-16s %4s %3s %s" % ("PID", "COMM", "FD", "ERR", "PATH"))
start_ts = 0
# format output
while 1:
- (task, pid, cpu, flags, ts, msg) = b.trace_fields()
- (filename, ret_s) = msg.split(" ")
-
- ret = int(ret_s)
- if (args.failed and (ret >= 0)):
- continue
-
- # split return value into FD and errno columns
- if ret >= 0:
- fd_s = ret; err = 0
- else:
- fd_s = "-1"; err = - ret
-
- # print columns
- if args.timestamp:
- if start_ts == 0:
- start_ts = ts
- print("%-14.9f" % (ts - start_ts), end="")
- print("%-6d %-16s %4s %3s %s" % (pid, task, fd_s, err, filename))
+ (task, pid, cpu, flags, ts, msg) = b.trace_fields()
+ (filename, ret_s) = msg.split(" ")
+
+ ret = int(ret_s)
+ if (args.failed and (ret >= 0)):
+ continue
+
+ # split return value into FD and errno columns
+ if ret >= 0:
+ fd_s = ret
+ err = 0
+ else:
+ fd_s = "-1"
+ err = - ret
+
+ # print columns
+ if args.timestamp:
+ if start_ts == 0:
+ start_ts = ts
+ print("%-14.9f" % (ts - start_ts), end="")
+ print("%-6d %-16s %4s %3s %s" % (pid, task, fd_s, err, filename))
Example output:
# ./opensnoop
-PID COMM FD ERR PATH
-17326 <...> 7 0 /sys/kernel/debug/tracing/trace_pipe
-1576 snmpd 9 0 /proc/net/dev
-1576 snmpd 11 0 /proc/net/if_inet6
-1576 snmpd 11 0 /proc/sys/net/ipv4/neigh/eth0/retrans_time_ms
-1576 snmpd 11 0 /proc/sys/net/ipv6/neigh/eth0/retrans_time_ms
-1576 snmpd 11 0 /proc/sys/net/ipv6/conf/eth0/forwarding
-1576 snmpd 11 0 /proc/sys/net/ipv6/neigh/eth0/base_reachable_time_ms
-1576 snmpd 11 0 /proc/sys/net/ipv4/neigh/lo/retrans_time_ms
-1576 snmpd 11 0 /proc/sys/net/ipv6/neigh/lo/retrans_time_ms
-1576 snmpd 11 0 /proc/sys/net/ipv6/conf/lo/forwarding
-1576 snmpd 11 0 /proc/sys/net/ipv6/neigh/lo/base_reachable_time_ms
-1576 snmpd 9 0 /proc/diskstats
-1576 snmpd 9 0 /proc/stat
-1576 snmpd 9 0 /proc/vmstat
-1956 supervise 9 0 supervise/status.new
-1956 supervise 9 0 supervise/status.new
-17358 run 3 0 /etc/ld.so.cache
-17358 run 3 0 /lib/x86_64-linux-gnu/libtinfo.so.5
-17358 run 3 0 /lib/x86_64-linux-gnu/libdl.so.2
-17358 run 3 0 /lib/x86_64-linux-gnu/libc.so.6
-17358 run -1 6 /dev/tty
-17358 run 3 0 /proc/meminfo
-17358 run 3 0 /etc/nsswitch.conf
-17358 run 3 0 /etc/ld.so.cache
-17358 run 3 0 /lib/x86_64-linux-gnu/libnss_compat.so.2
-17358 run 3 0 /lib/x86_64-linux-gnu/libnsl.so.1
-17358 run 3 0 /etc/ld.so.cache
-17358 run 3 0 /lib/x86_64-linux-gnu/libnss_nis.so.2
-17358 run 3 0 /lib/x86_64-linux-gnu/libnss_files.so.2
-17358 run 3 0 /etc/passwd
-17358 run 3 0 ./run
+PID COMM FD ERR PATH
+17326 <...> 7 0 /sys/kernel/debug/tracing/trace_pipe
+1576 snmpd 9 0 /proc/net/dev
+1576 snmpd 11 0 /proc/net/if_inet6
+1576 snmpd 11 0 /proc/sys/net/ipv4/neigh/eth0/retrans_time_ms
+1576 snmpd 11 0 /proc/sys/net/ipv6/neigh/eth0/retrans_time_ms
+1576 snmpd 11 0 /proc/sys/net/ipv6/conf/eth0/forwarding
+1576 snmpd 11 0 /proc/sys/net/ipv6/neigh/eth0/base_reachable_time_ms
+1576 snmpd 11 0 /proc/sys/net/ipv4/neigh/lo/retrans_time_ms
+1576 snmpd 11 0 /proc/sys/net/ipv6/neigh/lo/retrans_time_ms
+1576 snmpd 11 0 /proc/sys/net/ipv6/conf/lo/forwarding
+1576 snmpd 11 0 /proc/sys/net/ipv6/neigh/lo/base_reachable_time_ms
+1576 snmpd 9 0 /proc/diskstats
+1576 snmpd 9 0 /proc/stat
+1576 snmpd 9 0 /proc/vmstat
+1956 supervise 9 0 supervise/status.new
+1956 supervise 9 0 supervise/status.new
+17358 run 3 0 /etc/ld.so.cache
+17358 run 3 0 /lib/x86_64-linux-gnu/libtinfo.so.5
+17358 run 3 0 /lib/x86_64-linux-gnu/libdl.so.2
+17358 run 3 0 /lib/x86_64-linux-gnu/libc.so.6
+17358 run -1 6 /dev/tty
+17358 run 3 0 /proc/meminfo
+17358 run 3 0 /etc/nsswitch.conf
+17358 run 3 0 /etc/ld.so.cache
+17358 run 3 0 /lib/x86_64-linux-gnu/libnss_compat.so.2
+17358 run 3 0 /lib/x86_64-linux-gnu/libnsl.so.1
+17358 run 3 0 /etc/ld.so.cache
+17358 run 3 0 /lib/x86_64-linux-gnu/libnss_nis.so.2
+17358 run 3 0 /lib/x86_64-linux-gnu/libnss_files.so.2
+17358 run 3 0 /etc/passwd
+17358 run 3 0 ./run
^C
While tracing, the snmpd process opened various /proc files (reading metrics),
The -x option only prints failed opens:
# ./opensnoop -x
-PID COMM FD ERR PATH
-18372 run -1 6 /dev/tty
-18373 run -1 6 /dev/tty
-18373 multilog -1 13 lock
-18372 multilog -1 13 lock
-18384 df -1 2 /usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale/en_US/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale/en/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale-langpack/en_US.UTF-8/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale-langpack/en_US.utf8/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale-langpack/en_US/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale-langpack/en.UTF-8/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale-langpack/en.utf8/LC_MESSAGES/coreutils.mo
-18384 df -1 2 /usr/share/locale-langpack/en/LC_MESSAGES/coreutils.mo
-18385 run -1 6 /dev/tty
-18386 run -1 6 /dev/tty
+PID COMM FD ERR PATH
+18372 run -1 6 /dev/tty
+18373 run -1 6 /dev/tty
+18373 multilog -1 13 lock
+18372 multilog -1 13 lock
+18384 df -1 2 /usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo
+18384 df -1 2 /usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo
+18384 df -1 2 /usr/share/locale/en_US/LC_MESSAGES/coreutils.mo
+18384 df -1 2 /usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo
+18384 df -1 2 /usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo
+18384 df -1 2 /usr/share/locale/en/LC_MESSAGES/coreutils.mo
+18385 run -1 6 /dev/tty
+18386 run -1 6 /dev/tty
This caught a df command failing to open a coreutils.mo file, and trying from
different directories.
#!/usr/bin/python
-# vim: ts=8 noet sw=8
+# @lint-avoid-python-3-compatibility-imports
#
-# pidpersec Count new processes (via fork).
-# For Linux, uses BCC, eBPF. See .c file.
+# pidpersec Count new processes (via fork).
+# For Linux, uses BCC, eBPF. See .c file.
#
# USAGE: pidpersec
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 11-Aug-2015 Brendan Gregg Created this.
+# 11-Aug-2015 Brendan Gregg Created this.
from bcc import BPF
-from ctypes import c_ushort, c_int, c_ulonglong
+from ctypes import c_int
from time import sleep, strftime
# load BPF program
-b = BPF(src_file = "pidpersec.c")
+b = BPF(src_file="pidpersec.c")
b.attach_kprobe(event="sched_fork", fn_name="do_count")
# stat indexes
# output
while (1):
- try:
- sleep(1)
- except KeyboardInterrupt:
- exit()
+ try:
+ sleep(1)
+ except KeyboardInterrupt:
+ exit()
- print("%s: PIDs/sec: %d" % (strftime("%H:%M:%S"),
- b["stats"][S_COUNT].value))
- b["stats"].clear()
+ print("%s: PIDs/sec: %d" % (strftime("%H:%M:%S"),
+ b["stats"][S_COUNT].value))
+ b["stats"].clear()
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# softirqs Summarize soft IRQ (interrupt) event time.
-# For Linux, uses BCC, eBPF.
+# softirqs Summarize soft IRQ (interrupt) event time.
+# For Linux, uses BCC, eBPF.
#
# USAGE: softirqs [-h] [-T] [-N] [-d] [interval] [count]
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 20-Oct-2015 Brendan Gregg Created this.
+# 20-Oct-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
./softirqs -NT 1 # 1s summaries, nanoseconds, and timestamps
"""
parser = argparse.ArgumentParser(
- description="Summarize soft irq event time as histograms",
- formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog=examples)
+ description="Summarize soft irq event time as histograms",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=examples)
parser.add_argument("-T", "--timestamp", action="store_true",
- help="include timestamp on output")
+ help="include timestamp on output")
parser.add_argument("-N", "--nanoseconds", action="store_true",
- help="output in nanoseconds")
+ help="output in nanoseconds")
parser.add_argument("-d", "--dist", action="store_true",
- help="show distributions as histograms")
+ help="show distributions as histograms")
parser.add_argument("interval", nargs="?", default=99999999,
- help="output interval, in seconds")
+ help="output interval, in seconds")
parser.add_argument("count", nargs="?", default=99999999,
- help="number of outputs")
+ help="number of outputs")
args = parser.parse_args()
countdown = int(args.count)
if args.nanoseconds:
- factor = 1
- label = "nsecs"
+ factor = 1
+ label = "nsecs"
else:
- factor = 1000
- label = "usecs"
+ factor = 1000
+ label = "usecs"
debug = 0
### define BPF program
#include <uapi/linux/ptrace.h>
typedef struct irq_key {
- u64 ip;
- u64 slot;
+ u64 ip;
+ u64 slot;
} irq_key_t;
BPF_HASH(start, u32);
BPF_HASH(iptr, u32);
// time IRQ
int trace_start(struct pt_regs *ctx)
{
- u32 pid = bpf_get_current_pid_tgid();
- u64 ip = ctx->ip, ts = bpf_ktime_get_ns();
- start.update(&pid, &ts);
- iptr.update(&pid, &ip);
- return 0;
+ u32 pid = bpf_get_current_pid_tgid();
+ u64 ip = ctx->ip, ts = bpf_ktime_get_ns();
+ start.update(&pid, &ts);
+ iptr.update(&pid, &ip);
+ return 0;
}
int trace_completion(struct pt_regs *ctx)
{
- u64 *tsp, delta, ip, *ipp;
- u32 pid = bpf_get_current_pid_tgid();
-
- // fetch timestamp and calculate delta
- tsp = start.lookup(&pid);
- ipp = iptr.lookup(&pid);
- if (tsp == 0 || ipp == 0) {
- return 0; // missed start
- }
- delta = bpf_ktime_get_ns() - *tsp;
- ip = *ipp;
-
- // store as sum or histogram
- STORE
-
- start.delete(&pid);
- iptr.delete(&pid);
- return 0;
+ u64 *tsp, delta, ip, *ipp;
+ u32 pid = bpf_get_current_pid_tgid();
+
+ // fetch timestamp and calculate delta
+ tsp = start.lookup(&pid);
+ ipp = iptr.lookup(&pid);
+ if (tsp == 0 || ipp == 0) {
+ return 0; // missed start
+ }
+ delta = bpf_ktime_get_ns() - *tsp;
+ ip = *ipp;
+
+ // store as sum or histogram
+ STORE
+
+ start.delete(&pid);
+ iptr.delete(&pid);
+ return 0;
}
"""
### code substitutions
if args.dist:
- bpf_text = bpf_text.replace('STORE',
- 'irq_key_t key = {.ip = ip, .slot = bpf_log2l(delta)};' +
- 'dist.increment(key);')
+ bpf_text = bpf_text.replace('STORE',
+ 'irq_key_t key = {.ip = ip, .slot = bpf_log2l(delta)};' +
+ 'dist.increment(key);')
else:
- bpf_text = bpf_text.replace('STORE',
- 'irq_key_t key = {.ip = ip, .slot = 0 /* ignore */};' +
- 'u64 zero = 0, *vp = dist.lookup_or_init(&key, &zero);' +
- '(*vp) += delta;')
+ bpf_text = bpf_text.replace('STORE',
+ 'irq_key_t key = {.ip = ip, .slot = 0 /* ignore */};' +
+ 'u64 zero = 0, *vp = dist.lookup_or_init(&key, &zero);' +
+ '(*vp) += delta;')
if debug:
- print(bpf_text)
+ print(bpf_text)
### load BPF program
b = BPF(text=bpf_text)
"rcu_process_callbacks", "run_rebalance_domains", "tasklet_action",
"tasklet_hi_action", "run_timer_softirq", "net_tx_action",
"net_rx_action"):
- b.attach_kprobe(event=softirqfunc, fn_name="trace_start")
- b.attach_kretprobe(event=softirqfunc, fn_name="trace_completion")
+ b.attach_kprobe(event=softirqfunc, fn_name="trace_start")
+ b.attach_kretprobe(event=softirqfunc, fn_name="trace_completion")
print("Tracing soft irq event time... Hit Ctrl-C to end.")
exiting = 0 if args.interval else 1
dist = b.get_table("dist")
while (1):
- try:
- sleep(int(args.interval))
- except KeyboardInterrupt:
- exiting=1
-
- print()
- if args.timestamp:
- print("%-8s\n" % strftime("%H:%M:%S"), end="")
-
- if args.dist:
- dist.print_log2_hist(label, "softirq", section_print_fn=b.ksym)
- else:
- print("%-26s %11s" % ("SOFTIRQ", "TOTAL_" + label))
- for k, v in sorted(dist.items(), key=lambda dist: dist[1].value):
- print("%-26s %11d" % (b.ksym(k.ip), v.value / factor))
- dist.clear()
-
- countdown -= 1
- if exiting or countdown == 0:
- exit()
+ try:
+ sleep(int(args.interval))
+ except KeyboardInterrupt:
+ exiting = 1
+
+ print()
+ if args.timestamp:
+ print("%-8s\n" % strftime("%H:%M:%S"), end="")
+
+ if args.dist:
+ dist.print_log2_hist(label, "softirq", section_print_fn=b.ksym)
+ else:
+ print("%-26s %11s" % ("SOFTIRQ", "TOTAL_" + label))
+ for k, v in sorted(dist.items(), key=lambda dist: dist[1].value):
+ print("%-26s %11d" % (b.ksym(k.ip), v.value / factor))
+ dist.clear()
+
+ countdown -= 1
+ if exiting or countdown == 0:
+ exit()
This program traces soft interrupts (irqs), and stores timing statistics
in-kernel for efficiency. For example:
-# ./softirqs
+# ./softirqs
Tracing soft irq event time... Hit Ctrl-C to end.
^C
SOFTIRQ TOTAL_usecs
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# syncsnoop Trace sync() syscall.
-# For Linux, uses BCC, eBPF. Embedded C.
+# syncsnoop Trace sync() syscall.
+# For Linux, uses BCC, eBPF. Embedded C.
#
# Written as a basic example of BCC trace & reformat. See
# examples/hello_world.py for a BCC trace with default output example.
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 13-Aug-2015 Brendan Gregg Created this.
+# 13-Aug-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
# load BPF program
-b = BPF(text = """
+b = BPF(text="""
void kprobe__sys_sync(void *ctx) {
- bpf_trace_printk("sync()\\n");
+ bpf_trace_printk("sync()\\n");
};
""")
# format output
while 1:
- (task, pid, cpu, flags, ts, msg) = b.trace_fields()
- print("%-18.9f %s" % (ts, msg))
+ (task, pid, cpu, flags, ts, msg) = b.trace_fields()
+ print("%-18.9f %s" % (ts, msg))
This program traces calls to the kernel sync() routine, with basic timestamps:
-# ./syncsnoop
+# ./syncsnoop
TIME(s) CALL
16458148.611952 sync()
16458151.533709 sync()
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# tcpaccept Trace TCP accept()s.
-# For Linux, uses BCC, eBPF. Embedded C.
+# tcpaccept Trace TCP accept()s.
+# For Linux, uses BCC, eBPF. Embedded C.
#
# USAGE: tcpaccept [-h] [-t] [-p PID]
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 13-Oct-2015 Brendan Gregg Created this.
+# 13-Oct-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
./tcpaccept -p 181 # only trace PID 181
"""
parser = argparse.ArgumentParser(
- description="Trace TCP accepts",
- formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog=examples)
+ description="Trace TCP accepts",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=examples)
parser.add_argument("-t", "--timestamp", action="store_true",
- help="include timestamp on output")
+ help="include timestamp on output")
parser.add_argument("-p", "--pid",
- help="trace this PID only")
+ help="trace this PID only")
args = parser.parse_args()
debug = 0
int kretprobe__inet_csk_accept(struct pt_regs *ctx)
{
- struct sock *newsk = (struct sock *)ctx->ax;
- u32 pid = bpf_get_current_pid_tgid();
-
- if (newsk == NULL)
- return 0;
-
- // check this is TCP
- u8 protocol = 0;
- // workaround for reading the sk_protocol bitfield:
- bpf_probe_read(&protocol, 1, (void *)((long)&newsk->sk_wmem_queued) - 3);
- if (protocol != IPPROTO_TCP)
- return 0;
-
- // pull in details
- u16 family = 0, lport = 0;
- u32 saddr = 0, daddr = 0;
- bpf_probe_read(&family, sizeof(family), &newsk->__sk_common.skc_family);
- bpf_probe_read(&lport, sizeof(lport), &newsk->__sk_common.skc_num);
- if (family == AF_INET) {
- bpf_probe_read(&saddr, sizeof(saddr),
- &newsk->__sk_common.skc_rcv_saddr);
- bpf_probe_read(&daddr, sizeof(daddr),
- &newsk->__sk_common.skc_daddr);
-
- // output
- bpf_trace_printk("4 %x %x %d\\n", daddr, saddr, lport);
- } else if (family == AF_INET6) {
- // just grab the last 4 bytes for now
- bpf_probe_read(&saddr, sizeof(saddr),
- &newsk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[3]);
- bpf_probe_read(&daddr, sizeof(daddr),
- &newsk->__sk_common.skc_v6_daddr.in6_u.u6_addr32[3]);
-
- // output and flip byte order of addresses
- bpf_trace_printk("6 %x %x %d\\n", bpf_ntohl(daddr),
- bpf_ntohl(saddr), lport);
- }
- // else drop
-
- return 0;
+ struct sock *newsk = (struct sock *)ctx->ax;
+ u32 pid = bpf_get_current_pid_tgid();
+
+ if (newsk == NULL)
+ return 0;
+
+ // check this is TCP
+ u8 protocol = 0;
+ // workaround for reading the sk_protocol bitfield:
+ bpf_probe_read(&protocol, 1, (void *)((long)&newsk->sk_wmem_queued) - 3);
+ if (protocol != IPPROTO_TCP)
+ return 0;
+
+ // pull in details
+ u16 family = 0, lport = 0;
+ u32 saddr = 0, daddr = 0;
+ bpf_probe_read(&family, sizeof(family), &newsk->__sk_common.skc_family);
+ bpf_probe_read(&lport, sizeof(lport), &newsk->__sk_common.skc_num);
+ if (family == AF_INET) {
+ bpf_probe_read(&saddr, sizeof(saddr),
+ &newsk->__sk_common.skc_rcv_saddr);
+ bpf_probe_read(&daddr, sizeof(daddr),
+ &newsk->__sk_common.skc_daddr);
+
+ // output
+ bpf_trace_printk("4 %x %x %d\\n", daddr, saddr, lport);
+ } else if (family == AF_INET6) {
+ // just grab the last 4 bytes for now
+ bpf_probe_read(&saddr, sizeof(saddr),
+ &newsk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[3]);
+ bpf_probe_read(&daddr, sizeof(daddr),
+ &newsk->__sk_common.skc_v6_daddr.in6_u.u6_addr32[3]);
+
+ // output and flip byte order of addresses
+ bpf_trace_printk("6 %x %x %d\\n", bpf_ntohl(daddr),
+ bpf_ntohl(saddr), lport);
+ }
+ // else drop
+
+ return 0;
}
"""
# code substitutions
if args.pid:
- bpf_text = bpf_text.replace('FILTER',
- 'if (pid != %s) { return 0; }' % args.pid)
+ bpf_text = bpf_text.replace('FILTER',
+ 'if (pid != %s) { return 0; }' % args.pid)
else:
- bpf_text = bpf_text.replace('FILTER', '')
+ bpf_text = bpf_text.replace('FILTER', '')
if debug:
- print(bpf_text)
+ print(bpf_text)
# initialize BPF
b = BPF(text=bpf_text)
# header
if args.timestamp:
- print("%-9s" % ("TIME(s)"), end="")
+ print("%-9s" % ("TIME(s)"), end="")
print("%-6s %-12s %-2s %-16s %-16s %-4s" % ("PID", "COMM", "IP", "RADDR",
"LADDR", "LPORT"))
start_ts = 0
def inet_ntoa(addr):
- dq = ''
- for i in range(0, 4):
- dq = dq + str(addr & 0xff)
- if (i != 3):
- dq = dq + '.'
- addr = addr >> 8
- return dq
+ dq = ''
+ for i in range(0, 4):
+ dq = dq + str(addr & 0xff)
+ if (i != 3):
+ dq = dq + '.'
+ addr = addr >> 8
+ return dq
# format output
while 1:
- (task, pid, cpu, flags, ts, msg) = b.trace_fields()
- (ip_s, raddr_hs, laddr_hs, lport_s) = msg.split(" ")
-
- if args.timestamp:
- if start_ts == 0:
- start_ts = ts
- print("%-9.3f" % (ts - start_ts), end="")
- print("%-6d %-12.12s %-2s %-16s %-16s %-4s" % (pid, task, ip_s,
- inet_ntoa(int(raddr_hs, 16)) if ip_s == "4" else "..." + raddr_hs,
- inet_ntoa(int(laddr_hs, 16)) if ip_s == "4" else "..." + laddr_hs,
- lport_s))
+ (task, pid, cpu, flags, ts, msg) = b.trace_fields()
+ (ip_s, raddr_hs, laddr_hs, lport_s) = msg.split(" ")
+
+ if args.timestamp:
+ if start_ts == 0:
+ start_ts = ts
+ print("%-9.3f" % (ts - start_ts), end="")
+ print("%-6d %-12.12s %-2s %-16s %-16s %-4s" % (pid, task, ip_s,
+ inet_ntoa(int(raddr_hs, 16)) if ip_s == "4" else "..." + raddr_hs,
+ inet_ntoa(int(laddr_hs, 16)) if ip_s == "4" else "..." + laddr_hs,
+ lport_s))
# ./tcpaccept
PID COMM IP RADDR LADDR LPORT
-907 sshd 4 192.168.56.1 192.168.56.102 22
-907 sshd 4 127.0.0.1 127.0.0.1 22
+907 sshd 4 192.168.56.1 192.168.56.102 22
+907 sshd 4 127.0.0.1 127.0.0.1 22
5389 perl 6 ...fec0ae21 ...fec0ae21 7001
This output shows three connections, two to PID 907, an "sshd" process listening
# ./tcpaccept -t
TIME(s) PID COMM IP RADDR LADDR LPORT
-0.000 907 sshd 4 127.0.0.1 127.0.0.1 22
-0.992 907 sshd 4 127.0.0.1 127.0.0.1 22
-1.984 907 sshd 4 127.0.0.1 127.0.0.1 22
+0.000 907 sshd 4 127.0.0.1 127.0.0.1 22
+0.992 907 sshd 4 127.0.0.1 127.0.0.1 22
+1.984 907 sshd 4 127.0.0.1 127.0.0.1 22
USAGE message:
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# tcpconnect Trace TCP connect()s.
-# For Linux, uses BCC, eBPF. Embedded C.
+# tcpconnect Trace TCP connect()s.
+# For Linux, uses BCC, eBPF. Embedded C.
#
# USAGE: tcpconnect [-h] [-t] [-p PID]
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 25-Sep-2015 Brendan Gregg Created this.
+# 25-Sep-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
./tcpconnect -p 181 # only trace PID 181
"""
parser = argparse.ArgumentParser(
- description="Trace TCP connects",
- formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog=examples)
+ description="Trace TCP connects",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=examples)
parser.add_argument("-t", "--timestamp", action="store_true",
- help="include timestamp on output")
+ help="include timestamp on output")
parser.add_argument("-p", "--pid",
- help="trace this PID only")
+ help="trace this PID only")
args = parser.parse_args()
debug = 0
int trace_connect_entry(struct pt_regs *ctx, struct sock *sk)
{
- u32 pid = bpf_get_current_pid_tgid();
- FILTER
+ u32 pid = bpf_get_current_pid_tgid();
+ FILTER
- // stash the sock ptr for lookup on return
- currsock.update(&pid, &sk);
+ // stash the sock ptr for lookup on return
+ currsock.update(&pid, &sk);
- return 0;
+ return 0;
};
static int trace_connect_return(struct pt_regs *ctx, short ipver)
{
- int ret = ctx->ax;
- u32 pid = bpf_get_current_pid_tgid();
-
- struct sock **skpp;
- skpp = currsock.lookup(&pid);
- if (skpp == 0) {
- return 0; // missed entry
- }
-
- if (ret != 0) {
- // failed to send SYNC packet, may not have populated
- // socket __sk_common.{skc_rcv_saddr, ...}
- currsock.delete(&pid);
- return 0;
- }
-
- // pull in details
- struct sock *skp = *skpp;
- u32 saddr = 0, daddr = 0;
- u16 dport = 0;
- bpf_probe_read(&dport, sizeof(dport), &skp->__sk_common.skc_dport);
- if (ipver == 4) {
- bpf_probe_read(&saddr, sizeof(saddr),
- &skp->__sk_common.skc_rcv_saddr);
- bpf_probe_read(&daddr, sizeof(daddr),
- &skp->__sk_common.skc_daddr);
-
- // output
- bpf_trace_printk("4 %x %x %d\\n", saddr, daddr, ntohs(dport));
- } else /* 6 */ {
- // just grab the last 4 bytes for now
- bpf_probe_read(&saddr, sizeof(saddr),
- &skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[3]);
- bpf_probe_read(&daddr, sizeof(daddr),
- &skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32[3]);
-
- // output and flip byte order of addresses
- bpf_trace_printk("6 %x %x %d\\n", bpf_ntohl(saddr),
- bpf_ntohl(daddr), ntohs(dport));
- }
-
- currsock.delete(&pid);
-
- return 0;
+ int ret = ctx->ax;
+ u32 pid = bpf_get_current_pid_tgid();
+
+ struct sock **skpp;
+ skpp = currsock.lookup(&pid);
+ if (skpp == 0) {
+ return 0; // missed entry
+ }
+
+ if (ret != 0) {
+ // failed to send SYNC packet, may not have populated
+ // socket __sk_common.{skc_rcv_saddr, ...}
+ currsock.delete(&pid);
+ return 0;
+ }
+
+ // pull in details
+ struct sock *skp = *skpp;
+ u32 saddr = 0, daddr = 0;
+ u16 dport = 0;
+ bpf_probe_read(&dport, sizeof(dport), &skp->__sk_common.skc_dport);
+ if (ipver == 4) {
+ bpf_probe_read(&saddr, sizeof(saddr),
+ &skp->__sk_common.skc_rcv_saddr);
+ bpf_probe_read(&daddr, sizeof(daddr),
+ &skp->__sk_common.skc_daddr);
+
+ // output
+ bpf_trace_printk("4 %x %x %d\\n", saddr, daddr, ntohs(dport));
+ } else /* 6 */ {
+ // just grab the last 4 bytes for now
+ bpf_probe_read(&saddr, sizeof(saddr),
+ &skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[3]);
+ bpf_probe_read(&daddr, sizeof(daddr),
+ &skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32[3]);
+
+ // output and flip byte order of addresses
+ bpf_trace_printk("6 %x %x %d\\n", bpf_ntohl(saddr),
+ bpf_ntohl(daddr), ntohs(dport));
+ }
+
+ currsock.delete(&pid);
+
+ return 0;
}
int trace_connect_v4_return(struct pt_regs *ctx)
{
- return trace_connect_return(ctx, 4);
+ return trace_connect_return(ctx, 4);
}
int trace_connect_v6_return(struct pt_regs *ctx)
{
- return trace_connect_return(ctx, 6);
+ return trace_connect_return(ctx, 6);
}
"""
# code substitutions
if args.pid:
- bpf_text = bpf_text.replace('FILTER',
- 'if (pid != %s) { return 0; }' % args.pid)
+ bpf_text = bpf_text.replace('FILTER',
+ 'if (pid != %s) { return 0; }' % args.pid)
else:
- bpf_text = bpf_text.replace('FILTER', '')
+ bpf_text = bpf_text.replace('FILTER', '')
if debug:
- print(bpf_text)
+ print(bpf_text)
# initialize BPF
b = BPF(text=bpf_text)
# header
if args.timestamp:
- print("%-9s" % ("TIME(s)"), end="")
+ print("%-9s" % ("TIME(s)"), end="")
print("%-6s %-12s %-2s %-16s %-16s %-4s" % ("PID", "COMM", "IP", "SADDR",
"DADDR", "DPORT"))
start_ts = 0
def inet_ntoa(addr):
- dq = ''
- for i in range(0, 4):
- dq = dq + str(addr & 0xff)
- if (i != 3):
- dq = dq + '.'
- addr = addr >> 8
- return dq
+ dq = ''
+ for i in range(0, 4):
+ dq = dq + str(addr & 0xff)
+ if (i != 3):
+ dq = dq + '.'
+ addr = addr >> 8
+ return dq
# format output
while 1:
- (task, pid, cpu, flags, ts, msg) = b.trace_fields()
- (ip_s, saddr_hs, daddr_hs, dport_s) = msg.split(" ")
-
- if args.timestamp:
- if start_ts == 0:
- start_ts = ts
- print("%-9.3f" % (ts - start_ts), end="")
- print("%-6d %-12.12s %-2s %-16s %-16s %-4s" % (pid, task, ip_s,
- inet_ntoa(int(saddr_hs, 16)) if ip_s == "4" else "..." + saddr_hs,
- inet_ntoa(int(daddr_hs, 16)) if ip_s == "4" else "..." + daddr_hs,
- dport_s))
+ (task, pid, cpu, flags, ts, msg) = b.trace_fields()
+ (ip_s, saddr_hs, daddr_hs, dport_s) = msg.split(" ")
+
+ if args.timestamp:
+ if start_ts == 0:
+ start_ts = ts
+ print("%-9.3f" % (ts - start_ts), end="")
+ print("%-6d %-12.12s %-2s %-16s %-16s %-4s" % (pid, task, ip_s,
+ inet_ntoa(int(saddr_hs, 16)) if ip_s == "4" else "..." + saddr_hs,
+ inet_ntoa(int(daddr_hs, 16)) if ip_s == "4" else "..." + daddr_hs,
+ dport_s))
(eg, via a connect() syscall; accept() are passive connections). Some example
output (IP addresses changed to protect the innocent):
-# ./tcpconnect
+# ./tcpconnect
PID COMM IP SADDR DADDR DPORT
-1479 telnet 4 127.0.0.1 127.0.0.1 23
-1469 curl 4 10.201.219.236 54.245.105.25 80
-1469 curl 4 10.201.219.236 54.67.101.145 80
-11072 ssh 6 ...fe8203ac ...fe82abcd 22
+1479 telnet 4 127.0.0.1 127.0.0.1 23
+1469 curl 4 10.201.219.236 54.245.105.25 80
+1469 curl 4 10.201.219.236 54.67.101.145 80
+11072 ssh 6 ...fe8203ac ...fe82abcd 22
This output shows four connections, one from a "telnet" process, two from
"curl", and one from "ssh". The output details shows the IP version, source
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# vfscount Count VFS calls ("vfs_*").
-# For Linux, uses BCC, eBPF. See .c file.
+# vfscount Count VFS calls ("vfs_*").
+# For Linux, uses BCC, eBPF. See .c file.
#
# Written as a basic example of counting functions.
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 14-Aug-2015 Brendan Gregg Created this.
+# 14-Aug-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
from time import sleep
# load BPF program
-b = BPF(src_file = "vfscount.c")
+b = BPF(src_file="vfscount.c")
b.attach_kprobe(event_re="^vfs_.*", fn_name="do_count")
# header
# output
try:
- sleep(99999999)
+ sleep(99999999)
except KeyboardInterrupt:
- pass
+ pass
print("\n%-16s %-26s %8s" % ("ADDR", "FUNC", "COUNT"))
counts = b.get_table("counts")
for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
- print("%-16x %-26s %8d" % (k.ip, b.ksym(k.ip), v.value))
+ print("%-16x %-26s %8d" % (k.ip, b.ksym(k.ip), v.value))
This counts VFS calls, by tracing all kernel functions beginning with "vfs_":
-# ./vfscount
+# ./vfscount
Tracing... Ctrl-C to end.
^C
ADDR FUNC COUNT
#!/usr/bin/python
+# @lint-avoid-python-3-compatibility-imports
#
-# vfsstat Count some VFS calls.
-# For Linux, uses BCC, eBPF. See .c file.
+# vfsstat Count some VFS calls.
+# For Linux, uses BCC, eBPF. See .c file.
#
# Written as a basic example of counting multiple events as a stat tool.
#
# Copyright (c) 2015 Brendan Gregg.
# Licensed under the Apache License, Version 2.0 (the "License")
#
-# 14-Aug-2015 Brendan Gregg Created this.
+# 14-Aug-2015 Brendan Gregg Created this.
from __future__ import print_function
from bcc import BPF
-from ctypes import c_ushort, c_int, c_ulonglong
+from ctypes import c_int
from time import sleep, strftime
from sys import argv
def usage():
- print("USAGE: %s [interval [count]]" % argv[0])
- exit()
+ print("USAGE: %s [interval [count]]" % argv[0])
+ exit()
# arguments
interval = 1
count = -1
if len(argv) > 1:
- try:
- interval = int(argv[1])
- if interval == 0:
- raise
- if len(argv) > 2:
- count = int(argv[2])
- except: # also catches -h, --help
- usage()
+ try:
+ interval = int(argv[1])
+ if interval == 0:
+ raise
+ if len(argv) > 2:
+ count = int(argv[2])
+ except: # also catches -h, --help
+ usage()
# load BPF program
-b = BPF(src_file = "vfsstat.c")
+b = BPF(src_file="vfsstat.c")
b.attach_kprobe(event="vfs_read", fn_name="do_read")
b.attach_kprobe(event="vfs_write", fn_name="do_write")
b.attach_kprobe(event="vfs_fsync", fn_name="do_fsync")
# stat column labels and indexes
stat_types = {
- "READ" : 1,
- "WRITE" : 2,
- "FSYNC" : 3,
- "OPEN" : 4,
- "CREATE" : 5
+ "READ": 1,
+ "WRITE": 2,
+ "FSYNC": 3,
+ "OPEN": 4,
+ "CREATE": 5
}
# header
print("%-8s " % "TIME", end="")
for stype in stat_types.keys():
- print(" %8s" % (stype + "/s"), end="")
- idx = stat_types[stype]
+ print(" %8s" % (stype + "/s"), end="")
+ idx = stat_types[stype]
print("")
# output
i = 0
while (1):
- if count > 0:
- i += 1
- if i > count:
- exit()
- try:
- sleep(interval)
- except KeyboardInterrupt:
- pass; exit()
+ if count > 0:
+ i += 1
+ if i > count:
+ exit()
+ try:
+ sleep(interval)
+ except KeyboardInterrupt:
+ pass
+ exit()
- print("%-8s: " % strftime("%H:%M:%S"), end="")
- # print each statistic as a column
- for stype in stat_types.keys():
- idx = stat_types[stype]
- try:
- val = b["stats"][c_int(idx)].value / interval
- print(" %8d" % val, end="")
- except:
- print(" %8d" % 0, end="")
- b["stats"].clear()
- print("")
+ print("%-8s: " % strftime("%H:%M:%S"), end="")
+ # print each statistic as a column
+ for stype in stat_types.keys():
+ idx = stat_types[stype]
+ try:
+ val = b["stats"][c_int(idx)].value / interval
+ print(" %8d" % val, end="")
+ except:
+ print(" %8d" % 0, end="")
+ b["stats"].clear()
+ print("")
This traces some common VFS calls and prints per-second summaries. By default,
the output interval is one second:
-# ./vfsstat
+# ./vfsstat
TIME READ/s WRITE/s CREATE/s OPEN/s FSYNC/s
18:35:32: 231 12 4 98 0
18:35:33: 274 13 4 106 0
projects / platform / upstream / bcc.git / commitdiff