--- /dev/null
+.TH biosnoop 8 "2015-09-16" "USER COMMANDS"
+.SH NAME
+biosnoop \- Trace block device I/O and print details incl. issuing PID.
+.SH SYNOPSIS
+.B biosnoop
+.SH DESCRIPTION
+This tools traces block device I/O (disk I/O), and prints a one-line summary
+for each I/O showing various details. These include the latency from the time of
+issue to the device to its completion, and the PID and process name from when
+the I/O was first created (which usually identifies the responsible process).
+
+This uses in-kernel eBPF maps to cache process details (PID and comm) by I/O
+request, as well as a starting timestamp for calculating I/O latency.
+
+Since this uses BPF, only the root user can use this tool.
+.SH REQUIREMENTS
+CONFIG_BPF and bcc.
+.SH EXAMPLES
+.TP
+Trace all block device I/O and print a summary line per I/O:
+#
+.B biosnoop
+.SH FIELDS
+.TP
+TIME(s)
+Time of the I/O, in seconds since the first I/O was seen.
+.TP
+COMM
+Cached process name, if present. This usually (but isn't guaranteed) to identify
+the responsible process for the I/O.
+.TP
+PID
+Cached process ID, if present. This usually (but isn't guaranteed) to identify
+the responsible process for the I/O.
+.TP
+DISK
+Disk device name.
+.TP
+T
+Type of I/O: R = read, W = write. This is a simplification.
+.TP
+SECTOR
+Device sector for the I/O.
+.TP
+BYTES
+Size of the I/O, in bytes.
+.TP
+LAT(ms)
+Time for the I/O (latency) from the issue to the device, to its completion,
+in milliseconds.
+.SH OVERHEAD
+Since block device I/O usually has a relatively low frequency (< 10,000/s),
+the overhead for this tool is expected to be negligible. For high IOPS storage
+systems, test and quantify before use.
+.SH SOURCE
+This is from bcc.
+.IP
+https://github.com/iovisor/bcc
+.PP
+Also look in the bcc distribution for a companion _examples.txt file containing
+example usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Brendan Gregg
+.SH SEE ALSO
+disksnoop(8), iostat(1)
--- /dev/null
+#!/usr/bin/python
+#
+# biosnoop Trace block device I/O and print details including issuing PID.
+# For Linux, uses BCC, eBPF.
+#
+# This uses in-kernel eBPF maps to cache process details (PID and comm) by I/O
+# request, as well as a starting timestamp for calculating I/O latency.
+#
+# Copyright (c) 2015 Brendan Gregg.
+# Licensed under the Apache License, Version 2.0 (the "License")
+#
+# 16-Sep-2015 Brendan Gregg Created this.
+
+from __future__ import print_function
+from bcc import BPF
+
+# load BPF program
+b = BPF(text="""
+#include <uapi/linux/ptrace.h>
+#include <linux/blkdev.h>
+
+struct val_t {
+ char name[TASK_COMM_LEN];
+};
+
+BPF_HASH(start, struct request *);
+BPF_HASH(pidbyreq, struct request *, u32);
+BPF_HASH(commbyreq, struct request *, struct val_t);
+
+// cache PID and comm by-req
+int trace_pid_start(struct pt_regs *ctx, struct request *req)
+{
+ u32 pid;
+ struct val_t val = {};
+
+ pid = bpf_get_current_pid_tgid();
+ pidbyreq.update(&req, &pid);
+ if (bpf_get_current_comm(&val.name, sizeof(val.name)) == 0) {
+ commbyreq.update(&req, &val);
+ }
+
+ return 0;
+}
+
+// time block I/O
+int trace_req_start(struct pt_regs *ctx, struct request *req)
+{
+ u64 ts;
+
+ ts = bpf_ktime_get_ns();
+ start.update(&req, &ts);
+
+ return 0;
+}
+
+// output
+int trace_req_completion(struct pt_regs *ctx, struct request *req)
+{
+ u64 *tsp, delta;
+ u32 *pidp = 0;
+ struct val_t *valp;
+
+ // fetch timestamp and calculate delta
+ tsp = start.lookup(&req);
+ if (tsp == 0) {
+ // missed tracing issue
+ return 0;
+ }
+ delta = bpf_ktime_get_ns() - *tsp;
+
+ //
+ // Fetch and output issuing pid and comm.
+ // As bpf_trace_prink() is limited to a maximum of 1 string and 2
+ // integers, we'll use more than one to output the data.
+ //
+ valp = commbyreq.lookup(&req);
+ pidp = pidbyreq.lookup(&req);
+ if (pidp == 0 || valp == 0) {
+ bpf_trace_printk("0 0 ? %d\\n", req->__data_len);
+ } else {
+ bpf_trace_printk("0 %d %s %d\\n", *pidp, valp->name,
+ req->__data_len);
+ }
+
+ // output remaining details
+ if (req->cmd_flags & REQ_WRITE) {
+ bpf_trace_printk("1 W %s %d %d ?\\n", req->rq_disk->disk_name,
+ req->__sector, delta / 1000);
+ } else {
+ bpf_trace_printk("1 R %s %d %d ?\\n", req->rq_disk->disk_name,
+ req->__sector, delta / 1000);
+ }
+
+ start.delete(&req);
+ pidbyreq.delete(&req);
+ commbyreq.delete(&req);
+
+ return 0;
+}
+""")
+# blk_account_io_start is chosen to be as
+b.attach_kprobe(event="blk_account_io_start", fn_name="trace_pid_start")
+b.attach_kprobe(event="blk_start_request", fn_name="trace_req_start")
+b.attach_kprobe(event="blk_mq_start_request", fn_name="trace_req_start")
+b.attach_kprobe(event="blk_account_io_completion",
+ fn_name="trace_req_completion")
+
+# header
+print("%-14s %-14s %-6s %-7s %-2s %-9s %-7s %7s" % ("TIME(s)", "COMM", "PID",
+ "DISK", "T", "SECTOR", "BYTES", "LAT(ms)"))
+
+start_ts = 0
+
+# format output
+while 1:
+ (task, pid, cpu, flags, ts, msg) = b.trace_fields()
+ args = msg.split(" ")
+
+ if start_ts == 0:
+ start_ts = ts
+
+ if args[0] == "0":
+ (real_pid, real_comm, bytes_s) = (args[1], args[2], args[3])
+ continue
+ else:
+ (type_s, disk_s, sector_s, us_s) = (args[1], args[2], args[3],
+ args[4])
+
+ ms = float(int(us_s, 10)) / 1000
+
+ print("%-14.9f %-14.14s %-6s %-7s %-2s %-9s %-7s %7.2f" % (
+ ts - start_ts, real_comm, real_pid, disk_s, type_s, sector_s,
+ bytes_s, ms))
--- /dev/null
+Demonstrations of biosnoop, the Linux eBPF/bcc version.
+
+
+biosnoop traces block device I/O (disk I/O), and prints a line of output
+per I/O. Example:
+
+# ./biosnoop
+TIME(s) COMM PID DISK T SECTOR BYTES LAT(ms)
+0.000004001 supervise 1950 xvda1 W 13092560 4096 0.74
+0.000178002 supervise 1950 xvda1 W 13092432 4096 0.61
+0.001469001 supervise 1956 xvda1 W 13092440 4096 1.24
+0.001588002 supervise 1956 xvda1 W 13115128 4096 1.09
+1.022346001 supervise 1950 xvda1 W 13115272 4096 0.98
+1.022568002 supervise 1950 xvda1 W 13188496 4096 0.93
+1.023534000 supervise 1956 xvda1 W 13188520 4096 0.79
+1.023585003 supervise 1956 xvda1 W 13189512 4096 0.60
+2.003920000 xfsaild/md0 456 xvdc W 62901512 8192 0.23
+2.003931001 xfsaild/md0 456 xvdb W 62901513 512 0.25
+2.004034001 xfsaild/md0 456 xvdb W 62901520 8192 0.35
+2.004042000 xfsaild/md0 456 xvdb W 63542016 4096 0.36
+2.004204001 kworker/0:3 26040 xvdb W 41950344 65536 0.34
+2.044352002 supervise 1950 xvda1 W 13192672 4096 0.65
+2.044574000 supervise 1950 xvda1 W 13189072 4096 0.58
+
+This includes the PID and comm (process name) that were on-CPU at the time of
+issue (which usually means the process responsible).
+
+The latency of the disk I/O, measured from the issue to the device to its
+completion, is included as the last column.
+
+This example output is from what should be an idle system, however, the
+following is visible in iostat:
+
+$ iostat -x 1
+[...]
+avg-cpu: %user %nice %system %iowait %steal %idle
+ 0.12 0.00 0.12 0.00 0.00 99.75
+
+Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
+xvda 0.00 0.00 0.00 4.00 0.00 16.00 8.00 0.00 0.00 0.00 0.00 0.00 0.00
+xvdb 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
+xvdc 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
+md0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
+
+There are 4 write IOPS.
+
+The output of biosnoop identifies the reason: multiple supervise processes are
+issuing writes to the xvda1 disk. I can now drill down on supervise using other
+tools to understand its file system workload.
projects / platform / upstream / bcc.git / commitdiff