From d9e578b15c0c89ebd926ed4d12ea026dff3eb9d0 Mon Sep 17 00:00:00 2001 From: Brendan Gregg Date: Mon, 21 Sep 2015 11:59:42 -0700 Subject: [PATCH] killsnoop --- README.md | 1 + man/man8/killsnoop.8 | 86 ++++++++++++++++++++++++++++++++++++ tools/killsnoop | 105 ++++++++++++++++++++++++++++++++++++++++++++ tools/killsnoop_example.txt | 36 +++++++++++++++ 4 files changed, 228 insertions(+) create mode 100644 man/man8/killsnoop.8 create mode 100755 tools/killsnoop create mode 100644 tools/killsnoop_example.txt diff --git a/README.md b/README.md index 5ddc811..9a4c5a2 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ Tools: - tools/[biosnoop](tools/biosnoop): Trace block device I/O with PID and latency. [Examples](tools/biosnoop_example.txt). - tools/[funccount](tools/funccount): Count kernel function calls. [Examples](tools/funccount_example.txt). +- tools/[killsnoop](tools/killsnoop): Trace signals issued by the kill() syscall. [Examples](tools/killsnoop_example.txt). - tools/[opensnoop](tools/opensnoop): Trace open() syscalls. [Examples](tools/opensnoop_example.txt). - tools/[pidpersec](tools/pidpersec): Count new processes (via fork). [Examples](tools/pidpersec_example.txt). - tools/[syncsnoop](tools/syncsnoop): Trace sync() syscall. [Examples](tools/syncsnoop_example.txt). diff --git a/man/man8/killsnoop.8 b/man/man8/killsnoop.8 new file mode 100644 index 0000000..b3c08cc --- /dev/null +++ b/man/man8/killsnoop.8 @@ -0,0 +1,86 @@ +.TH killsnoop 8 "2015-08-20" "USER COMMANDS" +.SH NAME +killsnoop \- Trace signals issued by the kill() syscall. Uses Linux eBPF/bcc. +.SH SYNOPSIS +.B killsnoop [\-h] [\-t] [\-x] [-p PID] +.SH DESCRIPTION +killsnoop traces the kill() syscall, to show signals sent via this method. This +may be useful to troubleshoot failing applications, where an unknown mechanism +is sending signals. + +This works by tracing the kernel sys_kill() function using dynamic tracing, and +will need updating to match any changes to this function. + +Since this uses BPF, only the root user can use this tool. +.SH REQUIREMENTS +CONFIG_BPF and bcc. +.SH OPTIONS +.TP +\-h +Print usage message. +.TP +\-t +Include a timestamp column. +.TP +\-x +Only print failed kill() syscalls. +.TP +\-p PID +Trace this process ID only (filtered in-kernel). +.SH EXAMPLES +.TP +Trace all kill() syscalls: +# +.B killsnoop +.TP +Trace all kill() syscalls, and include timestamps: +# +.B killsnoop \-t +.TP +Trace only kill() syscalls that failed: +# +.B killsnoop \-x +.TP +Trace PID 181 only: +# +.B killsnoop \-p 181 +.SH FIELDS +.TP +TIME(s) +Time of the call, in seconds. +.TP +PID +Source process ID +.TP +COMM +Source process name +.TP +SIG +Signal number. See signal(7). +.TP +TPID +Target process ID +.TP +RES +Result. 0 == success, a negative value (of the error code) for failure. +.SH OVERHEAD +This traces the kernel kill function and prints output for each event. As the +rate of this is generally expected to be low (< 100/s), the overhead is also +expected to be negligible. If you have an application that is calling a very +high rate of kill()s for some reason, then test and understand overhead before +use. +.SH SOURCE +This is from bcc. +.IP +https://github.com/iovisor/bcc +.PP +Also look in the bcc distribution for a companion _examples.txt file containing +example usage, output, and commentary for this tool. +.SH OS +Linux +.SH STABILITY +Unstable - in development. +.SH AUTHOR +Brendan Gregg +.SH SEE ALSO +opensnoop(8), funccount(8) diff --git a/tools/killsnoop b/tools/killsnoop new file mode 100755 index 0000000..ea9a843 --- /dev/null +++ b/tools/killsnoop @@ -0,0 +1,105 @@ +#!/usr/bin/python +# +# killsnoop Trace signals issued by the kill() syscall. +# For Linux, uses BCC, eBPF. Embedded C. +# +# USAGE: killsnoop [-h] [-t] [-x] [-p PID] +# +# Copyright (c) 2015 Brendan Gregg. +# Licensed under the Apache License, Version 2.0 (the "License") +# +# 20-Sep-2015 Brendan Gregg Created this. + +from __future__ import print_function +from bcc import BPF +import argparse + +# arguments +examples = """examples: + ./killsnoop # trace all kill() signals + ./killsnoop -t # include timestamps + ./killsnoop -x # only show failed kills + ./killsnoop -p 181 # only trace PID 181 +""" +parser = argparse.ArgumentParser( + description="Trace signals issued by the kill() syscall", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) +parser.add_argument("-t", "--timestamp", action="store_true", + help="include timestamp on output") +parser.add_argument("-x", "--failed", action="store_true", + help="only show failed opens") +parser.add_argument("-p", "--pid", + help="trace this PID only") +args = parser.parse_args() +debug = 0 + +# define BPF program +bpf_text = """ +#include + +BPF_HASH(args_pid, u32, int); +BPF_HASH(args_sig, u32, int); + +int kprobe__sys_kill(struct pt_regs *ctx, int tpid, int sig) +{ + u32 pid = bpf_get_current_pid_tgid(); + + FILTER + args_pid.update(&pid, &tpid); + args_sig.update(&pid, &sig); + + return 0; +}; + +int kretprobe__sys_kill(struct pt_regs *ctx) +{ + int *tpidp, *sigp, ret = ctx->ax; + u32 pid = bpf_get_current_pid_tgid(); + + tpidp = args_pid.lookup(&pid); + sigp = args_sig.lookup(&pid); + if (tpidp == 0 || sigp == 0) { + return 0; // missed entry + } + + bpf_trace_printk("%d %d %d\\n", *tpidp, *sigp, ret); + args_pid.delete(&pid); + args_sig.delete(&pid); + + return 0; +} +""" +if args.pid: + bpf_text = bpf_text.replace('FILTER', + 'if (pid != %s) { return 0; }' % args.pid) +else: + bpf_text = bpf_text.replace('FILTER', '') +if debug: + print(bpf_text) + +# initialize BPF +b = BPF(text=bpf_text) + +# header +if args.timestamp: + print("%-14s" % ("TIME(s)"), end="") +print("%-6s %-16s %-4s %-6s %s" % ("PID", "COMM", "SIG", "TPID", "RESULT")) + +start_ts = 0 + +# format output +while 1: + (task, pid, cpu, flags, ts, msg) = b.trace_fields() + (tpid_s, sig_s, ret_s) = msg.split(" ") + + ret = int(ret_s) + if (args.failed and (ret >= 0)): + continue + + # print columns + if args.timestamp: + if start_ts == 0: + start_ts = ts + print("%-14.9f" % (ts - start_ts), end="") + print("%-6d %-16s %-4s %-6s %s" % (pid, task, sig_s, tpid_s, ret_s)) diff --git a/tools/killsnoop_example.txt b/tools/killsnoop_example.txt new file mode 100644 index 0000000..f80dc77 --- /dev/null +++ b/tools/killsnoop_example.txt @@ -0,0 +1,36 @@ +Demonstrations of killsnoop, the Linux eBPF/bcc version. + + +This traces signals sent via the kill() syscall. For example: + +# ./killsnoop +PID COMM SIG TPID RESULT +17064 bash 9 27682 0 +17064 bash 9 27682 -3 +17064 bash 0 17064 0 + +The first line showed a SIGKILL (9) sent from PID 17064 (a bash shell) to +PID 27682. The result, 0, means success. + +The second line showed the same signal sent, this time resulting in a -3 +(ESRCH: no such process). + + +USAGE message: + +# ./killsnoop -h +usage: killsnoop [-h] [-t] [-x] [-p PID] + +Trace signals issued by the kill() syscall + +optional arguments: + -h, --help show this help message and exit + -t, --timestamp include timestamp on output + -x, --failed only show failed opens + -p PID, --pid PID trace this PID only + +examples: + ./killsnoop # trace all kill() signals + ./killsnoop -t # include timestamps + ./killsnoop -x # only show failed kills + ./killsnoop -p 181 # only trace PID 181 -- 2.7.4