From 7e71aef910c3712ef008b6e0233fc25212deaff9 Mon Sep 17 00:00:00 2001 From: Brenden Blanco Date: Wed, 9 Sep 2015 18:28:21 -0700 Subject: [PATCH] Improve coverage for kprobe event_re This makes the attachment of kprobes to arbitrary events more robust. Issue 1: Functions with '.' characters should not have similarly named probes. Issue 2: Functions in the blacklist should not be attached to. Issue 3: Some functions matched by regex cannot actually be attached to, despite not being in the blacklist...possibly the blacklist is outdated? Instead, warn instead of error during bulk regex attach. Issue 4: Attaching to large numbers of kprobes gets to be very slow. For now, leave this unresolved. For reasonably sized regexes, startup times may be acceptable, and shutdown times are actually the worse part. To speed up shutdown, one could add the following after the last attach_kprobe to disable auto-cleanup: ``` from bcc import open_kprobes open_kprobes = {} ``` Then, once the program is exited, one must manually echo "" > kprobe_events Some numbers: attaching to event_re='tcp_*': 2 sec startup, 15 sec shutdown attaching to event_re='b*': 10 sec startup, 75 sec shutdown attaching to event_re='*': unknown (>20 min) startup, unknown shutdown The slowdowns appear to be exponential, doubtful that '*' will ever complete. Fixes: #199 Signed-off-by: Brenden Blanco --- src/cc/libbpf.c | 4 +++- src/python/bcc/__init__.py | 27 ++++++++++++++++++--------- tests/cc/test_trace4.py | 7 +++++++ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/cc/libbpf.c b/src/cc/libbpf.c index a817907..20ce1fc 100644 --- a/src/cc/libbpf.c +++ b/src/cc/libbpf.c @@ -239,7 +239,9 @@ int bpf_attach_kprobe(int progfd, const char *event, } if (write(kfd, event_desc, strlen(event_desc)) < 0) { - perror("write(kprobe_events)"); + fprintf(stderr, "write of \"%s\" into kprobe_events failed: %s\n", event_desc, strerror(errno)); + if (errno == EINVAL) + fprintf(stderr, "check dmesg output for possible cause\n"); goto cleanup; } diff --git a/src/python/bcc/__init__.py b/src/python/bcc/__init__.py index 776c329..b5af5c6 100644 --- a/src/python/bcc/__init__.py +++ b/src/python/bcc/__init__.py @@ -473,7 +473,10 @@ class BPF(object): p = Popen(["awk", "$1 ~ /%s/ { print $1 }" % event_re, "%s/available_filter_functions" % TRACEFS], stdout=PIPE) lines = p.communicate()[0].decode().split() - return [line.rstrip() for line in lines if line != "\n"] + with open("%s/../kprobes/blacklist" % TRACEFS) as f: + blacklist = [line.split()[1] for line in f.readlines()] + return [line.rstrip() for line in lines if + (line != "\n" and line not in blacklist)] def attach_kprobe(self, event="", fn_name="", event_re="", pid=0, cpu=-1, group_fd=-1): @@ -481,12 +484,15 @@ class BPF(object): # allow the caller to glob multiple functions together if event_re: for line in BPF._get_kprobe_functions(event_re): - self.attach_kprobe(event=line, fn_name=fn_name, pid=pid, - cpu=cpu, group_fd=group_fd) + try: + self.attach_kprobe(event=line, fn_name=fn_name, pid=pid, + cpu=cpu, group_fd=group_fd) + except: + pass return fn = self.load_func(fn_name, BPF.KPROBE) - ev_name = "p_" + event.replace("+", "_") + ev_name = "p_" + event.replace("+", "_").replace(".", "_") desc = "p:kprobes/%s %s" % (ev_name, event) res = lib.bpf_attach_kprobe(fn.fd, ev_name.encode("ascii"), desc.encode("ascii"), pid, cpu, group_fd) @@ -497,7 +503,7 @@ class BPF(object): @staticmethod def detach_kprobe(event): - ev_name = "p_" + event.replace("+", "_") + ev_name = "p_" + event.replace("+", "_").replace(".", "_") if ev_name not in open_kprobes: raise Exception("Kprobe %s is not attached" % event) os.close(open_kprobes[ev_name]) @@ -513,12 +519,15 @@ class BPF(object): # allow the caller to glob multiple functions together if event_re: for line in BPF._get_kprobe_functions(event_re): - self.attach_kretprobe(event=line, fn_name=fn_name, pid=pid, - cpu=cpu, group_fd=group_fd) + try: + self.attach_kretprobe(event=line, fn_name=fn_name, pid=pid, + cpu=cpu, group_fd=group_fd) + except: + pass return fn = self.load_func(fn_name, BPF.KPROBE) - ev_name = "r_" + event.replace("+", "_") + ev_name = "r_" + event.replace("+", "_").replace(".", "_") desc = "r:kprobes/%s %s" % (ev_name, event) res = lib.bpf_attach_kprobe(fn.fd, ev_name.encode("ascii"), desc.encode("ascii"), pid, cpu, group_fd) @@ -529,7 +538,7 @@ class BPF(object): @staticmethod def detach_kretprobe(event): - ev_name = "r_" + event.replace("+", "_") + ev_name = "r_" + event.replace("+", "_").replace(".", "_") if ev_name not in open_kprobes: raise Exception("Kretprobe %s is not attached" % event) os.close(open_kprobes[ev_name]) diff --git a/tests/cc/test_trace4.py b/tests/cc/test_trace4.py index bcfbfda..a35c718 100755 --- a/tests/cc/test_trace4.py +++ b/tests/cc/test_trace4.py @@ -30,5 +30,12 @@ class TestKprobeRgx(TestCase): k2 = self.b["stats"].Key(2) self.assertEqual(self.b["stats"][k1].val, self.b["stats"][k2].val + 1) +class TestKprobeReplace(TestCase): + def setUp(self): + self.b = BPF(text="int empty(void *ctx) { return 0; }") + + def test_periods(self): + self.b.attach_kprobe(event_re="^tcp_enter_cwr.*", fn_name="empty") + if __name__ == "__main__": main() -- 2.7.4