1 // SPDX-License-Identifier: GPL-2.0
2 #include <test_progs.h>
3 #include "kprobe_multi.skel.h"
4 #include "trace_helpers.h"
5 #include "kprobe_multi_empty.skel.h"
6 #include "kprobe_multi_override.skel.h"
7 #include "bpf/libbpf_internal.h"
8 #include "bpf/hashmap.h"
10 static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return)
12 LIBBPF_OPTS(bpf_test_run_opts, topts);
15 prog_fd = bpf_program__fd(skel->progs.trigger);
16 err = bpf_prog_test_run_opts(prog_fd, &topts);
17 ASSERT_OK(err, "test_run");
18 ASSERT_EQ(topts.retval, 0, "test_run");
20 ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result");
21 ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result");
22 ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result");
23 ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result");
24 ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result");
25 ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result");
26 ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result");
27 ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result");
30 ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result");
31 ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result");
32 ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result");
33 ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result");
34 ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result");
35 ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result");
36 ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result");
37 ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result");
41 static void test_skel_api(void)
43 struct kprobe_multi *skel = NULL;
46 skel = kprobe_multi__open_and_load();
47 if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
50 skel->bss->pid = getpid();
51 err = kprobe_multi__attach(skel);
52 if (!ASSERT_OK(err, "kprobe_multi__attach"))
55 kprobe_multi_test_run(skel, true);
58 kprobe_multi__destroy(skel);
61 static void test_link_api(struct bpf_link_create_opts *opts)
63 int prog_fd, link1_fd = -1, link2_fd = -1;
64 struct kprobe_multi *skel = NULL;
66 skel = kprobe_multi__open_and_load();
67 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
70 skel->bss->pid = getpid();
71 prog_fd = bpf_program__fd(skel->progs.test_kprobe);
72 link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
73 if (!ASSERT_GE(link1_fd, 0, "link_fd"))
76 opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN;
77 prog_fd = bpf_program__fd(skel->progs.test_kretprobe);
78 link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
79 if (!ASSERT_GE(link2_fd, 0, "link_fd"))
82 kprobe_multi_test_run(skel, true);
89 kprobe_multi__destroy(skel);
92 #define GET_ADDR(__sym, __addr) ({ \
93 __addr = ksym_get_addr(__sym); \
94 if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym)) \
98 static void test_link_api_addrs(void)
100 LIBBPF_OPTS(bpf_link_create_opts, opts);
101 unsigned long long addrs[8];
103 GET_ADDR("bpf_fentry_test1", addrs[0]);
104 GET_ADDR("bpf_fentry_test2", addrs[1]);
105 GET_ADDR("bpf_fentry_test3", addrs[2]);
106 GET_ADDR("bpf_fentry_test4", addrs[3]);
107 GET_ADDR("bpf_fentry_test5", addrs[4]);
108 GET_ADDR("bpf_fentry_test6", addrs[5]);
109 GET_ADDR("bpf_fentry_test7", addrs[6]);
110 GET_ADDR("bpf_fentry_test8", addrs[7]);
112 opts.kprobe_multi.addrs = (const unsigned long*) addrs;
113 opts.kprobe_multi.cnt = ARRAY_SIZE(addrs);
114 test_link_api(&opts);
117 static void test_link_api_syms(void)
119 LIBBPF_OPTS(bpf_link_create_opts, opts);
120 const char *syms[8] = {
131 opts.kprobe_multi.syms = syms;
132 opts.kprobe_multi.cnt = ARRAY_SIZE(syms);
133 test_link_api(&opts);
137 test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts)
139 struct bpf_link *link1 = NULL, *link2 = NULL;
140 struct kprobe_multi *skel = NULL;
142 skel = kprobe_multi__open_and_load();
143 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
146 skel->bss->pid = getpid();
147 link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
149 if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts"))
153 opts->retprobe = true;
154 link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual,
156 if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts"))
160 kprobe_multi_test_run(skel, !!opts);
163 bpf_link__destroy(link2);
164 bpf_link__destroy(link1);
165 kprobe_multi__destroy(skel);
168 static void test_attach_api_pattern(void)
170 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
172 test_attach_api("bpf_fentry_test*", &opts);
173 test_attach_api("bpf_fentry_test?", NULL);
176 static void test_attach_api_addrs(void)
178 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
179 unsigned long long addrs[8];
181 GET_ADDR("bpf_fentry_test1", addrs[0]);
182 GET_ADDR("bpf_fentry_test2", addrs[1]);
183 GET_ADDR("bpf_fentry_test3", addrs[2]);
184 GET_ADDR("bpf_fentry_test4", addrs[3]);
185 GET_ADDR("bpf_fentry_test5", addrs[4]);
186 GET_ADDR("bpf_fentry_test6", addrs[5]);
187 GET_ADDR("bpf_fentry_test7", addrs[6]);
188 GET_ADDR("bpf_fentry_test8", addrs[7]);
190 opts.addrs = (const unsigned long *) addrs;
191 opts.cnt = ARRAY_SIZE(addrs);
192 test_attach_api(NULL, &opts);
195 static void test_attach_api_syms(void)
197 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
198 const char *syms[8] = {
210 opts.cnt = ARRAY_SIZE(syms);
211 test_attach_api(NULL, &opts);
214 static void test_attach_api_fails(void)
216 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
217 struct kprobe_multi *skel = NULL;
218 struct bpf_link *link = NULL;
219 unsigned long long addrs[2];
220 const char *syms[2] = {
226 addrs[0] = ksym_get_addr("bpf_fentry_test1");
227 addrs[1] = ksym_get_addr("bpf_fentry_test2");
229 if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr"))
232 skel = kprobe_multi__open_and_load();
233 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
236 skel->bss->pid = getpid();
238 /* fail_1 - pattern and opts NULL */
239 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
241 if (!ASSERT_ERR_PTR(link, "fail_1"))
244 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_1_error"))
247 /* fail_2 - both addrs and syms set */
248 opts.addrs = (const unsigned long *) addrs;
250 opts.cnt = ARRAY_SIZE(syms);
253 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
255 if (!ASSERT_ERR_PTR(link, "fail_2"))
258 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_2_error"))
261 /* fail_3 - pattern and addrs set */
262 opts.addrs = (const unsigned long *) addrs;
264 opts.cnt = ARRAY_SIZE(syms);
267 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
269 if (!ASSERT_ERR_PTR(link, "fail_3"))
272 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_3_error"))
275 /* fail_4 - pattern and cnt set */
278 opts.cnt = ARRAY_SIZE(syms);
281 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
283 if (!ASSERT_ERR_PTR(link, "fail_4"))
286 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_4_error"))
289 /* fail_5 - pattern and cookies */
293 opts.cookies = cookies;
295 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
297 if (!ASSERT_ERR_PTR(link, "fail_5"))
300 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_5_error"))
304 bpf_link__destroy(link);
305 kprobe_multi__destroy(skel);
308 static size_t symbol_hash(long key, void *ctx __maybe_unused)
310 return str_hash((const char *) key);
313 static bool symbol_equal(long key1, long key2, void *ctx __maybe_unused)
315 return strcmp((const char *) key1, (const char *) key2) == 0;
318 static int get_syms(char ***symsp, size_t *cntp, bool kernel)
320 size_t cap = 0, cnt = 0, i;
321 char *name = NULL, **syms = NULL;
328 * The available_filter_functions contains many duplicates,
329 * but other than that all symbols are usable in kprobe multi
331 * Filtering out duplicates by using hashmap__add, which won't
332 * add existing entry.
335 if (access("/sys/kernel/tracing/trace", F_OK) == 0)
336 f = fopen("/sys/kernel/tracing/available_filter_functions", "r");
338 f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
343 map = hashmap__new(symbol_hash, symbol_equal, NULL);
345 err = libbpf_get_error(map);
349 while (fgets(buf, sizeof(buf), f)) {
350 if (kernel && strchr(buf, '['))
352 if (!kernel && !strchr(buf, '['))
356 if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1)
359 * We attach to almost all kernel functions and some of them
360 * will cause 'suspicious RCU usage' when fprobe is attached
361 * to them. Filter out the current culprits - arch_cpu_idle
362 * default_idle and rcu_* functions.
364 if (!strcmp(name, "arch_cpu_idle"))
366 if (!strcmp(name, "default_idle"))
368 if (!strncmp(name, "rcu_", 4))
370 if (!strcmp(name, "bpf_dispatcher_xdp_func"))
372 if (!strncmp(name, "__ftrace_invalid_address__",
373 sizeof("__ftrace_invalid_address__") - 1))
376 err = hashmap__add(map, name, 0);
377 if (err == -EEXIST) {
384 err = libbpf_ensure_mem((void **) &syms, &cap,
385 sizeof(*syms), cnt + 1);
401 for (i = 0; i < cnt; i++)
408 static void test_kprobe_multi_bench_attach(bool kernel)
410 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
411 struct kprobe_multi_empty *skel = NULL;
412 long attach_start_ns, attach_end_ns;
413 long detach_start_ns, detach_end_ns;
414 double attach_delta, detach_delta;
415 struct bpf_link *link = NULL;
419 if (!ASSERT_OK(get_syms(&syms, &cnt, kernel), "get_syms"))
422 skel = kprobe_multi_empty__open_and_load();
423 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
426 opts.syms = (const char **) syms;
429 attach_start_ns = get_time_ns();
430 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty,
432 attach_end_ns = get_time_ns();
434 if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
437 detach_start_ns = get_time_ns();
438 bpf_link__destroy(link);
439 detach_end_ns = get_time_ns();
441 attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0;
442 detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0;
444 printf("%s: found %lu functions\n", __func__, cnt);
445 printf("%s: attached in %7.3lfs\n", __func__, attach_delta);
446 printf("%s: detached in %7.3lfs\n", __func__, detach_delta);
449 kprobe_multi_empty__destroy(skel);
451 for (i = 0; i < cnt; i++)
457 static void test_attach_override(void)
459 struct kprobe_multi_override *skel = NULL;
460 struct bpf_link *link = NULL;
462 skel = kprobe_multi_override__open_and_load();
463 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
466 /* The test_override calls bpf_override_return so it should fail
467 * to attach to bpf_fentry_test1 function, which is not on error
470 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_override,
471 "bpf_fentry_test1", NULL);
472 if (!ASSERT_ERR_PTR(link, "override_attached_bpf_fentry_test1")) {
473 bpf_link__destroy(link);
477 /* The should_fail_bio function is on error injection list,
478 * attach should succeed.
480 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_override,
481 "should_fail_bio", NULL);
482 if (!ASSERT_OK_PTR(link, "override_attached_should_fail_bio"))
485 bpf_link__destroy(link);
488 kprobe_multi_override__destroy(skel);
491 void serial_test_kprobe_multi_bench_attach(void)
493 if (test__start_subtest("kernel"))
494 test_kprobe_multi_bench_attach(true);
495 if (test__start_subtest("modules"))
496 test_kprobe_multi_bench_attach(false);
499 void test_kprobe_multi_test(void)
501 if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
504 if (test__start_subtest("skel_api"))
506 if (test__start_subtest("link_api_addrs"))
507 test_link_api_syms();
508 if (test__start_subtest("link_api_syms"))
509 test_link_api_addrs();
510 if (test__start_subtest("attach_api_pattern"))
511 test_attach_api_pattern();
512 if (test__start_subtest("attach_api_addrs"))
513 test_attach_api_addrs();
514 if (test__start_subtest("attach_api_syms"))
515 test_attach_api_syms();
516 if (test__start_subtest("attach_api_fails"))
517 test_attach_api_fails();
518 if (test__start_subtest("attach_override"))
519 test_attach_override();