tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <test_progs.h>
   3 #include "kprobe_multi.skel.h"
   4 #include "trace_helpers.h"
   5 #include "kprobe_multi_empty.skel.h"
   6 #include "bpf/libbpf_internal.h"
   7 #include "bpf/hashmap.h"
   8
   9 static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return)
  10 {
  11         LIBBPF_OPTS(bpf_test_run_opts, topts);
  12         int err, prog_fd;
  13
  14         prog_fd = bpf_program__fd(skel->progs.trigger);
  15         err = bpf_prog_test_run_opts(prog_fd, &topts);
  16         ASSERT_OK(err, "test_run");
  17         ASSERT_EQ(topts.retval, 0, "test_run");
  18
  19         ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result");
  20         ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result");
  21         ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result");
  22         ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result");
  23         ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result");
  24         ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result");
  25         ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result");
  26         ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result");
  27
  28         if (test_return) {
  29                 ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result");
  30                 ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result");
  31                 ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result");
  32                 ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result");
  33                 ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result");
  34                 ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result");
  35                 ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result");
  36                 ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result");
  37         }
  38 }
  39
  40 static void test_skel_api(void)
  41 {
  42         struct kprobe_multi *skel = NULL;
  43         int err;
  44
  45         skel = kprobe_multi__open_and_load();
  46         if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
  47                 goto cleanup;
  48
  49         skel->bss->pid = getpid();
  50         err = kprobe_multi__attach(skel);
  51         if (!ASSERT_OK(err, "kprobe_multi__attach"))
  52                 goto cleanup;
  53
  54         kprobe_multi_test_run(skel, true);
  55
  56 cleanup:
  57         kprobe_multi__destroy(skel);
  58 }
  59
  60 static void test_link_api(struct bpf_link_create_opts *opts)
  61 {
  62         int prog_fd, link1_fd = -1, link2_fd = -1;
  63         struct kprobe_multi *skel = NULL;
  64
  65         skel = kprobe_multi__open_and_load();
  66         if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
  67                 goto cleanup;
  68
  69         skel->bss->pid = getpid();
  70         prog_fd = bpf_program__fd(skel->progs.test_kprobe);
  71         link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
  72         if (!ASSERT_GE(link1_fd, 0, "link_fd"))
  73                 goto cleanup;
  74
  75         opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN;
  76         prog_fd = bpf_program__fd(skel->progs.test_kretprobe);
  77         link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
  78         if (!ASSERT_GE(link2_fd, 0, "link_fd"))
  79                 goto cleanup;
  80
  81         kprobe_multi_test_run(skel, true);
  82
  83 cleanup:
  84         if (link1_fd != -1)
  85                 close(link1_fd);
  86         if (link2_fd != -1)
  87                 close(link2_fd);
  88         kprobe_multi__destroy(skel);
  89 }
  90
  91 #define GET_ADDR(__sym, __addr) ({                                      \
  92         __addr = ksym_get_addr(__sym);                                  \
  93         if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym)) \
  94                 return;                                                 \
  95 })
  96
  97 static void test_link_api_addrs(void)
  98 {
  99         LIBBPF_OPTS(bpf_link_create_opts, opts);
 100         unsigned long long addrs[8];
 101
 102         GET_ADDR("bpf_fentry_test1", addrs[0]);
 103         GET_ADDR("bpf_fentry_test2", addrs[1]);
 104         GET_ADDR("bpf_fentry_test3", addrs[2]);
 105         GET_ADDR("bpf_fentry_test4", addrs[3]);
 106         GET_ADDR("bpf_fentry_test5", addrs[4]);
 107         GET_ADDR("bpf_fentry_test6", addrs[5]);
 108         GET_ADDR("bpf_fentry_test7", addrs[6]);
 109         GET_ADDR("bpf_fentry_test8", addrs[7]);
 110
 111         opts.kprobe_multi.addrs = (const unsigned long*) addrs;
 112         opts.kprobe_multi.cnt = ARRAY_SIZE(addrs);
 113         test_link_api(&opts);
 114 }
 115
 116 static void test_link_api_syms(void)
 117 {
 118         LIBBPF_OPTS(bpf_link_create_opts, opts);
 119         const char *syms[8] = {
 120                 "bpf_fentry_test1",
 121                 "bpf_fentry_test2",
 122                 "bpf_fentry_test3",
 123                 "bpf_fentry_test4",
 124                 "bpf_fentry_test5",
 125                 "bpf_fentry_test6",
 126                 "bpf_fentry_test7",
 127                 "bpf_fentry_test8",
 128         };
 129
 130         opts.kprobe_multi.syms = syms;
 131         opts.kprobe_multi.cnt = ARRAY_SIZE(syms);
 132         test_link_api(&opts);
 133 }
 134
 135 static void
 136 test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts)
 137 {
 138         struct bpf_link *link1 = NULL, *link2 = NULL;
 139         struct kprobe_multi *skel = NULL;
 140
 141         skel = kprobe_multi__open_and_load();
 142         if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
 143                 goto cleanup;
 144
 145         skel->bss->pid = getpid();
 146         link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 147                                                       pattern, opts);
 148         if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts"))
 149                 goto cleanup;
 150
 151         if (opts) {
 152                 opts->retprobe = true;
 153                 link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual,
 154                                                               pattern, opts);
 155                 if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts"))
 156                         goto cleanup;
 157         }
 158
 159         kprobe_multi_test_run(skel, !!opts);
 160
 161 cleanup:
 162         bpf_link__destroy(link2);
 163         bpf_link__destroy(link1);
 164         kprobe_multi__destroy(skel);
 165 }
 166
 167 static void test_attach_api_pattern(void)
 168 {
 169         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
 170
 171         test_attach_api("bpf_fentry_test*", &opts);
 172         test_attach_api("bpf_fentry_test?", NULL);
 173 }
 174
 175 static void test_attach_api_addrs(void)
 176 {
 177         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
 178         unsigned long long addrs[8];
 179
 180         GET_ADDR("bpf_fentry_test1", addrs[0]);
 181         GET_ADDR("bpf_fentry_test2", addrs[1]);
 182         GET_ADDR("bpf_fentry_test3", addrs[2]);
 183         GET_ADDR("bpf_fentry_test4", addrs[3]);
 184         GET_ADDR("bpf_fentry_test5", addrs[4]);
 185         GET_ADDR("bpf_fentry_test6", addrs[5]);
 186         GET_ADDR("bpf_fentry_test7", addrs[6]);
 187         GET_ADDR("bpf_fentry_test8", addrs[7]);
 188
 189         opts.addrs = (const unsigned long *) addrs;
 190         opts.cnt = ARRAY_SIZE(addrs);
 191         test_attach_api(NULL, &opts);
 192 }
 193
 194 static void test_attach_api_syms(void)
 195 {
 196         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
 197         const char *syms[8] = {
 198                 "bpf_fentry_test1",
 199                 "bpf_fentry_test2",
 200                 "bpf_fentry_test3",
 201                 "bpf_fentry_test4",
 202                 "bpf_fentry_test5",
 203                 "bpf_fentry_test6",
 204                 "bpf_fentry_test7",
 205                 "bpf_fentry_test8",
 206         };
 207
 208         opts.syms = syms;
 209         opts.cnt = ARRAY_SIZE(syms);
 210         test_attach_api(NULL, &opts);
 211 }
 212
 213 static void test_attach_api_fails(void)
 214 {
 215         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
 216         struct kprobe_multi *skel = NULL;
 217         struct bpf_link *link = NULL;
 218         unsigned long long addrs[2];
 219         const char *syms[2] = {
 220                 "bpf_fentry_test1",
 221                 "bpf_fentry_test2",
 222         };
 223         __u64 cookies[2];
 224
 225         addrs[0] = ksym_get_addr("bpf_fentry_test1");
 226         addrs[1] = ksym_get_addr("bpf_fentry_test2");
 227
 228         if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr"))
 229                 goto cleanup;
 230
 231         skel = kprobe_multi__open_and_load();
 232         if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
 233                 goto cleanup;
 234
 235         skel->bss->pid = getpid();
 236
 237         /* fail_1 - pattern and opts NULL */
 238         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 239                                                      NULL, NULL);
 240         if (!ASSERT_ERR_PTR(link, "fail_1"))
 241                 goto cleanup;
 242
 243         if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_1_error"))
 244                 goto cleanup;
 245
 246         /* fail_2 - both addrs and syms set */
 247         opts.addrs = (const unsigned long *) addrs;
 248         opts.syms = syms;
 249         opts.cnt = ARRAY_SIZE(syms);
 250         opts.cookies = NULL;
 251
 252         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 253                                                      NULL, &opts);
 254         if (!ASSERT_ERR_PTR(link, "fail_2"))
 255                 goto cleanup;
 256
 257         if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_2_error"))
 258                 goto cleanup;
 259
 260         /* fail_3 - pattern and addrs set */
 261         opts.addrs = (const unsigned long *) addrs;
 262         opts.syms = NULL;
 263         opts.cnt = ARRAY_SIZE(syms);
 264         opts.cookies = NULL;
 265
 266         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 267                                                      "ksys_*", &opts);
 268         if (!ASSERT_ERR_PTR(link, "fail_3"))
 269                 goto cleanup;
 270
 271         if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_3_error"))
 272                 goto cleanup;
 273
 274         /* fail_4 - pattern and cnt set */
 275         opts.addrs = NULL;
 276         opts.syms = NULL;
 277         opts.cnt = ARRAY_SIZE(syms);
 278         opts.cookies = NULL;
 279
 280         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 281                                                      "ksys_*", &opts);
 282         if (!ASSERT_ERR_PTR(link, "fail_4"))
 283                 goto cleanup;
 284
 285         if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_4_error"))
 286                 goto cleanup;
 287
 288         /* fail_5 - pattern and cookies */
 289         opts.addrs = NULL;
 290         opts.syms = NULL;
 291         opts.cnt = 0;
 292         opts.cookies = cookies;
 293
 294         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 295                                                      "ksys_*", &opts);
 296         if (!ASSERT_ERR_PTR(link, "fail_5"))
 297                 goto cleanup;
 298
 299         if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_5_error"))
 300                 goto cleanup;
 301
 302 cleanup:
 303         bpf_link__destroy(link);
 304         kprobe_multi__destroy(skel);
 305 }
 306
 307 static inline __u64 get_time_ns(void)
 308 {
 309         struct timespec t;
 310
 311         clock_gettime(CLOCK_MONOTONIC, &t);
 312         return (__u64) t.tv_sec * 1000000000 + t.tv_nsec;
 313 }
 314
 315 static size_t symbol_hash(const void *key, void *ctx __maybe_unused)
 316 {
 317         return str_hash((const char *) key);
 318 }
 319
 320 static bool symbol_equal(const void *key1, const void *key2, void *ctx __maybe_unused)
 321 {
 322         return strcmp((const char *) key1, (const char *) key2) == 0;
 323 }
 324
 325 static int get_syms(char ***symsp, size_t *cntp)
 326 {
 327         size_t cap = 0, cnt = 0, i;
 328         char *name = NULL, **syms = NULL;
 329         struct hashmap *map;
 330         char buf[256];
 331         FILE *f;
 332         int err = 0;
 333
 334         /*
 335          * The available_filter_functions contains many duplicates,
 336          * but other than that all symbols are usable in kprobe multi
 337          * interface.
 338          * Filtering out duplicates by using hashmap__add, which won't
 339          * add existing entry.
 340          */
 341         f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
 342         if (!f)
 343                 return -EINVAL;
 344
 345         map = hashmap__new(symbol_hash, symbol_equal, NULL);
 346         if (IS_ERR(map)) {
 347                 err = libbpf_get_error(map);
 348                 goto error;
 349         }
 350
 351         while (fgets(buf, sizeof(buf), f)) {
 352                 /* skip modules */
 353                 if (strchr(buf, '['))
 354                         continue;
 355
 356                 free(name);
 357                 if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1)
 358                         continue;
 359                 /*
 360                  * We attach to almost all kernel functions and some of them
 361                  * will cause 'suspicious RCU usage' when fprobe is attached
 362                  * to them. Filter out the current culprits - arch_cpu_idle
 363                  * and rcu_* functions.
 364                  */
 365                 if (!strcmp(name, "arch_cpu_idle"))
 366                         continue;
 367                 if (!strncmp(name, "rcu_", 4))
 368                         continue;
 369                 if (!strcmp(name, "bpf_dispatcher_xdp_func"))
 370                         continue;
 371                 if (!strncmp(name, "__ftrace_invalid_address__",
 372                              sizeof("__ftrace_invalid_address__") - 1))
 373                         continue;
 374
 375                 err = hashmap__add(map, name, NULL);
 376                 if (err == -EEXIST)
 377                         continue;
 378                 if (err)
 379                         goto error;
 380
 381                 err = libbpf_ensure_mem((void **) &syms, &cap,
 382                                         sizeof(*syms), cnt + 1);
 383                 if (err)
 384                         goto error;
 385
 386                 syms[cnt++] = name;
 387                 name = NULL;
 388         }
 389
 390         *symsp = syms;
 391         *cntp = cnt;
 392
 393 error:
 394         free(name);
 395         fclose(f);
 396         hashmap__free(map);
 397         if (err) {
 398                 for (i = 0; i < cnt; i++)
 399                         free(syms[i]);
 400                 free(syms);
 401         }
 402         return err;
 403 }
 404
 405 static void test_bench_attach(void)
 406 {
 407         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
 408         struct kprobe_multi_empty *skel = NULL;
 409         long attach_start_ns, attach_end_ns;
 410         long detach_start_ns, detach_end_ns;
 411         double attach_delta, detach_delta;
 412         struct bpf_link *link = NULL;
 413         char **syms = NULL;
 414         size_t cnt = 0, i;
 415
 416         if (!ASSERT_OK(get_syms(&syms, &cnt), "get_syms"))
 417                 return;
 418
 419         skel = kprobe_multi_empty__open_and_load();
 420         if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
 421                 goto cleanup;
 422
 423         opts.syms = (const char **) syms;
 424         opts.cnt = cnt;
 425
 426         attach_start_ns = get_time_ns();
 427         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty,
 428                                                      NULL, &opts);
 429         attach_end_ns = get_time_ns();
 430
 431         if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
 432                 goto cleanup;
 433
 434         detach_start_ns = get_time_ns();
 435         bpf_link__destroy(link);
 436         detach_end_ns = get_time_ns();
 437
 438         attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0;
 439         detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0;
 440
 441         printf("%s: found %lu functions\n", __func__, cnt);
 442         printf("%s: attached in %7.3lfs\n", __func__, attach_delta);
 443         printf("%s: detached in %7.3lfs\n", __func__, detach_delta);
 444
 445 cleanup:
 446         kprobe_multi_empty__destroy(skel);
 447         if (syms) {
 448                 for (i = 0; i < cnt; i++)
 449                         free(syms[i]);
 450                 free(syms);
 451         }
 452 }
 453
 454 void test_kprobe_multi_test(void)
 455 {
 456         if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
 457                 return;
 458
 459         if (test__start_subtest("skel_api"))
 460                 test_skel_api();
 461         if (test__start_subtest("link_api_addrs"))
 462                 test_link_api_syms();
 463         if (test__start_subtest("link_api_syms"))
 464                 test_link_api_addrs();
 465         if (test__start_subtest("attach_api_pattern"))
 466                 test_attach_api_pattern();
 467         if (test__start_subtest("attach_api_addrs"))
 468                 test_attach_api_addrs();
 469         if (test__start_subtest("attach_api_syms"))
 470                 test_attach_api_syms();
 471         if (test__start_subtest("attach_api_fails"))
 472                 test_attach_api_fails();
 473         if (test__start_subtest("bench_attach"))
 474                 test_bench_attach();
 475 }