1 /* Copyright (c) 2017 Facebook
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
16 #include <linux/types.h>
17 typedef __u16 __sum16;
18 #include <arpa/inet.h>
19 #include <linux/if_ether.h>
20 #include <linux/if_packet.h>
22 #include <linux/ipv6.h>
23 #include <linux/tcp.h>
24 #include <linux/filter.h>
25 #include <linux/perf_event.h>
26 #include <linux/unistd.h>
28 #include <sys/ioctl.h>
30 #include <sys/types.h>
33 #include <linux/bpf.h>
34 #include <linux/err.h>
36 #include <bpf/libbpf.h>
38 #include "test_iptunnel_common.h"
40 #include "bpf_endian.h"
41 #include "bpf_rlimit.h"
42 #include "trace_helpers.h"
43 #include "flow_dissector_load.h"
45 static int error_cnt, pass_cnt;
46 static bool jit_enabled;
48 #define MAGIC_BYTES 123
50 /* ipv4 test vector */
56 .eth.h_proto = __bpf_constant_htons(ETH_P_IP),
58 .iph.protocol = IPPROTO_TCP,
59 .iph.tot_len = __bpf_constant_htons(MAGIC_BYTES),
64 /* ipv6 test vector */
70 .eth.h_proto = __bpf_constant_htons(ETH_P_IPV6),
71 .iph.nexthdr = IPPROTO_TCP,
72 .iph.payload_len = __bpf_constant_htons(MAGIC_BYTES),
77 #define _CHECK(condition, tag, duration, format...) ({ \
78 int __ret = !!(condition); \
81 printf("%s:FAIL:%s ", __func__, tag); \
85 printf("%s:PASS:%s %d nsec\n", __func__, tag, duration);\
90 #define CHECK(condition, tag, format...) \
91 _CHECK(condition, tag, duration, format)
92 #define CHECK_ATTR(condition, tag, format...) \
93 _CHECK(condition, tag, tattr.duration, format)
95 static int bpf_find_map(const char *test, struct bpf_object *obj,
100 map = bpf_object__find_map_by_name(obj, name);
102 printf("%s:FAIL:map '%s' not found\n", test, name);
106 return bpf_map__fd(map);
109 static void test_pkt_access(void)
111 const char *file = "./test_pkt_access.o";
112 struct bpf_object *obj;
113 __u32 duration, retval;
116 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
122 err = bpf_prog_test_run(prog_fd, 100000, &pkt_v4, sizeof(pkt_v4),
123 NULL, NULL, &retval, &duration);
124 CHECK(err || retval, "ipv4",
125 "err %d errno %d retval %d duration %d\n",
126 err, errno, retval, duration);
128 err = bpf_prog_test_run(prog_fd, 100000, &pkt_v6, sizeof(pkt_v6),
129 NULL, NULL, &retval, &duration);
130 CHECK(err || retval, "ipv6",
131 "err %d errno %d retval %d duration %d\n",
132 err, errno, retval, duration);
133 bpf_object__close(obj);
136 static void test_prog_run_xattr(void)
138 const char *file = "./test_pkt_access.o";
139 struct bpf_object *obj;
142 struct bpf_prog_test_run_attr tattr = {
145 .data_size_in = sizeof(pkt_v4),
150 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj,
152 if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno))
155 memset(buf, 0, sizeof(buf));
157 err = bpf_prog_test_run_xattr(&tattr);
158 CHECK_ATTR(err != -1 || errno != ENOSPC || tattr.retval, "run",
159 "err %d errno %d retval %d\n", err, errno, tattr.retval);
161 CHECK_ATTR(tattr.data_size_out != sizeof(pkt_v4), "data_size_out",
162 "incorrect output size, want %lu have %u\n",
163 sizeof(pkt_v4), tattr.data_size_out);
165 CHECK_ATTR(buf[5] != 0, "overflow",
166 "BPF_PROG_TEST_RUN ignored size hint\n");
168 tattr.data_out = NULL;
169 tattr.data_size_out = 0;
172 err = bpf_prog_test_run_xattr(&tattr);
173 CHECK_ATTR(err || errno || tattr.retval, "run_no_output",
174 "err %d errno %d retval %d\n", err, errno, tattr.retval);
176 tattr.data_size_out = 1;
177 err = bpf_prog_test_run_xattr(&tattr);
178 CHECK_ATTR(err != -EINVAL, "run_wrong_size_out", "err %d\n", err);
180 bpf_object__close(obj);
183 static void test_xdp(void)
185 struct vip key4 = {.protocol = 6, .family = AF_INET};
186 struct vip key6 = {.protocol = 6, .family = AF_INET6};
187 struct iptnl_info value4 = {.family = AF_INET};
188 struct iptnl_info value6 = {.family = AF_INET6};
189 const char *file = "./test_xdp.o";
190 struct bpf_object *obj;
192 struct ipv6hdr *iph6 = (void *)buf + sizeof(struct ethhdr);
193 struct iphdr *iph = (void *)buf + sizeof(struct ethhdr);
194 __u32 duration, retval, size;
195 int err, prog_fd, map_fd;
197 err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
203 map_fd = bpf_find_map(__func__, obj, "vip2tnl");
206 bpf_map_update_elem(map_fd, &key4, &value4, 0);
207 bpf_map_update_elem(map_fd, &key6, &value6, 0);
209 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
210 buf, &size, &retval, &duration);
212 CHECK(err || retval != XDP_TX || size != 74 ||
213 iph->protocol != IPPROTO_IPIP, "ipv4",
214 "err %d errno %d retval %d size %d\n",
215 err, errno, retval, size);
217 err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6),
218 buf, &size, &retval, &duration);
219 CHECK(err || retval != XDP_TX || size != 114 ||
220 iph6->nexthdr != IPPROTO_IPV6, "ipv6",
221 "err %d errno %d retval %d size %d\n",
222 err, errno, retval, size);
224 bpf_object__close(obj);
227 static void test_xdp_adjust_tail(void)
229 const char *file = "./test_adjust_tail.o";
230 struct bpf_object *obj;
232 __u32 duration, retval, size;
235 err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
241 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
242 buf, &size, &retval, &duration);
244 CHECK(err || retval != XDP_DROP,
245 "ipv4", "err %d errno %d retval %d size %d\n",
246 err, errno, retval, size);
248 err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6),
249 buf, &size, &retval, &duration);
250 CHECK(err || retval != XDP_TX || size != 54,
251 "ipv6", "err %d errno %d retval %d size %d\n",
252 err, errno, retval, size);
253 bpf_object__close(obj);
258 #define MAGIC_VAL 0x1234
259 #define NUM_ITER 100000
262 static void test_l4lb(const char *file)
264 unsigned int nr_cpus = bpf_num_possible_cpus();
265 struct vip key = {.protocol = 6};
269 } value = {.vip_num = VIP_NUM};
270 __u32 stats_key = VIP_NUM;
275 struct real_definition {
281 } real_def = {.dst = MAGIC_VAL};
282 __u32 ch_key = 11, real_num = 3;
283 __u32 duration, retval, size;
284 int err, i, prog_fd, map_fd;
285 __u64 bytes = 0, pkts = 0;
286 struct bpf_object *obj;
288 u32 *magic = (u32 *)buf;
290 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
296 map_fd = bpf_find_map(__func__, obj, "vip_map");
299 bpf_map_update_elem(map_fd, &key, &value, 0);
301 map_fd = bpf_find_map(__func__, obj, "ch_rings");
304 bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
306 map_fd = bpf_find_map(__func__, obj, "reals");
309 bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
311 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
312 buf, &size, &retval, &duration);
313 CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 54 ||
314 *magic != MAGIC_VAL, "ipv4",
315 "err %d errno %d retval %d size %d magic %x\n",
316 err, errno, retval, size, *magic);
318 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
319 buf, &size, &retval, &duration);
320 CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 74 ||
321 *magic != MAGIC_VAL, "ipv6",
322 "err %d errno %d retval %d size %d magic %x\n",
323 err, errno, retval, size, *magic);
325 map_fd = bpf_find_map(__func__, obj, "stats");
328 bpf_map_lookup_elem(map_fd, &stats_key, stats);
329 for (i = 0; i < nr_cpus; i++) {
330 bytes += stats[i].bytes;
331 pkts += stats[i].pkts;
333 if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
335 printf("test_l4lb:FAIL:stats %lld %lld\n", bytes, pkts);
338 bpf_object__close(obj);
341 static void test_l4lb_all(void)
343 const char *file1 = "./test_l4lb.o";
344 const char *file2 = "./test_l4lb_noinline.o";
350 static void test_xdp_noinline(void)
352 const char *file = "./test_xdp_noinline.o";
353 unsigned int nr_cpus = bpf_num_possible_cpus();
354 struct vip key = {.protocol = 6};
358 } value = {.vip_num = VIP_NUM};
359 __u32 stats_key = VIP_NUM;
364 struct real_definition {
370 } real_def = {.dst = MAGIC_VAL};
371 __u32 ch_key = 11, real_num = 3;
372 __u32 duration, retval, size;
373 int err, i, prog_fd, map_fd;
374 __u64 bytes = 0, pkts = 0;
375 struct bpf_object *obj;
377 u32 *magic = (u32 *)buf;
379 err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
385 map_fd = bpf_find_map(__func__, obj, "vip_map");
388 bpf_map_update_elem(map_fd, &key, &value, 0);
390 map_fd = bpf_find_map(__func__, obj, "ch_rings");
393 bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
395 map_fd = bpf_find_map(__func__, obj, "reals");
398 bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
400 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
401 buf, &size, &retval, &duration);
402 CHECK(err || retval != 1 || size != 54 ||
403 *magic != MAGIC_VAL, "ipv4",
404 "err %d errno %d retval %d size %d magic %x\n",
405 err, errno, retval, size, *magic);
407 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
408 buf, &size, &retval, &duration);
409 CHECK(err || retval != 1 || size != 74 ||
410 *magic != MAGIC_VAL, "ipv6",
411 "err %d errno %d retval %d size %d magic %x\n",
412 err, errno, retval, size, *magic);
414 map_fd = bpf_find_map(__func__, obj, "stats");
417 bpf_map_lookup_elem(map_fd, &stats_key, stats);
418 for (i = 0; i < nr_cpus; i++) {
419 bytes += stats[i].bytes;
420 pkts += stats[i].pkts;
422 if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
424 printf("test_xdp_noinline:FAIL:stats %lld %lld\n", bytes, pkts);
427 bpf_object__close(obj);
430 static void test_tcp_estats(void)
432 const char *file = "./test_tcp_estats.o";
434 struct bpf_object *obj;
437 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
438 CHECK(err, "", "err %d errno %d\n", err, errno);
444 bpf_object__close(obj);
447 static inline __u64 ptr_to_u64(const void *ptr)
449 return (__u64) (unsigned long) ptr;
452 static bool is_jit_enabled(void)
454 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
455 bool enabled = false;
458 sysctl_fd = open(jit_sysctl, 0, O_RDONLY);
459 if (sysctl_fd != -1) {
462 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
463 enabled = (tmpc != '0');
470 static void test_bpf_obj_id(void)
472 const __u64 array_magic_value = 0xfaceb00c;
473 const __u32 array_key = 0;
474 const int nr_iters = 2;
475 const char *file = "./test_obj_id.o";
476 const char *expected_prog_name = "test_obj_id";
477 const char *expected_map_name = "test_map_id";
478 const __u64 nsec_per_sec = 1000000000;
480 struct bpf_object *objs[nr_iters];
481 int prog_fds[nr_iters], map_fds[nr_iters];
482 /* +1 to test for the info_len returned by kernel */
483 struct bpf_prog_info prog_infos[nr_iters + 1];
484 struct bpf_map_info map_infos[nr_iters + 1];
485 /* Each prog only uses one map. +1 to test nr_map_ids
486 * returned by kernel.
488 __u32 map_ids[nr_iters + 1];
489 char jited_insns[128], xlated_insns[128], zeros[128];
490 __u32 i, next_id, info_len, nr_id_found, duration = 0;
491 struct timespec real_time_ts, boot_time_ts;
494 uid_t my_uid = getuid();
495 time_t now, load_time;
497 err = bpf_prog_get_fd_by_id(0);
498 CHECK(err >= 0 || errno != ENOENT,
499 "get-fd-by-notexist-prog-id", "err %d errno %d\n", err, errno);
501 err = bpf_map_get_fd_by_id(0);
502 CHECK(err >= 0 || errno != ENOENT,
503 "get-fd-by-notexist-map-id", "err %d errno %d\n", err, errno);
505 for (i = 0; i < nr_iters; i++)
508 /* Check bpf_obj_get_info_by_fd() */
509 bzero(zeros, sizeof(zeros));
510 for (i = 0; i < nr_iters; i++) {
512 err = bpf_prog_load(file, BPF_PROG_TYPE_SOCKET_FILTER,
513 &objs[i], &prog_fds[i]);
514 /* test_obj_id.o is a dumb prog. It should never fail
521 /* Insert a magic value to the map */
522 map_fds[i] = bpf_find_map(__func__, objs[i], "test_map_id");
523 assert(map_fds[i] >= 0);
524 err = bpf_map_update_elem(map_fds[i], &array_key,
525 &array_magic_value, 0);
528 /* Check getting map info */
529 info_len = sizeof(struct bpf_map_info) * 2;
530 bzero(&map_infos[i], info_len);
531 err = bpf_obj_get_info_by_fd(map_fds[i], &map_infos[i],
534 map_infos[i].type != BPF_MAP_TYPE_ARRAY ||
535 map_infos[i].key_size != sizeof(__u32) ||
536 map_infos[i].value_size != sizeof(__u64) ||
537 map_infos[i].max_entries != 1 ||
538 map_infos[i].map_flags != 0 ||
539 info_len != sizeof(struct bpf_map_info) ||
540 strcmp((char *)map_infos[i].name, expected_map_name),
542 "err %d errno %d type %d(%d) info_len %u(%Zu) key_size %u value_size %u max_entries %u map_flags %X name %s(%s)\n",
544 map_infos[i].type, BPF_MAP_TYPE_ARRAY,
545 info_len, sizeof(struct bpf_map_info),
546 map_infos[i].key_size,
547 map_infos[i].value_size,
548 map_infos[i].max_entries,
549 map_infos[i].map_flags,
550 map_infos[i].name, expected_map_name))
553 /* Check getting prog info */
554 info_len = sizeof(struct bpf_prog_info) * 2;
555 bzero(&prog_infos[i], info_len);
556 bzero(jited_insns, sizeof(jited_insns));
557 bzero(xlated_insns, sizeof(xlated_insns));
558 prog_infos[i].jited_prog_insns = ptr_to_u64(jited_insns);
559 prog_infos[i].jited_prog_len = sizeof(jited_insns);
560 prog_infos[i].xlated_prog_insns = ptr_to_u64(xlated_insns);
561 prog_infos[i].xlated_prog_len = sizeof(xlated_insns);
562 prog_infos[i].map_ids = ptr_to_u64(map_ids + i);
563 prog_infos[i].nr_map_ids = 2;
564 err = clock_gettime(CLOCK_REALTIME, &real_time_ts);
566 err = clock_gettime(CLOCK_BOOTTIME, &boot_time_ts);
568 err = bpf_obj_get_info_by_fd(prog_fds[i], &prog_infos[i],
570 load_time = (real_time_ts.tv_sec - boot_time_ts.tv_sec)
571 + (prog_infos[i].load_time / nsec_per_sec);
573 prog_infos[i].type != BPF_PROG_TYPE_SOCKET_FILTER ||
574 info_len != sizeof(struct bpf_prog_info) ||
575 (jit_enabled && !prog_infos[i].jited_prog_len) ||
577 !memcmp(jited_insns, zeros, sizeof(zeros))) ||
578 !prog_infos[i].xlated_prog_len ||
579 !memcmp(xlated_insns, zeros, sizeof(zeros)) ||
580 load_time < now - 60 || load_time > now + 60 ||
581 prog_infos[i].created_by_uid != my_uid ||
582 prog_infos[i].nr_map_ids != 1 ||
583 *(int *)(long)prog_infos[i].map_ids != map_infos[i].id ||
584 strcmp((char *)prog_infos[i].name, expected_prog_name),
586 "err %d errno %d i %d type %d(%d) info_len %u(%Zu) jit_enabled %d jited_prog_len %u xlated_prog_len %u jited_prog %d xlated_prog %d load_time %lu(%lu) uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) name %s(%s)\n",
588 prog_infos[i].type, BPF_PROG_TYPE_SOCKET_FILTER,
589 info_len, sizeof(struct bpf_prog_info),
591 prog_infos[i].jited_prog_len,
592 prog_infos[i].xlated_prog_len,
593 !!memcmp(jited_insns, zeros, sizeof(zeros)),
594 !!memcmp(xlated_insns, zeros, sizeof(zeros)),
596 prog_infos[i].created_by_uid, my_uid,
597 prog_infos[i].nr_map_ids, 1,
598 *(int *)(long)prog_infos[i].map_ids, map_infos[i].id,
599 prog_infos[i].name, expected_prog_name))
603 /* Check bpf_prog_get_next_id() */
606 while (!bpf_prog_get_next_id(next_id, &next_id)) {
607 struct bpf_prog_info prog_info = {};
611 info_len = sizeof(prog_info);
613 prog_fd = bpf_prog_get_fd_by_id(next_id);
614 if (prog_fd < 0 && errno == ENOENT)
615 /* The bpf_prog is in the dead row */
617 if (CHECK(prog_fd < 0, "get-prog-fd(next_id)",
618 "prog_fd %d next_id %d errno %d\n",
619 prog_fd, next_id, errno))
622 for (i = 0; i < nr_iters; i++)
623 if (prog_infos[i].id == next_id)
632 * prog_info.nr_map_ids = 1
633 * prog_info.map_ids = NULL
635 prog_info.nr_map_ids = 1;
636 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
637 if (CHECK(!err || errno != EFAULT,
638 "get-prog-fd-bad-nr-map-ids", "err %d errno %d(%d)",
641 bzero(&prog_info, sizeof(prog_info));
642 info_len = sizeof(prog_info);
644 saved_map_id = *(int *)((long)prog_infos[i].map_ids);
645 prog_info.map_ids = prog_infos[i].map_ids;
646 prog_info.nr_map_ids = 2;
647 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
648 prog_infos[i].jited_prog_insns = 0;
649 prog_infos[i].xlated_prog_insns = 0;
650 CHECK(err || info_len != sizeof(struct bpf_prog_info) ||
651 memcmp(&prog_info, &prog_infos[i], info_len) ||
652 *(int *)(long)prog_info.map_ids != saved_map_id,
653 "get-prog-info(next_id->fd)",
654 "err %d errno %d info_len %u(%Zu) memcmp %d map_id %u(%u)\n",
655 err, errno, info_len, sizeof(struct bpf_prog_info),
656 memcmp(&prog_info, &prog_infos[i], info_len),
657 *(int *)(long)prog_info.map_ids, saved_map_id);
660 CHECK(nr_id_found != nr_iters,
661 "check total prog id found by get_next_id",
662 "nr_id_found %u(%u)\n",
663 nr_id_found, nr_iters);
665 /* Check bpf_map_get_next_id() */
668 while (!bpf_map_get_next_id(next_id, &next_id)) {
669 struct bpf_map_info map_info = {};
672 info_len = sizeof(map_info);
674 map_fd = bpf_map_get_fd_by_id(next_id);
675 if (map_fd < 0 && errno == ENOENT)
676 /* The bpf_map is in the dead row */
678 if (CHECK(map_fd < 0, "get-map-fd(next_id)",
679 "map_fd %d next_id %u errno %d\n",
680 map_fd, next_id, errno))
683 for (i = 0; i < nr_iters; i++)
684 if (map_infos[i].id == next_id)
692 err = bpf_map_lookup_elem(map_fd, &array_key, &array_value);
695 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &info_len);
696 CHECK(err || info_len != sizeof(struct bpf_map_info) ||
697 memcmp(&map_info, &map_infos[i], info_len) ||
698 array_value != array_magic_value,
699 "check get-map-info(next_id->fd)",
700 "err %d errno %d info_len %u(%Zu) memcmp %d array_value %llu(%llu)\n",
701 err, errno, info_len, sizeof(struct bpf_map_info),
702 memcmp(&map_info, &map_infos[i], info_len),
703 array_value, array_magic_value);
707 CHECK(nr_id_found != nr_iters,
708 "check total map id found by get_next_id",
709 "nr_id_found %u(%u)\n",
710 nr_id_found, nr_iters);
713 for (i = 0; i < nr_iters; i++)
714 bpf_object__close(objs[i]);
717 static void test_pkt_md_access(void)
719 const char *file = "./test_pkt_md_access.o";
720 struct bpf_object *obj;
721 __u32 duration, retval;
724 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
730 err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4),
731 NULL, NULL, &retval, &duration);
732 CHECK(err || retval, "",
733 "err %d errno %d retval %d duration %d\n",
734 err, errno, retval, duration);
736 bpf_object__close(obj);
739 static void test_obj_name(void)
747 { "_123456789ABCDE", 1, 0 },
748 { "_123456789ABCDEF", 0, EINVAL },
749 { "_123456789ABCD\n", 0, EINVAL },
751 struct bpf_insn prog[] = {
752 BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0),
758 for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) {
759 size_t name_len = strlen(tests[i].name) + 1;
764 /* test different attr.prog_name during BPF_PROG_LOAD */
765 ncopy = name_len < sizeof(attr.prog_name) ?
766 name_len : sizeof(attr.prog_name);
767 bzero(&attr, sizeof(attr));
768 attr.prog_type = BPF_PROG_TYPE_SCHED_CLS;
770 attr.insns = ptr_to_u64(prog);
771 attr.license = ptr_to_u64("");
772 memcpy(attr.prog_name, tests[i].name, ncopy);
774 fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr));
775 CHECK((tests[i].success && fd < 0) ||
776 (!tests[i].success && fd != -1) ||
777 (!tests[i].success && errno != tests[i].expected_errno),
778 "check-bpf-prog-name",
779 "fd %d(%d) errno %d(%d)\n",
780 fd, tests[i].success, errno, tests[i].expected_errno);
785 /* test different attr.map_name during BPF_MAP_CREATE */
786 ncopy = name_len < sizeof(attr.map_name) ?
787 name_len : sizeof(attr.map_name);
788 bzero(&attr, sizeof(attr));
789 attr.map_type = BPF_MAP_TYPE_ARRAY;
792 attr.max_entries = 1;
794 memcpy(attr.map_name, tests[i].name, ncopy);
795 fd = syscall(__NR_bpf, BPF_MAP_CREATE, &attr, sizeof(attr));
796 CHECK((tests[i].success && fd < 0) ||
797 (!tests[i].success && fd != -1) ||
798 (!tests[i].success && errno != tests[i].expected_errno),
799 "check-bpf-map-name",
800 "fd %d(%d) errno %d(%d)\n",
801 fd, tests[i].success, errno, tests[i].expected_errno);
808 static void test_tp_attach_query(void)
810 const int num_progs = 3;
811 int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs];
812 __u32 duration = 0, info_len, saved_prog_ids[num_progs];
813 const char *file = "./test_tracepoint.o";
814 struct perf_event_query_bpf *query;
815 struct perf_event_attr attr = {};
816 struct bpf_object *obj[num_progs];
817 struct bpf_prog_info prog_info;
820 snprintf(buf, sizeof(buf),
821 "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
822 efd = open(buf, O_RDONLY, 0);
823 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
825 bytes = read(efd, buf, sizeof(buf));
827 if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
828 "read", "bytes %d errno %d\n", bytes, errno))
831 attr.config = strtol(buf, NULL, 0);
832 attr.type = PERF_TYPE_TRACEPOINT;
833 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
834 attr.sample_period = 1;
835 attr.wakeup_events = 1;
837 query = malloc(sizeof(*query) + sizeof(__u32) * num_progs);
838 for (i = 0; i < num_progs; i++) {
839 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj[i],
841 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
844 bzero(&prog_info, sizeof(prog_info));
845 prog_info.jited_prog_len = 0;
846 prog_info.xlated_prog_len = 0;
847 prog_info.nr_map_ids = 0;
848 info_len = sizeof(prog_info);
849 err = bpf_obj_get_info_by_fd(prog_fd[i], &prog_info, &info_len);
850 if (CHECK(err, "bpf_obj_get_info_by_fd", "err %d errno %d\n",
853 saved_prog_ids[i] = prog_info.id;
855 pmu_fd[i] = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
856 0 /* cpu 0 */, -1 /* group id */,
858 if (CHECK(pmu_fd[i] < 0, "perf_event_open", "err %d errno %d\n",
861 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_ENABLE, 0);
862 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
867 /* check NULL prog array query */
868 query->ids_len = num_progs;
869 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
870 if (CHECK(err || query->prog_cnt != 0,
871 "perf_event_ioc_query_bpf",
872 "err %d errno %d query->prog_cnt %u\n",
873 err, errno, query->prog_cnt))
877 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_SET_BPF, prog_fd[i]);
878 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
883 /* try to get # of programs only */
885 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
886 if (CHECK(err || query->prog_cnt != 2,
887 "perf_event_ioc_query_bpf",
888 "err %d errno %d query->prog_cnt %u\n",
889 err, errno, query->prog_cnt))
892 /* try a few negative tests */
893 /* invalid query pointer */
894 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF,
895 (struct perf_event_query_bpf *)0x1);
896 if (CHECK(!err || errno != EFAULT,
897 "perf_event_ioc_query_bpf",
898 "err %d errno %d\n", err, errno))
901 /* no enough space */
903 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
904 if (CHECK(!err || errno != ENOSPC || query->prog_cnt != 2,
905 "perf_event_ioc_query_bpf",
906 "err %d errno %d query->prog_cnt %u\n",
907 err, errno, query->prog_cnt))
911 query->ids_len = num_progs;
912 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
913 if (CHECK(err || query->prog_cnt != (i + 1),
914 "perf_event_ioc_query_bpf",
915 "err %d errno %d query->prog_cnt %u\n",
916 err, errno, query->prog_cnt))
918 for (j = 0; j < i + 1; j++)
919 if (CHECK(saved_prog_ids[j] != query->ids[j],
920 "perf_event_ioc_query_bpf",
921 "#%d saved_prog_id %x query prog_id %x\n",
922 j, saved_prog_ids[j], query->ids[j]))
927 for (; i >= 0; i--) {
929 ioctl(pmu_fd[i], PERF_EVENT_IOC_DISABLE);
933 bpf_object__close(obj[i]);
938 static int compare_map_keys(int map1_fd, int map2_fd)
941 char val_buf[PERF_MAX_STACK_DEPTH *
942 sizeof(struct bpf_stack_build_id)];
945 err = bpf_map_get_next_key(map1_fd, NULL, &key);
948 err = bpf_map_lookup_elem(map2_fd, &key, val_buf);
952 while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) {
953 err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf);
965 static int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len)
967 __u32 key, next_key, *cur_key_p, *next_key_p;
968 char *val_buf1, *val_buf2;
971 val_buf1 = malloc(stack_trace_len);
972 val_buf2 = malloc(stack_trace_len);
975 while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) {
976 err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1);
979 err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2);
982 for (i = 0; i < stack_trace_len; i++) {
983 if (val_buf1[i] != val_buf2[i]) {
990 next_key_p = &next_key;
1001 static void test_stacktrace_map()
1003 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1004 const char *file = "./test_stacktrace_map.o";
1005 int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1006 struct perf_event_attr attr = {};
1007 __u32 key, val, duration = 0;
1008 struct bpf_object *obj;
1011 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1012 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1015 /* Get the ID for the sched/sched_switch tracepoint */
1016 snprintf(buf, sizeof(buf),
1017 "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
1018 efd = open(buf, O_RDONLY, 0);
1019 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1022 bytes = read(efd, buf, sizeof(buf));
1024 if (bytes <= 0 || bytes >= sizeof(buf))
1027 /* Open the perf event and attach bpf progrram */
1028 attr.config = strtol(buf, NULL, 0);
1029 attr.type = PERF_TYPE_TRACEPOINT;
1030 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
1031 attr.sample_period = 1;
1032 attr.wakeup_events = 1;
1033 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1034 0 /* cpu 0 */, -1 /* group id */,
1036 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
1040 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1044 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1049 control_map_fd = bpf_find_map(__func__, obj, "control_map");
1050 if (control_map_fd < 0)
1053 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1054 if (stackid_hmap_fd < 0)
1057 stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1058 if (stackmap_fd < 0)
1061 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1062 if (stack_amap_fd < 0)
1065 /* give some time for bpf program run */
1068 /* disable stack trace collection */
1071 bpf_map_update_elem(control_map_fd, &key, &val, 0);
1073 /* for every element in stackid_hmap, we can find a corresponding one
1074 * in stackmap, and vise versa.
1076 err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1077 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1078 "err %d errno %d\n", err, errno))
1079 goto disable_pmu_noerr;
1081 err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1082 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1083 "err %d errno %d\n", err, errno))
1084 goto disable_pmu_noerr;
1086 stack_trace_len = PERF_MAX_STACK_DEPTH * sizeof(__u64);
1087 err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1088 if (CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1089 "err %d errno %d\n", err, errno))
1090 goto disable_pmu_noerr;
1092 goto disable_pmu_noerr;
1096 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1099 bpf_object__close(obj);
1102 static void test_stacktrace_map_raw_tp()
1104 int control_map_fd, stackid_hmap_fd, stackmap_fd;
1105 const char *file = "./test_stacktrace_map.o";
1106 int efd, err, prog_fd;
1107 __u32 key, val, duration = 0;
1108 struct bpf_object *obj;
1110 err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1111 if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1114 efd = bpf_raw_tracepoint_open("sched_switch", prog_fd);
1115 if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1119 control_map_fd = bpf_find_map(__func__, obj, "control_map");
1120 if (control_map_fd < 0)
1123 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1124 if (stackid_hmap_fd < 0)
1127 stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1128 if (stackmap_fd < 0)
1131 /* give some time for bpf program run */
1134 /* disable stack trace collection */
1137 bpf_map_update_elem(control_map_fd, &key, &val, 0);
1139 /* for every element in stackid_hmap, we can find a corresponding one
1140 * in stackmap, and vise versa.
1142 err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1143 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1144 "err %d errno %d\n", err, errno))
1147 err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1148 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1149 "err %d errno %d\n", err, errno))
1152 goto close_prog_noerr;
1156 bpf_object__close(obj);
1159 static int extract_build_id(char *build_id, size_t size)
1165 fp = popen("readelf -n ./urandom_read | grep 'Build ID'", "r");
1169 if (getline(&line, &len, fp) == -1)
1175 memcpy(build_id, line, len);
1176 build_id[len] = '\0';
1183 static void test_stacktrace_build_id(void)
1185 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1186 const char *file = "./test_stacktrace_build_id.o";
1187 int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1188 struct perf_event_attr attr = {};
1189 __u32 key, previous_key, val, duration = 0;
1190 struct bpf_object *obj;
1193 struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
1194 int build_id_matches = 0;
1198 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1199 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1202 /* Get the ID for the sched/sched_switch tracepoint */
1203 snprintf(buf, sizeof(buf),
1204 "/sys/kernel/debug/tracing/events/random/urandom_read/id");
1205 efd = open(buf, O_RDONLY, 0);
1206 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1209 bytes = read(efd, buf, sizeof(buf));
1211 if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
1212 "read", "bytes %d errno %d\n", bytes, errno))
1215 /* Open the perf event and attach bpf progrram */
1216 attr.config = strtol(buf, NULL, 0);
1217 attr.type = PERF_TYPE_TRACEPOINT;
1218 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
1219 attr.sample_period = 1;
1220 attr.wakeup_events = 1;
1221 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1222 0 /* cpu 0 */, -1 /* group id */,
1224 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
1228 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1229 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
1233 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1234 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
1239 control_map_fd = bpf_find_map(__func__, obj, "control_map");
1240 if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
1241 "err %d errno %d\n", err, errno))
1244 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1245 if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
1246 "err %d errno %d\n", err, errno))
1249 stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1250 if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
1254 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1255 if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1256 "err %d errno %d\n", err, errno))
1259 assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1261 assert(system("./urandom_read") == 0);
1262 /* disable stack trace collection */
1265 bpf_map_update_elem(control_map_fd, &key, &val, 0);
1267 /* for every element in stackid_hmap, we can find a corresponding one
1268 * in stackmap, and vise versa.
1270 err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1271 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1272 "err %d errno %d\n", err, errno))
1275 err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1276 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1277 "err %d errno %d\n", err, errno))
1280 err = extract_build_id(buf, 256);
1282 if (CHECK(err, "get build_id with readelf",
1283 "err %d errno %d\n", err, errno))
1286 err = bpf_map_get_next_key(stackmap_fd, NULL, &key);
1287 if (CHECK(err, "get_next_key from stackmap",
1288 "err %d, errno %d\n", err, errno))
1294 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs);
1295 if (CHECK(err, "lookup_elem from stackmap",
1296 "err %d, errno %d\n", err, errno))
1298 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i)
1299 if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID &&
1300 id_offs[i].offset != 0) {
1301 for (j = 0; j < 20; ++j)
1302 sprintf(build_id + 2 * j, "%02x",
1303 id_offs[i].build_id[j] & 0xff);
1304 if (strstr(buf, build_id) != NULL)
1305 build_id_matches = 1;
1308 } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1310 /* stack_map_get_build_id_offset() is racy and sometimes can return
1311 * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID;
1312 * try it one more time.
1314 if (build_id_matches < 1 && retry--) {
1315 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1317 bpf_object__close(obj);
1318 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
1323 if (CHECK(build_id_matches < 1, "build id match",
1324 "Didn't find expected build ID from the map\n"))
1327 stack_trace_len = PERF_MAX_STACK_DEPTH
1328 * sizeof(struct bpf_stack_build_id);
1329 err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1330 CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1331 "err %d errno %d\n", err, errno);
1334 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1340 bpf_object__close(obj);
1346 static void test_stacktrace_build_id_nmi(void)
1348 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1349 const char *file = "./test_stacktrace_build_id.o";
1350 int err, pmu_fd, prog_fd;
1351 struct perf_event_attr attr = {
1352 .sample_freq = 5000,
1354 .type = PERF_TYPE_HARDWARE,
1355 .config = PERF_COUNT_HW_CPU_CYCLES,
1357 __u32 key, previous_key, val, duration = 0;
1358 struct bpf_object *obj;
1361 struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
1362 int build_id_matches = 0;
1366 err = bpf_prog_load(file, BPF_PROG_TYPE_PERF_EVENT, &obj, &prog_fd);
1367 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1370 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1371 0 /* cpu 0 */, -1 /* group id */,
1373 if (CHECK(pmu_fd < 0, "perf_event_open",
1374 "err %d errno %d. Does the test host support PERF_COUNT_HW_CPU_CYCLES?\n",
1378 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1379 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
1383 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1384 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
1389 control_map_fd = bpf_find_map(__func__, obj, "control_map");
1390 if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
1391 "err %d errno %d\n", err, errno))
1394 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1395 if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
1396 "err %d errno %d\n", err, errno))
1399 stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1400 if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
1404 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1405 if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1406 "err %d errno %d\n", err, errno))
1409 assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1411 assert(system("taskset 0x1 ./urandom_read 100000") == 0);
1412 /* disable stack trace collection */
1415 bpf_map_update_elem(control_map_fd, &key, &val, 0);
1417 /* for every element in stackid_hmap, we can find a corresponding one
1418 * in stackmap, and vise versa.
1420 err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1421 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1422 "err %d errno %d\n", err, errno))
1425 err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1426 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1427 "err %d errno %d\n", err, errno))
1430 err = extract_build_id(buf, 256);
1432 if (CHECK(err, "get build_id with readelf",
1433 "err %d errno %d\n", err, errno))
1436 err = bpf_map_get_next_key(stackmap_fd, NULL, &key);
1437 if (CHECK(err, "get_next_key from stackmap",
1438 "err %d, errno %d\n", err, errno))
1444 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs);
1445 if (CHECK(err, "lookup_elem from stackmap",
1446 "err %d, errno %d\n", err, errno))
1448 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i)
1449 if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID &&
1450 id_offs[i].offset != 0) {
1451 for (j = 0; j < 20; ++j)
1452 sprintf(build_id + 2 * j, "%02x",
1453 id_offs[i].build_id[j] & 0xff);
1454 if (strstr(buf, build_id) != NULL)
1455 build_id_matches = 1;
1458 } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1460 /* stack_map_get_build_id_offset() is racy and sometimes can return
1461 * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID;
1462 * try it one more time.
1464 if (build_id_matches < 1 && retry--) {
1465 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1467 bpf_object__close(obj);
1468 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
1473 if (CHECK(build_id_matches < 1, "build id match",
1474 "Didn't find expected build ID from the map\n"))
1478 * We intentionally skip compare_stack_ips(). This is because we
1479 * only support one in_nmi() ips-to-build_id translation per cpu
1480 * at any time, thus stack_amap here will always fallback to
1481 * BPF_STACK_BUILD_ID_IP;
1485 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1491 bpf_object__close(obj);
1494 #define MAX_CNT_RAWTP 10ull
1495 #define MAX_STACK_RAWTP 100
1496 struct get_stack_trace_t {
1498 int kern_stack_size;
1499 int user_stack_size;
1500 int user_stack_buildid_size;
1501 __u64 kern_stack[MAX_STACK_RAWTP];
1502 __u64 user_stack[MAX_STACK_RAWTP];
1503 struct bpf_stack_build_id user_stack_buildid[MAX_STACK_RAWTP];
1506 static int get_stack_print_output(void *data, int size)
1508 bool good_kern_stack = false, good_user_stack = false;
1509 const char *nonjit_func = "___bpf_prog_run";
1510 struct get_stack_trace_t *e = data;
1517 if (size < sizeof(struct get_stack_trace_t)) {
1518 __u64 *raw_data = data;
1521 num_stack = size / sizeof(__u64);
1522 /* If jit is enabled, we do not have a good way to
1523 * verify the sanity of the kernel stack. So we
1524 * just assume it is good if the stack is not empty.
1525 * This could be improved in the future.
1528 found = num_stack > 0;
1530 for (i = 0; i < num_stack; i++) {
1531 ks = ksym_search(raw_data[i]);
1532 if (strcmp(ks->name, nonjit_func) == 0) {
1539 good_kern_stack = true;
1540 good_user_stack = true;
1543 num_stack = e->kern_stack_size / sizeof(__u64);
1545 good_kern_stack = num_stack > 0;
1547 for (i = 0; i < num_stack; i++) {
1548 ks = ksym_search(e->kern_stack[i]);
1549 if (strcmp(ks->name, nonjit_func) == 0) {
1550 good_kern_stack = true;
1555 if (e->user_stack_size > 0 && e->user_stack_buildid_size > 0)
1556 good_user_stack = true;
1558 if (!good_kern_stack || !good_user_stack)
1559 return LIBBPF_PERF_EVENT_ERROR;
1561 if (cnt == MAX_CNT_RAWTP)
1562 return LIBBPF_PERF_EVENT_DONE;
1564 return LIBBPF_PERF_EVENT_CONT;
1567 static void test_get_stack_raw_tp(void)
1569 const char *file = "./test_get_stack_rawtp.o";
1570 int i, efd, err, prog_fd, pmu_fd, perfmap_fd;
1571 struct perf_event_attr attr = {};
1572 struct timespec tv = {0, 10};
1573 __u32 key = 0, duration = 0;
1574 struct bpf_object *obj;
1576 err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1577 if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1580 efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1581 if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1584 perfmap_fd = bpf_find_map(__func__, obj, "perfmap");
1585 if (CHECK(perfmap_fd < 0, "bpf_find_map", "err %d errno %d\n",
1589 err = load_kallsyms();
1590 if (CHECK(err < 0, "load_kallsyms", "err %d errno %d\n", err, errno))
1593 attr.sample_type = PERF_SAMPLE_RAW;
1594 attr.type = PERF_TYPE_SOFTWARE;
1595 attr.config = PERF_COUNT_SW_BPF_OUTPUT;
1596 pmu_fd = syscall(__NR_perf_event_open, &attr, getpid()/*pid*/, -1/*cpu*/,
1598 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", pmu_fd,
1602 err = bpf_map_update_elem(perfmap_fd, &key, &pmu_fd, BPF_ANY);
1603 if (CHECK(err < 0, "bpf_map_update_elem", "err %d errno %d\n", err,
1607 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1608 if (CHECK(err < 0, "ioctl PERF_EVENT_IOC_ENABLE", "err %d errno %d\n",
1612 err = perf_event_mmap(pmu_fd);
1613 if (CHECK(err < 0, "perf_event_mmap", "err %d errno %d\n", err, errno))
1616 /* trigger some syscall action */
1617 for (i = 0; i < MAX_CNT_RAWTP; i++)
1618 nanosleep(&tv, NULL);
1620 err = perf_event_poller(pmu_fd, get_stack_print_output);
1621 if (CHECK(err < 0, "perf_event_poller", "err %d errno %d\n", err, errno))
1624 goto close_prog_noerr;
1628 bpf_object__close(obj);
1631 static void test_task_fd_query_rawtp(void)
1633 const char *file = "./test_get_stack_rawtp.o";
1634 __u64 probe_offset, probe_addr;
1635 __u32 len, prog_id, fd_type;
1636 struct bpf_object *obj;
1637 int efd, err, prog_fd;
1641 err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1642 if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1645 efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1646 if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1649 /* query (getpid(), efd) */
1651 err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1652 &fd_type, &probe_offset, &probe_addr);
1653 if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err,
1657 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1658 strcmp(buf, "sys_enter") == 0;
1659 if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n",
1665 err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1666 &fd_type, &probe_offset, &probe_addr);
1667 if (CHECK(err < 0, "bpf_task_fd_query (len = 0)", "err %d errno %d\n",
1670 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1671 len == strlen("sys_enter");
1672 if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1675 /* test empty buffer */
1677 err = bpf_task_fd_query(getpid(), efd, 0, 0, &len, &prog_id,
1678 &fd_type, &probe_offset, &probe_addr);
1679 if (CHECK(err < 0, "bpf_task_fd_query (buf = 0)", "err %d errno %d\n",
1682 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1683 len == strlen("sys_enter");
1684 if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1687 /* test smaller buffer */
1689 err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1690 &fd_type, &probe_offset, &probe_addr);
1691 if (CHECK(err >= 0 || errno != ENOSPC, "bpf_task_fd_query (len = 3)",
1692 "err %d errno %d\n", err, errno))
1694 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1695 len == strlen("sys_enter") &&
1696 strcmp(buf, "sy") == 0;
1697 if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1700 goto close_prog_noerr;
1704 bpf_object__close(obj);
1707 static void test_task_fd_query_tp_core(const char *probe_name,
1708 const char *tp_name)
1710 const char *file = "./test_tracepoint.o";
1711 int err, bytes, efd, prog_fd, pmu_fd;
1712 struct perf_event_attr attr = {};
1713 __u64 probe_offset, probe_addr;
1714 __u32 len, prog_id, fd_type;
1715 struct bpf_object *obj;
1719 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1720 if (CHECK(err, "bpf_prog_load", "err %d errno %d\n", err, errno))
1723 snprintf(buf, sizeof(buf),
1724 "/sys/kernel/debug/tracing/events/%s/id", probe_name);
1725 efd = open(buf, O_RDONLY, 0);
1726 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1728 bytes = read(efd, buf, sizeof(buf));
1730 if (CHECK(bytes <= 0 || bytes >= sizeof(buf), "read",
1731 "bytes %d errno %d\n", bytes, errno))
1734 attr.config = strtol(buf, NULL, 0);
1735 attr.type = PERF_TYPE_TRACEPOINT;
1736 attr.sample_type = PERF_SAMPLE_RAW;
1737 attr.sample_period = 1;
1738 attr.wakeup_events = 1;
1739 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1740 0 /* cpu 0 */, -1 /* group id */,
1742 if (CHECK(err, "perf_event_open", "err %d errno %d\n", err, errno))
1745 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1746 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", err,
1750 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1751 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", err,
1755 /* query (getpid(), pmu_fd) */
1757 err = bpf_task_fd_query(getpid(), pmu_fd, 0, buf, &len, &prog_id,
1758 &fd_type, &probe_offset, &probe_addr);
1759 if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err,
1763 err = (fd_type == BPF_FD_TYPE_TRACEPOINT) && !strcmp(buf, tp_name);
1764 if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n",
1769 goto close_prog_noerr;
1776 bpf_object__close(obj);
1779 static void test_task_fd_query_tp(void)
1781 test_task_fd_query_tp_core("sched/sched_switch",
1783 test_task_fd_query_tp_core("syscalls/sys_enter_read",
1787 static int libbpf_debug_print(enum libbpf_print_level level,
1788 const char *format, ...)
1793 if (level == LIBBPF_DEBUG)
1796 va_start(args, format);
1797 ret = vfprintf(stderr, format, args);
1802 static void test_reference_tracking()
1804 const char *file = "./test_sk_lookup_kern.o";
1805 struct bpf_object *obj;
1806 struct bpf_program *prog;
1810 obj = bpf_object__open(file);
1816 bpf_object__for_each_program(prog, obj) {
1819 /* Ignore .text sections */
1820 title = bpf_program__title(prog, false);
1821 if (strstr(title, ".text") != NULL)
1824 bpf_program__set_type(prog, BPF_PROG_TYPE_SCHED_CLS);
1826 /* Expect verifier failure if test name has 'fail' */
1827 if (strstr(title, "fail") != NULL) {
1828 libbpf_set_print(NULL);
1829 err = !bpf_program__load(prog, "GPL", 0);
1830 libbpf_set_print(libbpf_debug_print);
1832 err = bpf_program__load(prog, "GPL", 0);
1834 CHECK(err, title, "\n");
1836 bpf_object__close(obj);
1844 static void test_queue_stack_map(int type)
1846 const int MAP_SIZE = 32;
1847 __u32 vals[MAP_SIZE], duration, retval, size, val;
1848 int i, err, prog_fd, map_in_fd, map_out_fd;
1849 char file[32], buf[128];
1850 struct bpf_object *obj;
1851 struct iphdr *iph = (void *)buf + sizeof(struct ethhdr);
1853 /* Fill test values to be used */
1854 for (i = 0; i < MAP_SIZE; i++)
1858 strncpy(file, "./test_queue_map.o", sizeof(file));
1859 else if (type == STACK)
1860 strncpy(file, "./test_stack_map.o", sizeof(file));
1864 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
1870 map_in_fd = bpf_find_map(__func__, obj, "map_in");
1874 map_out_fd = bpf_find_map(__func__, obj, "map_out");
1878 /* Push 32 elements to the input map */
1879 for (i = 0; i < MAP_SIZE; i++) {
1880 err = bpf_map_update_elem(map_in_fd, NULL, &vals[i], 0);
1887 /* The eBPF program pushes iph.saddr in the output map,
1888 * pops the input map and saves this value in iph.daddr
1890 for (i = 0; i < MAP_SIZE; i++) {
1891 if (type == QUEUE) {
1893 pkt_v4.iph.saddr = vals[i] * 5;
1894 } else if (type == STACK) {
1895 val = vals[MAP_SIZE - 1 - i];
1896 pkt_v4.iph.saddr = vals[MAP_SIZE - 1 - i] * 5;
1899 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
1900 buf, &size, &retval, &duration);
1901 if (err || retval || size != sizeof(pkt_v4) ||
1906 CHECK(err || retval || size != sizeof(pkt_v4) || iph->daddr != val,
1908 "err %d errno %d retval %d size %d iph->daddr %u\n",
1909 err, errno, retval, size, iph->daddr);
1911 /* Queue is empty, program should return TC_ACT_SHOT */
1912 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
1913 buf, &size, &retval, &duration);
1914 CHECK(err || retval != 2 /* TC_ACT_SHOT */|| size != sizeof(pkt_v4),
1915 "check-queue-stack-map-empty",
1916 "err %d errno %d retval %d size %d\n",
1917 err, errno, retval, size);
1919 /* Check that the program pushed elements correctly */
1920 for (i = 0; i < MAP_SIZE; i++) {
1921 err = bpf_map_lookup_and_delete_elem(map_out_fd, NULL, &val);
1922 if (err || val != vals[i] * 5)
1926 CHECK(i != MAP_SIZE && (err || val != vals[i] * 5),
1927 "bpf_map_push_elem", "err %d value %u\n", err, val);
1930 pkt_v4.iph.saddr = 0;
1931 bpf_object__close(obj);
1934 #define CHECK_FLOW_KEYS(desc, got, expected) \
1935 CHECK(memcmp(&got, &expected, sizeof(got)) != 0, \
1939 "addr_proto=0x%x/0x%x " \
1941 "is_first_frag=%u/%u " \
1943 "n_proto=0x%x/0x%x " \
1946 got.nhoff, expected.nhoff, \
1947 got.thoff, expected.thoff, \
1948 got.addr_proto, expected.addr_proto, \
1949 got.is_frag, expected.is_frag, \
1950 got.is_first_frag, expected.is_first_frag, \
1951 got.is_encap, expected.is_encap, \
1952 got.n_proto, expected.n_proto, \
1953 got.sport, expected.sport, \
1954 got.dport, expected.dport)
1956 static struct bpf_flow_keys pkt_v4_flow_keys = {
1958 .thoff = sizeof(struct iphdr),
1959 .addr_proto = ETH_P_IP,
1960 .ip_proto = IPPROTO_TCP,
1961 .n_proto = bpf_htons(ETH_P_IP),
1964 static struct bpf_flow_keys pkt_v6_flow_keys = {
1966 .thoff = sizeof(struct ipv6hdr),
1967 .addr_proto = ETH_P_IPV6,
1968 .ip_proto = IPPROTO_TCP,
1969 .n_proto = bpf_htons(ETH_P_IPV6),
1972 static void test_flow_dissector(void)
1974 struct bpf_flow_keys flow_keys;
1975 struct bpf_object *obj;
1976 __u32 duration, retval;
1980 err = bpf_flow_load(&obj, "./bpf_flow.o", "flow_dissector",
1981 "jmp_table", &prog_fd);
1987 err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4),
1988 &flow_keys, &size, &retval, &duration);
1989 CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv4",
1990 "err %d errno %d retval %d duration %d size %u/%lu\n",
1991 err, errno, retval, duration, size, sizeof(flow_keys));
1992 CHECK_FLOW_KEYS("ipv4_flow_keys", flow_keys, pkt_v4_flow_keys);
1994 err = bpf_prog_test_run(prog_fd, 10, &pkt_v6, sizeof(pkt_v6),
1995 &flow_keys, &size, &retval, &duration);
1996 CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv6",
1997 "err %d errno %d retval %d duration %d size %u/%lu\n",
1998 err, errno, retval, duration, size, sizeof(flow_keys));
1999 CHECK_FLOW_KEYS("ipv6_flow_keys", flow_keys, pkt_v6_flow_keys);
2001 bpf_object__close(obj);
2004 static void *test_spin_lock(void *arg)
2006 __u32 duration, retval;
2007 int err, prog_fd = *(u32 *) arg;
2009 err = bpf_prog_test_run(prog_fd, 10000, &pkt_v4, sizeof(pkt_v4),
2010 NULL, NULL, &retval, &duration);
2011 CHECK(err || retval, "",
2012 "err %d errno %d retval %d duration %d\n",
2013 err, errno, retval, duration);
2017 static void test_spinlock(void)
2019 const char *file = "./test_spin_lock.o";
2020 pthread_t thread_id[4];
2021 struct bpf_object *obj;
2026 err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd);
2028 printf("test_spin_lock:bpf_prog_load errno %d\n", errno);
2031 for (i = 0; i < 4; i++)
2032 assert(pthread_create(&thread_id[i], NULL,
2033 &test_spin_lock, &prog_fd) == 0);
2034 for (i = 0; i < 4; i++)
2035 assert(pthread_join(thread_id[i], &ret) == 0 &&
2036 ret == (void *)&prog_fd);
2037 goto close_prog_noerr;
2041 bpf_object__close(obj);
2044 static void *parallel_map_access(void *arg)
2046 int err, map_fd = *(u32 *) arg;
2047 int vars[17], i, j, rnd, key = 0;
2049 for (i = 0; i < 10000; i++) {
2050 err = bpf_map_lookup_elem_flags(map_fd, &key, vars, BPF_F_LOCK);
2052 printf("lookup failed\n");
2057 printf("lookup #%d var[0]=%d\n", i, vars[0]);
2062 for (j = 2; j < 17; j++) {
2065 printf("lookup #%d var[1]=%d var[%d]=%d\n",
2066 i, rnd, j, vars[j]);
2075 static void test_map_lock(void)
2077 const char *file = "./test_map_lock.o";
2078 int prog_fd, map_fd[2], vars[17] = {};
2079 pthread_t thread_id[6];
2080 struct bpf_object *obj;
2081 int err = 0, key = 0, i;
2084 err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd);
2086 printf("test_map_lock:bpf_prog_load errno %d\n", errno);
2089 map_fd[0] = bpf_find_map(__func__, obj, "hash_map");
2092 map_fd[1] = bpf_find_map(__func__, obj, "array_map");
2096 bpf_map_update_elem(map_fd[0], &key, vars, BPF_F_LOCK);
2098 for (i = 0; i < 4; i++)
2099 assert(pthread_create(&thread_id[i], NULL,
2100 &test_spin_lock, &prog_fd) == 0);
2101 for (i = 4; i < 6; i++)
2102 assert(pthread_create(&thread_id[i], NULL,
2103 ¶llel_map_access, &map_fd[i - 4]) == 0);
2104 for (i = 0; i < 4; i++)
2105 assert(pthread_join(thread_id[i], &ret) == 0 &&
2106 ret == (void *)&prog_fd);
2107 for (i = 4; i < 6; i++)
2108 assert(pthread_join(thread_id[i], &ret) == 0 &&
2109 ret == (void *)&map_fd[i - 4]);
2110 goto close_prog_noerr;
2114 bpf_object__close(obj);
2121 jit_enabled = is_jit_enabled();
2124 test_prog_run_xattr();
2126 test_xdp_adjust_tail();
2128 test_xdp_noinline();
2131 test_pkt_md_access();
2133 test_tp_attach_query();
2134 test_stacktrace_map();
2135 test_stacktrace_build_id();
2136 test_stacktrace_build_id_nmi();
2137 test_stacktrace_map_raw_tp();
2138 test_get_stack_raw_tp();
2139 test_task_fd_query_rawtp();
2140 test_task_fd_query_tp();
2141 test_reference_tracking();
2142 test_queue_stack_map(QUEUE);
2143 test_queue_stack_map(STACK);
2144 test_flow_dissector();
2148 printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt);
2149 return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS;