tools/testing/selftests/bpf/test_progs.c

   1 /* Copyright (c) 2017 Facebook
   2  *
   3  * This program is free software; you can redistribute it and/or
   4  * modify it under the terms of version 2 of the GNU General Public
   5  * License as published by the Free Software Foundation.
   6  */
   7 #include <stdio.h>
   8 #include <unistd.h>
   9 #include <errno.h>
  10 #include <string.h>
  11 #include <assert.h>
  12 #include <stdlib.h>
  13 #include <stdarg.h>
  14 #include <time.h>
  15 #include <signal.h>
  16
  17 #include <linux/types.h>
  18 typedef __u16 __sum16;
  19 #include <arpa/inet.h>
  20 #include <linux/if_ether.h>
  21 #include <linux/if_packet.h>
  22 #include <linux/ip.h>
  23 #include <linux/ipv6.h>
  24 #include <linux/tcp.h>
  25 #include <linux/filter.h>
  26 #include <linux/perf_event.h>
  27 #include <linux/unistd.h>
  28
  29 #include <sys/ioctl.h>
  30 #include <sys/wait.h>
  31 #include <sys/types.h>
  32 #include <sys/time.h>
  33 #include <fcntl.h>
  34 #include <pthread.h>
  35 #include <linux/bpf.h>
  36 #include <linux/err.h>
  37 #include <bpf/bpf.h>
  38 #include <bpf/libbpf.h>
  39
  40 #include "test_iptunnel_common.h"
  41 #include "bpf_util.h"
  42 #include "bpf_endian.h"
  43 #include "bpf_rlimit.h"
  44 #include "trace_helpers.h"
  45 #include "flow_dissector_load.h"
  46
  47 static int error_cnt, pass_cnt;
  48 static bool jit_enabled;
  49
  50 #define MAGIC_BYTES 123
  51
  52 /* ipv4 test vector */
  53 static struct {
  54         struct ethhdr eth;
  55         struct iphdr iph;
  56         struct tcphdr tcp;
  57 } __packed pkt_v4 = {
  58         .eth.h_proto = __bpf_constant_htons(ETH_P_IP),
  59         .iph.ihl = 5,
  60         .iph.protocol = IPPROTO_TCP,
  61         .iph.tot_len = __bpf_constant_htons(MAGIC_BYTES),
  62         .tcp.urg_ptr = 123,
  63         .tcp.doff = 5,
  64 };
  65
  66 /* ipv6 test vector */
  67 static struct {
  68         struct ethhdr eth;
  69         struct ipv6hdr iph;
  70         struct tcphdr tcp;
  71 } __packed pkt_v6 = {
  72         .eth.h_proto = __bpf_constant_htons(ETH_P_IPV6),
  73         .iph.nexthdr = IPPROTO_TCP,
  74         .iph.payload_len = __bpf_constant_htons(MAGIC_BYTES),
  75         .tcp.urg_ptr = 123,
  76         .tcp.doff = 5,
  77 };
  78
  79 #define _CHECK(condition, tag, duration, format...) ({                  \
  80         int __ret = !!(condition);                                      \
  81         if (__ret) {                                                    \
  82                 error_cnt++;                                            \
  83                 printf("%s:FAIL:%s ", __func__, tag);                   \
  84                 printf(format);                                         \
  85         } else {                                                        \
  86                 pass_cnt++;                                             \
  87                 printf("%s:PASS:%s %d nsec\n", __func__, tag, duration);\
  88         }                                                               \
  89         __ret;                                                          \
  90 })
  91
  92 #define CHECK(condition, tag, format...) \
  93         _CHECK(condition, tag, duration, format)
  94 #define CHECK_ATTR(condition, tag, format...) \
  95         _CHECK(condition, tag, tattr.duration, format)
  96
  97 static int bpf_find_map(const char *test, struct bpf_object *obj,
  98                         const char *name)
  99 {
 100         struct bpf_map *map;
 101
 102         map = bpf_object__find_map_by_name(obj, name);
 103         if (!map) {
 104                 printf("%s:FAIL:map '%s' not found\n", test, name);
 105                 error_cnt++;
 106                 return -1;
 107         }
 108         return bpf_map__fd(map);
 109 }
 110
 111 static void test_pkt_access(void)
 112 {
 113         const char *file = "./test_pkt_access.o";
 114         struct bpf_object *obj;
 115         __u32 duration, retval;
 116         int err, prog_fd;
 117
 118         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 119         if (err) {
 120                 error_cnt++;
 121                 return;
 122         }
 123
 124         err = bpf_prog_test_run(prog_fd, 100000, &pkt_v4, sizeof(pkt_v4),
 125                                 NULL, NULL, &retval, &duration);
 126         CHECK(err || retval, "ipv4",
 127               "err %d errno %d retval %d duration %d\n",
 128               err, errno, retval, duration);
 129
 130         err = bpf_prog_test_run(prog_fd, 100000, &pkt_v6, sizeof(pkt_v6),
 131                                 NULL, NULL, &retval, &duration);
 132         CHECK(err || retval, "ipv6",
 133               "err %d errno %d retval %d duration %d\n",
 134               err, errno, retval, duration);
 135         bpf_object__close(obj);
 136 }
 137
 138 static void test_prog_run_xattr(void)
 139 {
 140         const char *file = "./test_pkt_access.o";
 141         struct bpf_object *obj;
 142         char buf[10];
 143         int err;
 144         struct bpf_prog_test_run_attr tattr = {
 145                 .repeat = 1,
 146                 .data_in = &pkt_v4,
 147                 .data_size_in = sizeof(pkt_v4),
 148                 .data_out = buf,
 149                 .data_size_out = 5,
 150         };
 151
 152         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj,
 153                             &tattr.prog_fd);
 154         if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno))
 155                 return;
 156
 157         memset(buf, 0, sizeof(buf));
 158
 159         err = bpf_prog_test_run_xattr(&tattr);
 160         CHECK_ATTR(err != -1 || errno != ENOSPC || tattr.retval, "run",
 161               "err %d errno %d retval %d\n", err, errno, tattr.retval);
 162
 163         CHECK_ATTR(tattr.data_size_out != sizeof(pkt_v4), "data_size_out",
 164               "incorrect output size, want %lu have %u\n",
 165               sizeof(pkt_v4), tattr.data_size_out);
 166
 167         CHECK_ATTR(buf[5] != 0, "overflow",
 168               "BPF_PROG_TEST_RUN ignored size hint\n");
 169
 170         tattr.data_out = NULL;
 171         tattr.data_size_out = 0;
 172         errno = 0;
 173
 174         err = bpf_prog_test_run_xattr(&tattr);
 175         CHECK_ATTR(err || errno || tattr.retval, "run_no_output",
 176               "err %d errno %d retval %d\n", err, errno, tattr.retval);
 177
 178         tattr.data_size_out = 1;
 179         err = bpf_prog_test_run_xattr(&tattr);
 180         CHECK_ATTR(err != -EINVAL, "run_wrong_size_out", "err %d\n", err);
 181
 182         bpf_object__close(obj);
 183 }
 184
 185 static void test_xdp(void)
 186 {
 187         struct vip key4 = {.protocol = 6, .family = AF_INET};
 188         struct vip key6 = {.protocol = 6, .family = AF_INET6};
 189         struct iptnl_info value4 = {.family = AF_INET};
 190         struct iptnl_info value6 = {.family = AF_INET6};
 191         const char *file = "./test_xdp.o";
 192         struct bpf_object *obj;
 193         char buf[128];
 194         struct ipv6hdr *iph6 = (void *)buf + sizeof(struct ethhdr);
 195         struct iphdr *iph = (void *)buf + sizeof(struct ethhdr);
 196         __u32 duration, retval, size;
 197         int err, prog_fd, map_fd;
 198
 199         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 200         if (err) {
 201                 error_cnt++;
 202                 return;
 203         }
 204
 205         map_fd = bpf_find_map(__func__, obj, "vip2tnl");
 206         if (map_fd < 0)
 207                 goto out;
 208         bpf_map_update_elem(map_fd, &key4, &value4, 0);
 209         bpf_map_update_elem(map_fd, &key6, &value6, 0);
 210
 211         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
 212                                 buf, &size, &retval, &duration);
 213
 214         CHECK(err || retval != XDP_TX || size != 74 ||
 215               iph->protocol != IPPROTO_IPIP, "ipv4",
 216               "err %d errno %d retval %d size %d\n",
 217               err, errno, retval, size);
 218
 219         err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6),
 220                                 buf, &size, &retval, &duration);
 221         CHECK(err || retval != XDP_TX || size != 114 ||
 222               iph6->nexthdr != IPPROTO_IPV6, "ipv6",
 223               "err %d errno %d retval %d size %d\n",
 224               err, errno, retval, size);
 225 out:
 226         bpf_object__close(obj);
 227 }
 228
 229 static void test_xdp_adjust_tail(void)
 230 {
 231         const char *file = "./test_adjust_tail.o";
 232         struct bpf_object *obj;
 233         char buf[128];
 234         __u32 duration, retval, size;
 235         int err, prog_fd;
 236
 237         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 238         if (err) {
 239                 error_cnt++;
 240                 return;
 241         }
 242
 243         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
 244                                 buf, &size, &retval, &duration);
 245
 246         CHECK(err || retval != XDP_DROP,
 247               "ipv4", "err %d errno %d retval %d size %d\n",
 248               err, errno, retval, size);
 249
 250         err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6),
 251                                 buf, &size, &retval, &duration);
 252         CHECK(err || retval != XDP_TX || size != 54,
 253               "ipv6", "err %d errno %d retval %d size %d\n",
 254               err, errno, retval, size);
 255         bpf_object__close(obj);
 256 }
 257
 258
 259
 260 #define MAGIC_VAL 0x1234
 261 #define NUM_ITER 100000
 262 #define VIP_NUM 5
 263
 264 static void test_l4lb(const char *file)
 265 {
 266         unsigned int nr_cpus = bpf_num_possible_cpus();
 267         struct vip key = {.protocol = 6};
 268         struct vip_meta {
 269                 __u32 flags;
 270                 __u32 vip_num;
 271         } value = {.vip_num = VIP_NUM};
 272         __u32 stats_key = VIP_NUM;
 273         struct vip_stats {
 274                 __u64 bytes;
 275                 __u64 pkts;
 276         } stats[nr_cpus];
 277         struct real_definition {
 278                 union {
 279                         __be32 dst;
 280                         __be32 dstv6[4];
 281                 };
 282                 __u8 flags;
 283         } real_def = {.dst = MAGIC_VAL};
 284         __u32 ch_key = 11, real_num = 3;
 285         __u32 duration, retval, size;
 286         int err, i, prog_fd, map_fd;
 287         __u64 bytes = 0, pkts = 0;
 288         struct bpf_object *obj;
 289         char buf[128];
 290         u32 *magic = (u32 *)buf;
 291
 292         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 293         if (err) {
 294                 error_cnt++;
 295                 return;
 296         }
 297
 298         map_fd = bpf_find_map(__func__, obj, "vip_map");
 299         if (map_fd < 0)
 300                 goto out;
 301         bpf_map_update_elem(map_fd, &key, &value, 0);
 302
 303         map_fd = bpf_find_map(__func__, obj, "ch_rings");
 304         if (map_fd < 0)
 305                 goto out;
 306         bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
 307
 308         map_fd = bpf_find_map(__func__, obj, "reals");
 309         if (map_fd < 0)
 310                 goto out;
 311         bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
 312
 313         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
 314                                 buf, &size, &retval, &duration);
 315         CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 54 ||
 316               *magic != MAGIC_VAL, "ipv4",
 317               "err %d errno %d retval %d size %d magic %x\n",
 318               err, errno, retval, size, *magic);
 319
 320         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
 321                                 buf, &size, &retval, &duration);
 322         CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 74 ||
 323               *magic != MAGIC_VAL, "ipv6",
 324               "err %d errno %d retval %d size %d magic %x\n",
 325               err, errno, retval, size, *magic);
 326
 327         map_fd = bpf_find_map(__func__, obj, "stats");
 328         if (map_fd < 0)
 329                 goto out;
 330         bpf_map_lookup_elem(map_fd, &stats_key, stats);
 331         for (i = 0; i < nr_cpus; i++) {
 332                 bytes += stats[i].bytes;
 333                 pkts += stats[i].pkts;
 334         }
 335         if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
 336                 error_cnt++;
 337                 printf("test_l4lb:FAIL:stats %lld %lld\n", bytes, pkts);
 338         }
 339 out:
 340         bpf_object__close(obj);
 341 }
 342
 343 static void test_l4lb_all(void)
 344 {
 345         const char *file1 = "./test_l4lb.o";
 346         const char *file2 = "./test_l4lb_noinline.o";
 347
 348         test_l4lb(file1);
 349         test_l4lb(file2);
 350 }
 351
 352 static void test_xdp_noinline(void)
 353 {
 354         const char *file = "./test_xdp_noinline.o";
 355         unsigned int nr_cpus = bpf_num_possible_cpus();
 356         struct vip key = {.protocol = 6};
 357         struct vip_meta {
 358                 __u32 flags;
 359                 __u32 vip_num;
 360         } value = {.vip_num = VIP_NUM};
 361         __u32 stats_key = VIP_NUM;
 362         struct vip_stats {
 363                 __u64 bytes;
 364                 __u64 pkts;
 365         } stats[nr_cpus];
 366         struct real_definition {
 367                 union {
 368                         __be32 dst;
 369                         __be32 dstv6[4];
 370                 };
 371                 __u8 flags;
 372         } real_def = {.dst = MAGIC_VAL};
 373         __u32 ch_key = 11, real_num = 3;
 374         __u32 duration, retval, size;
 375         int err, i, prog_fd, map_fd;
 376         __u64 bytes = 0, pkts = 0;
 377         struct bpf_object *obj;
 378         char buf[128];
 379         u32 *magic = (u32 *)buf;
 380
 381         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 382         if (err) {
 383                 error_cnt++;
 384                 return;
 385         }
 386
 387         map_fd = bpf_find_map(__func__, obj, "vip_map");
 388         if (map_fd < 0)
 389                 goto out;
 390         bpf_map_update_elem(map_fd, &key, &value, 0);
 391
 392         map_fd = bpf_find_map(__func__, obj, "ch_rings");
 393         if (map_fd < 0)
 394                 goto out;
 395         bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
 396
 397         map_fd = bpf_find_map(__func__, obj, "reals");
 398         if (map_fd < 0)
 399                 goto out;
 400         bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
 401
 402         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
 403                                 buf, &size, &retval, &duration);
 404         CHECK(err || retval != 1 || size != 54 ||
 405               *magic != MAGIC_VAL, "ipv4",
 406               "err %d errno %d retval %d size %d magic %x\n",
 407               err, errno, retval, size, *magic);
 408
 409         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
 410                                 buf, &size, &retval, &duration);
 411         CHECK(err || retval != 1 || size != 74 ||
 412               *magic != MAGIC_VAL, "ipv6",
 413               "err %d errno %d retval %d size %d magic %x\n",
 414               err, errno, retval, size, *magic);
 415
 416         map_fd = bpf_find_map(__func__, obj, "stats");
 417         if (map_fd < 0)
 418                 goto out;
 419         bpf_map_lookup_elem(map_fd, &stats_key, stats);
 420         for (i = 0; i < nr_cpus; i++) {
 421                 bytes += stats[i].bytes;
 422                 pkts += stats[i].pkts;
 423         }
 424         if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
 425                 error_cnt++;
 426                 printf("test_xdp_noinline:FAIL:stats %lld %lld\n", bytes, pkts);
 427         }
 428 out:
 429         bpf_object__close(obj);
 430 }
 431
 432 static void test_tcp_estats(void)
 433 {
 434         const char *file = "./test_tcp_estats.o";
 435         int err, prog_fd;
 436         struct bpf_object *obj;
 437         __u32 duration = 0;
 438
 439         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
 440         CHECK(err, "", "err %d errno %d\n", err, errno);
 441         if (err) {
 442                 error_cnt++;
 443                 return;
 444         }
 445
 446         bpf_object__close(obj);
 447 }
 448
 449 static inline __u64 ptr_to_u64(const void *ptr)
 450 {
 451         return (__u64) (unsigned long) ptr;
 452 }
 453
 454 static bool is_jit_enabled(void)
 455 {
 456         const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
 457         bool enabled = false;
 458         int sysctl_fd;
 459
 460         sysctl_fd = open(jit_sysctl, 0, O_RDONLY);
 461         if (sysctl_fd != -1) {
 462                 char tmpc;
 463
 464                 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
 465                         enabled = (tmpc != '0');
 466                 close(sysctl_fd);
 467         }
 468
 469         return enabled;
 470 }
 471
 472 static void test_bpf_obj_id(void)
 473 {
 474         const __u64 array_magic_value = 0xfaceb00c;
 475         const __u32 array_key = 0;
 476         const int nr_iters = 2;
 477         const char *file = "./test_obj_id.o";
 478         const char *expected_prog_name = "test_obj_id";
 479         const char *expected_map_name = "test_map_id";
 480         const __u64 nsec_per_sec = 1000000000;
 481
 482         struct bpf_object *objs[nr_iters];
 483         int prog_fds[nr_iters], map_fds[nr_iters];
 484         /* +1 to test for the info_len returned by kernel */
 485         struct bpf_prog_info prog_infos[nr_iters + 1];
 486         struct bpf_map_info map_infos[nr_iters + 1];
 487         /* Each prog only uses one map. +1 to test nr_map_ids
 488          * returned by kernel.
 489          */
 490         __u32 map_ids[nr_iters + 1];
 491         char jited_insns[128], xlated_insns[128], zeros[128];
 492         __u32 i, next_id, info_len, nr_id_found, duration = 0;
 493         struct timespec real_time_ts, boot_time_ts;
 494         int err = 0;
 495         __u64 array_value;
 496         uid_t my_uid = getuid();
 497         time_t now, load_time;
 498
 499         err = bpf_prog_get_fd_by_id(0);
 500         CHECK(err >= 0 || errno != ENOENT,
 501               "get-fd-by-notexist-prog-id", "err %d errno %d\n", err, errno);
 502
 503         err = bpf_map_get_fd_by_id(0);
 504         CHECK(err >= 0 || errno != ENOENT,
 505               "get-fd-by-notexist-map-id", "err %d errno %d\n", err, errno);
 506
 507         for (i = 0; i < nr_iters; i++)
 508                 objs[i] = NULL;
 509
 510         /* Check bpf_obj_get_info_by_fd() */
 511         bzero(zeros, sizeof(zeros));
 512         for (i = 0; i < nr_iters; i++) {
 513                 now = time(NULL);
 514                 err = bpf_prog_load(file, BPF_PROG_TYPE_SOCKET_FILTER,
 515                                     &objs[i], &prog_fds[i]);
 516                 /* test_obj_id.o is a dumb prog. It should never fail
 517                  * to load.
 518                  */
 519                 if (err)
 520                         error_cnt++;
 521                 assert(!err);
 522
 523                 /* Insert a magic value to the map */
 524                 map_fds[i] = bpf_find_map(__func__, objs[i], "test_map_id");
 525                 assert(map_fds[i] >= 0);
 526                 err = bpf_map_update_elem(map_fds[i], &array_key,
 527                                           &array_magic_value, 0);
 528                 assert(!err);
 529
 530                 /* Check getting map info */
 531                 info_len = sizeof(struct bpf_map_info) * 2;
 532                 bzero(&map_infos[i], info_len);
 533                 err = bpf_obj_get_info_by_fd(map_fds[i], &map_infos[i],
 534                                              &info_len);
 535                 if (CHECK(err ||
 536                           map_infos[i].type != BPF_MAP_TYPE_ARRAY ||
 537                           map_infos[i].key_size != sizeof(__u32) ||
 538                           map_infos[i].value_size != sizeof(__u64) ||
 539                           map_infos[i].max_entries != 1 ||
 540                           map_infos[i].map_flags != 0 ||
 541                           info_len != sizeof(struct bpf_map_info) ||
 542                           strcmp((char *)map_infos[i].name, expected_map_name),
 543                           "get-map-info(fd)",
 544                           "err %d errno %d type %d(%d) info_len %u(%Zu) key_size %u value_size %u max_entries %u map_flags %X name %s(%s)\n",
 545                           err, errno,
 546                           map_infos[i].type, BPF_MAP_TYPE_ARRAY,
 547                           info_len, sizeof(struct bpf_map_info),
 548                           map_infos[i].key_size,
 549                           map_infos[i].value_size,
 550                           map_infos[i].max_entries,
 551                           map_infos[i].map_flags,
 552                           map_infos[i].name, expected_map_name))
 553                         goto done;
 554
 555                 /* Check getting prog info */
 556                 info_len = sizeof(struct bpf_prog_info) * 2;
 557                 bzero(&prog_infos[i], info_len);
 558                 bzero(jited_insns, sizeof(jited_insns));
 559                 bzero(xlated_insns, sizeof(xlated_insns));
 560                 prog_infos[i].jited_prog_insns = ptr_to_u64(jited_insns);
 561                 prog_infos[i].jited_prog_len = sizeof(jited_insns);
 562                 prog_infos[i].xlated_prog_insns = ptr_to_u64(xlated_insns);
 563                 prog_infos[i].xlated_prog_len = sizeof(xlated_insns);
 564                 prog_infos[i].map_ids = ptr_to_u64(map_ids + i);
 565                 prog_infos[i].nr_map_ids = 2;
 566                 err = clock_gettime(CLOCK_REALTIME, &real_time_ts);
 567                 assert(!err);
 568                 err = clock_gettime(CLOCK_BOOTTIME, &boot_time_ts);
 569                 assert(!err);
 570                 err = bpf_obj_get_info_by_fd(prog_fds[i], &prog_infos[i],
 571                                              &info_len);
 572                 load_time = (real_time_ts.tv_sec - boot_time_ts.tv_sec)
 573                         + (prog_infos[i].load_time / nsec_per_sec);
 574                 if (CHECK(err ||
 575                           prog_infos[i].type != BPF_PROG_TYPE_SOCKET_FILTER ||
 576                           info_len != sizeof(struct bpf_prog_info) ||
 577                           (jit_enabled && !prog_infos[i].jited_prog_len) ||
 578                           (jit_enabled &&
 579                            !memcmp(jited_insns, zeros, sizeof(zeros))) ||
 580                           !prog_infos[i].xlated_prog_len ||
 581                           !memcmp(xlated_insns, zeros, sizeof(zeros)) ||
 582                           load_time < now - 60 || load_time > now + 60 ||
 583                           prog_infos[i].created_by_uid != my_uid ||
 584                           prog_infos[i].nr_map_ids != 1 ||
 585                           *(int *)(long)prog_infos[i].map_ids != map_infos[i].id ||
 586                           strcmp((char *)prog_infos[i].name, expected_prog_name),
 587                           "get-prog-info(fd)",
 588                           "err %d errno %d i %d type %d(%d) info_len %u(%Zu) jit_enabled %d jited_prog_len %u xlated_prog_len %u jited_prog %d xlated_prog %d load_time %lu(%lu) uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) name %s(%s)\n",
 589                           err, errno, i,
 590                           prog_infos[i].type, BPF_PROG_TYPE_SOCKET_FILTER,
 591                           info_len, sizeof(struct bpf_prog_info),
 592                           jit_enabled,
 593                           prog_infos[i].jited_prog_len,
 594                           prog_infos[i].xlated_prog_len,
 595                           !!memcmp(jited_insns, zeros, sizeof(zeros)),
 596                           !!memcmp(xlated_insns, zeros, sizeof(zeros)),
 597                           load_time, now,
 598                           prog_infos[i].created_by_uid, my_uid,
 599                           prog_infos[i].nr_map_ids, 1,
 600                           *(int *)(long)prog_infos[i].map_ids, map_infos[i].id,
 601                           prog_infos[i].name, expected_prog_name))
 602                         goto done;
 603         }
 604
 605         /* Check bpf_prog_get_next_id() */
 606         nr_id_found = 0;
 607         next_id = 0;
 608         while (!bpf_prog_get_next_id(next_id, &next_id)) {
 609                 struct bpf_prog_info prog_info = {};
 610                 __u32 saved_map_id;
 611                 int prog_fd;
 612
 613                 info_len = sizeof(prog_info);
 614
 615                 prog_fd = bpf_prog_get_fd_by_id(next_id);
 616                 if (prog_fd < 0 && errno == ENOENT)
 617                         /* The bpf_prog is in the dead row */
 618                         continue;
 619                 if (CHECK(prog_fd < 0, "get-prog-fd(next_id)",
 620                           "prog_fd %d next_id %d errno %d\n",
 621                           prog_fd, next_id, errno))
 622                         break;
 623
 624                 for (i = 0; i < nr_iters; i++)
 625                         if (prog_infos[i].id == next_id)
 626                                 break;
 627
 628                 if (i == nr_iters)
 629                         continue;
 630
 631                 nr_id_found++;
 632
 633                 /* Negative test:
 634                  * prog_info.nr_map_ids = 1
 635                  * prog_info.map_ids = NULL
 636                  */
 637                 prog_info.nr_map_ids = 1;
 638                 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
 639                 if (CHECK(!err || errno != EFAULT,
 640                           "get-prog-fd-bad-nr-map-ids", "err %d errno %d(%d)",
 641                           err, errno, EFAULT))
 642                         break;
 643                 bzero(&prog_info, sizeof(prog_info));
 644                 info_len = sizeof(prog_info);
 645
 646                 saved_map_id = *(int *)((long)prog_infos[i].map_ids);
 647                 prog_info.map_ids = prog_infos[i].map_ids;
 648                 prog_info.nr_map_ids = 2;
 649                 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
 650                 prog_infos[i].jited_prog_insns = 0;
 651                 prog_infos[i].xlated_prog_insns = 0;
 652                 CHECK(err || info_len != sizeof(struct bpf_prog_info) ||
 653                       memcmp(&prog_info, &prog_infos[i], info_len) ||
 654                       *(int *)(long)prog_info.map_ids != saved_map_id,
 655                       "get-prog-info(next_id->fd)",
 656                       "err %d errno %d info_len %u(%Zu) memcmp %d map_id %u(%u)\n",
 657                       err, errno, info_len, sizeof(struct bpf_prog_info),
 658                       memcmp(&prog_info, &prog_infos[i], info_len),
 659                       *(int *)(long)prog_info.map_ids, saved_map_id);
 660                 close(prog_fd);
 661         }
 662         CHECK(nr_id_found != nr_iters,
 663               "check total prog id found by get_next_id",
 664               "nr_id_found %u(%u)\n",
 665               nr_id_found, nr_iters);
 666
 667         /* Check bpf_map_get_next_id() */
 668         nr_id_found = 0;
 669         next_id = 0;
 670         while (!bpf_map_get_next_id(next_id, &next_id)) {
 671                 struct bpf_map_info map_info = {};
 672                 int map_fd;
 673
 674                 info_len = sizeof(map_info);
 675
 676                 map_fd = bpf_map_get_fd_by_id(next_id);
 677                 if (map_fd < 0 && errno == ENOENT)
 678                         /* The bpf_map is in the dead row */
 679                         continue;
 680                 if (CHECK(map_fd < 0, "get-map-fd(next_id)",
 681                           "map_fd %d next_id %u errno %d\n",
 682                           map_fd, next_id, errno))
 683                         break;
 684
 685                 for (i = 0; i < nr_iters; i++)
 686                         if (map_infos[i].id == next_id)
 687                                 break;
 688
 689                 if (i == nr_iters)
 690                         continue;
 691
 692                 nr_id_found++;
 693
 694                 err = bpf_map_lookup_elem(map_fd, &array_key, &array_value);
 695                 assert(!err);
 696
 697                 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &info_len);
 698                 CHECK(err || info_len != sizeof(struct bpf_map_info) ||
 699                       memcmp(&map_info, &map_infos[i], info_len) ||
 700                       array_value != array_magic_value,
 701                       "check get-map-info(next_id->fd)",
 702                       "err %d errno %d info_len %u(%Zu) memcmp %d array_value %llu(%llu)\n",
 703                       err, errno, info_len, sizeof(struct bpf_map_info),
 704                       memcmp(&map_info, &map_infos[i], info_len),
 705                       array_value, array_magic_value);
 706
 707                 close(map_fd);
 708         }
 709         CHECK(nr_id_found != nr_iters,
 710               "check total map id found by get_next_id",
 711               "nr_id_found %u(%u)\n",
 712               nr_id_found, nr_iters);
 713
 714 done:
 715         for (i = 0; i < nr_iters; i++)
 716                 bpf_object__close(objs[i]);
 717 }
 718
 719 static void test_pkt_md_access(void)
 720 {
 721         const char *file = "./test_pkt_md_access.o";
 722         struct bpf_object *obj;
 723         __u32 duration, retval;
 724         int err, prog_fd;
 725
 726         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 727         if (err) {
 728                 error_cnt++;
 729                 return;
 730         }
 731
 732         err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4),
 733                                 NULL, NULL, &retval, &duration);
 734         CHECK(err || retval, "",
 735               "err %d errno %d retval %d duration %d\n",
 736               err, errno, retval, duration);
 737
 738         bpf_object__close(obj);
 739 }
 740
 741 static void test_obj_name(void)
 742 {
 743         struct {
 744                 const char *name;
 745                 int success;
 746                 int expected_errno;
 747         } tests[] = {
 748                 { "", 1, 0 },
 749                 { "_123456789ABCDE", 1, 0 },
 750                 { "_123456789ABCDEF", 0, EINVAL },
 751                 { "_123456789ABCD\n", 0, EINVAL },
 752         };
 753         struct bpf_insn prog[] = {
 754                 BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0),
 755                 BPF_EXIT_INSN(),
 756         };
 757         __u32 duration = 0;
 758         int i;
 759
 760         for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) {
 761                 size_t name_len = strlen(tests[i].name) + 1;
 762                 union bpf_attr attr;
 763                 size_t ncopy;
 764                 int fd;
 765
 766                 /* test different attr.prog_name during BPF_PROG_LOAD */
 767                 ncopy = name_len < sizeof(attr.prog_name) ?
 768                         name_len : sizeof(attr.prog_name);
 769                 bzero(&attr, sizeof(attr));
 770                 attr.prog_type = BPF_PROG_TYPE_SCHED_CLS;
 771                 attr.insn_cnt = 2;
 772                 attr.insns = ptr_to_u64(prog);
 773                 attr.license = ptr_to_u64("");
 774                 memcpy(attr.prog_name, tests[i].name, ncopy);
 775
 776                 fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr));
 777                 CHECK((tests[i].success && fd < 0) ||
 778                       (!tests[i].success && fd != -1) ||
 779                       (!tests[i].success && errno != tests[i].expected_errno),
 780                       "check-bpf-prog-name",
 781                       "fd %d(%d) errno %d(%d)\n",
 782                        fd, tests[i].success, errno, tests[i].expected_errno);
 783
 784                 if (fd != -1)
 785                         close(fd);
 786
 787                 /* test different attr.map_name during BPF_MAP_CREATE */
 788                 ncopy = name_len < sizeof(attr.map_name) ?
 789                         name_len : sizeof(attr.map_name);
 790                 bzero(&attr, sizeof(attr));
 791                 attr.map_type = BPF_MAP_TYPE_ARRAY;
 792                 attr.key_size = 4;
 793                 attr.value_size = 4;
 794                 attr.max_entries = 1;
 795                 attr.map_flags = 0;
 796                 memcpy(attr.map_name, tests[i].name, ncopy);
 797                 fd = syscall(__NR_bpf, BPF_MAP_CREATE, &attr, sizeof(attr));
 798                 CHECK((tests[i].success && fd < 0) ||
 799                       (!tests[i].success && fd != -1) ||
 800                       (!tests[i].success && errno != tests[i].expected_errno),
 801                       "check-bpf-map-name",
 802                       "fd %d(%d) errno %d(%d)\n",
 803                       fd, tests[i].success, errno, tests[i].expected_errno);
 804
 805                 if (fd != -1)
 806                         close(fd);
 807         }
 808 }
 809
 810 static void test_tp_attach_query(void)
 811 {
 812         const int num_progs = 3;
 813         int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs];
 814         __u32 duration = 0, info_len, saved_prog_ids[num_progs];
 815         const char *file = "./test_tracepoint.o";
 816         struct perf_event_query_bpf *query;
 817         struct perf_event_attr attr = {};
 818         struct bpf_object *obj[num_progs];
 819         struct bpf_prog_info prog_info;
 820         char buf[256];
 821
 822         snprintf(buf, sizeof(buf),
 823                  "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
 824         efd = open(buf, O_RDONLY, 0);
 825         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
 826                 return;
 827         bytes = read(efd, buf, sizeof(buf));
 828         close(efd);
 829         if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
 830                   "read", "bytes %d errno %d\n", bytes, errno))
 831                 return;
 832
 833         attr.config = strtol(buf, NULL, 0);
 834         attr.type = PERF_TYPE_TRACEPOINT;
 835         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
 836         attr.sample_period = 1;
 837         attr.wakeup_events = 1;
 838
 839         query = malloc(sizeof(*query) + sizeof(__u32) * num_progs);
 840         for (i = 0; i < num_progs; i++) {
 841                 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj[i],
 842                                     &prog_fd[i]);
 843                 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
 844                         goto cleanup1;
 845
 846                 bzero(&prog_info, sizeof(prog_info));
 847                 prog_info.jited_prog_len = 0;
 848                 prog_info.xlated_prog_len = 0;
 849                 prog_info.nr_map_ids = 0;
 850                 info_len = sizeof(prog_info);
 851                 err = bpf_obj_get_info_by_fd(prog_fd[i], &prog_info, &info_len);
 852                 if (CHECK(err, "bpf_obj_get_info_by_fd", "err %d errno %d\n",
 853                           err, errno))
 854                         goto cleanup1;
 855                 saved_prog_ids[i] = prog_info.id;
 856
 857                 pmu_fd[i] = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
 858                                     0 /* cpu 0 */, -1 /* group id */,
 859                                     0 /* flags */);
 860                 if (CHECK(pmu_fd[i] < 0, "perf_event_open", "err %d errno %d\n",
 861                           pmu_fd[i], errno))
 862                         goto cleanup2;
 863                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_ENABLE, 0);
 864                 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
 865                           err, errno))
 866                         goto cleanup3;
 867
 868                 if (i == 0) {
 869                         /* check NULL prog array query */
 870                         query->ids_len = num_progs;
 871                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 872                         if (CHECK(err || query->prog_cnt != 0,
 873                                   "perf_event_ioc_query_bpf",
 874                                   "err %d errno %d query->prog_cnt %u\n",
 875                                   err, errno, query->prog_cnt))
 876                                 goto cleanup3;
 877                 }
 878
 879                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_SET_BPF, prog_fd[i]);
 880                 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
 881                           err, errno))
 882                         goto cleanup3;
 883
 884                 if (i == 1) {
 885                         /* try to get # of programs only */
 886                         query->ids_len = 0;
 887                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 888                         if (CHECK(err || query->prog_cnt != 2,
 889                                   "perf_event_ioc_query_bpf",
 890                                   "err %d errno %d query->prog_cnt %u\n",
 891                                   err, errno, query->prog_cnt))
 892                                 goto cleanup3;
 893
 894                         /* try a few negative tests */
 895                         /* invalid query pointer */
 896                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF,
 897                                     (struct perf_event_query_bpf *)0x1);
 898                         if (CHECK(!err || errno != EFAULT,
 899                                   "perf_event_ioc_query_bpf",
 900                                   "err %d errno %d\n", err, errno))
 901                                 goto cleanup3;
 902
 903                         /* no enough space */
 904                         query->ids_len = 1;
 905                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 906                         if (CHECK(!err || errno != ENOSPC || query->prog_cnt != 2,
 907                                   "perf_event_ioc_query_bpf",
 908                                   "err %d errno %d query->prog_cnt %u\n",
 909                                   err, errno, query->prog_cnt))
 910                                 goto cleanup3;
 911                 }
 912
 913                 query->ids_len = num_progs;
 914                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 915                 if (CHECK(err || query->prog_cnt != (i + 1),
 916                           "perf_event_ioc_query_bpf",
 917                           "err %d errno %d query->prog_cnt %u\n",
 918                           err, errno, query->prog_cnt))
 919                         goto cleanup3;
 920                 for (j = 0; j < i + 1; j++)
 921                         if (CHECK(saved_prog_ids[j] != query->ids[j],
 922                                   "perf_event_ioc_query_bpf",
 923                                   "#%d saved_prog_id %x query prog_id %x\n",
 924                                   j, saved_prog_ids[j], query->ids[j]))
 925                                 goto cleanup3;
 926         }
 927
 928         i = num_progs - 1;
 929         for (; i >= 0; i--) {
 930  cleanup3:
 931                 ioctl(pmu_fd[i], PERF_EVENT_IOC_DISABLE);
 932  cleanup2:
 933                 close(pmu_fd[i]);
 934  cleanup1:
 935                 bpf_object__close(obj[i]);
 936         }
 937         free(query);
 938 }
 939
 940 static int compare_map_keys(int map1_fd, int map2_fd)
 941 {
 942         __u32 key, next_key;
 943         char val_buf[PERF_MAX_STACK_DEPTH *
 944                      sizeof(struct bpf_stack_build_id)];
 945         int err;
 946
 947         err = bpf_map_get_next_key(map1_fd, NULL, &key);
 948         if (err)
 949                 return err;
 950         err = bpf_map_lookup_elem(map2_fd, &key, val_buf);
 951         if (err)
 952                 return err;
 953
 954         while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) {
 955                 err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf);
 956                 if (err)
 957                         return err;
 958
 959                 key = next_key;
 960         }
 961         if (errno != ENOENT)
 962                 return -1;
 963
 964         return 0;
 965 }
 966
 967 static int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len)
 968 {
 969         __u32 key, next_key, *cur_key_p, *next_key_p;
 970         char *val_buf1, *val_buf2;
 971         int i, err = 0;
 972
 973         val_buf1 = malloc(stack_trace_len);
 974         val_buf2 = malloc(stack_trace_len);
 975         cur_key_p = NULL;
 976         next_key_p = &key;
 977         while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) {
 978                 err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1);
 979                 if (err)
 980                         goto out;
 981                 err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2);
 982                 if (err)
 983                         goto out;
 984                 for (i = 0; i < stack_trace_len; i++) {
 985                         if (val_buf1[i] != val_buf2[i]) {
 986                                 err = -1;
 987                                 goto out;
 988                         }
 989                 }
 990                 key = *next_key_p;
 991                 cur_key_p = &key;
 992                 next_key_p = &next_key;
 993         }
 994         if (errno != ENOENT)
 995                 err = -1;
 996
 997 out:
 998         free(val_buf1);
 999         free(val_buf2);
1000         return err;
1001 }
1002
1003 static void test_stacktrace_map()
1004 {
1005         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1006         const char *file = "./test_stacktrace_map.o";
1007         int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1008         struct perf_event_attr attr = {};
1009         __u32 key, val, duration = 0;
1010         struct bpf_object *obj;
1011         char buf[256];
1012
1013         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1014         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1015                 return;
1016
1017         /* Get the ID for the sched/sched_switch tracepoint */
1018         snprintf(buf, sizeof(buf),
1019                  "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
1020         efd = open(buf, O_RDONLY, 0);
1021         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1022                 goto close_prog;
1023
1024         bytes = read(efd, buf, sizeof(buf));
1025         close(efd);
1026         if (bytes <= 0 || bytes >= sizeof(buf))
1027                 goto close_prog;
1028
1029         /* Open the perf event and attach bpf progrram */
1030         attr.config = strtol(buf, NULL, 0);
1031         attr.type = PERF_TYPE_TRACEPOINT;
1032         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
1033         attr.sample_period = 1;
1034         attr.wakeup_events = 1;
1035         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1036                          0 /* cpu 0 */, -1 /* group id */,
1037                          0 /* flags */);
1038         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
1039                   pmu_fd, errno))
1040                 goto close_prog;
1041
1042         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1043         if (err)
1044                 goto disable_pmu;
1045
1046         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1047         if (err)
1048                 goto disable_pmu;
1049
1050         /* find map fds */
1051         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1052         if (control_map_fd < 0)
1053                 goto disable_pmu;
1054
1055         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1056         if (stackid_hmap_fd < 0)
1057                 goto disable_pmu;
1058
1059         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1060         if (stackmap_fd < 0)
1061                 goto disable_pmu;
1062
1063         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1064         if (stack_amap_fd < 0)
1065                 goto disable_pmu;
1066
1067         /* give some time for bpf program run */
1068         sleep(1);
1069
1070         /* disable stack trace collection */
1071         key = 0;
1072         val = 1;
1073         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1074
1075         /* for every element in stackid_hmap, we can find a corresponding one
1076          * in stackmap, and vise versa.
1077          */
1078         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1079         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1080                   "err %d errno %d\n", err, errno))
1081                 goto disable_pmu_noerr;
1082
1083         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1084         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1085                   "err %d errno %d\n", err, errno))
1086                 goto disable_pmu_noerr;
1087
1088         stack_trace_len = PERF_MAX_STACK_DEPTH * sizeof(__u64);
1089         err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1090         if (CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1091                   "err %d errno %d\n", err, errno))
1092                 goto disable_pmu_noerr;
1093
1094         goto disable_pmu_noerr;
1095 disable_pmu:
1096         error_cnt++;
1097 disable_pmu_noerr:
1098         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1099         close(pmu_fd);
1100 close_prog:
1101         bpf_object__close(obj);
1102 }
1103
1104 static void test_stacktrace_map_raw_tp()
1105 {
1106         int control_map_fd, stackid_hmap_fd, stackmap_fd;
1107         const char *file = "./test_stacktrace_map.o";
1108         int efd, err, prog_fd;
1109         __u32 key, val, duration = 0;
1110         struct bpf_object *obj;
1111
1112         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1113         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1114                 return;
1115
1116         efd = bpf_raw_tracepoint_open("sched_switch", prog_fd);
1117         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1118                 goto close_prog;
1119
1120         /* find map fds */
1121         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1122         if (control_map_fd < 0)
1123                 goto close_prog;
1124
1125         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1126         if (stackid_hmap_fd < 0)
1127                 goto close_prog;
1128
1129         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1130         if (stackmap_fd < 0)
1131                 goto close_prog;
1132
1133         /* give some time for bpf program run */
1134         sleep(1);
1135
1136         /* disable stack trace collection */
1137         key = 0;
1138         val = 1;
1139         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1140
1141         /* for every element in stackid_hmap, we can find a corresponding one
1142          * in stackmap, and vise versa.
1143          */
1144         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1145         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1146                   "err %d errno %d\n", err, errno))
1147                 goto close_prog;
1148
1149         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1150         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1151                   "err %d errno %d\n", err, errno))
1152                 goto close_prog;
1153
1154         goto close_prog_noerr;
1155 close_prog:
1156         error_cnt++;
1157 close_prog_noerr:
1158         bpf_object__close(obj);
1159 }
1160
1161 static int extract_build_id(char *build_id, size_t size)
1162 {
1163         FILE *fp;
1164         char *line = NULL;
1165         size_t len = 0;
1166
1167         fp = popen("readelf -n ./urandom_read | grep 'Build ID'", "r");
1168         if (fp == NULL)
1169                 return -1;
1170
1171         if (getline(&line, &len, fp) == -1)
1172                 goto err;
1173         fclose(fp);
1174
1175         if (len > size)
1176                 len = size;
1177         memcpy(build_id, line, len);
1178         build_id[len] = '\0';
1179         return 0;
1180 err:
1181         fclose(fp);
1182         return -1;
1183 }
1184
1185 static void test_stacktrace_build_id(void)
1186 {
1187         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1188         const char *file = "./test_stacktrace_build_id.o";
1189         int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1190         struct perf_event_attr attr = {};
1191         __u32 key, previous_key, val, duration = 0;
1192         struct bpf_object *obj;
1193         char buf[256];
1194         int i, j;
1195         struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
1196         int build_id_matches = 0;
1197         int retry = 1;
1198
1199 retry:
1200         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1201         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1202                 goto out;
1203
1204         /* Get the ID for the sched/sched_switch tracepoint */
1205         snprintf(buf, sizeof(buf),
1206                  "/sys/kernel/debug/tracing/events/random/urandom_read/id");
1207         efd = open(buf, O_RDONLY, 0);
1208         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1209                 goto close_prog;
1210
1211         bytes = read(efd, buf, sizeof(buf));
1212         close(efd);
1213         if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
1214                   "read", "bytes %d errno %d\n", bytes, errno))
1215                 goto close_prog;
1216
1217         /* Open the perf event and attach bpf progrram */
1218         attr.config = strtol(buf, NULL, 0);
1219         attr.type = PERF_TYPE_TRACEPOINT;
1220         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
1221         attr.sample_period = 1;
1222         attr.wakeup_events = 1;
1223         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1224                          0 /* cpu 0 */, -1 /* group id */,
1225                          0 /* flags */);
1226         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
1227                   pmu_fd, errno))
1228                 goto close_prog;
1229
1230         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1231         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
1232                   err, errno))
1233                 goto close_pmu;
1234
1235         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1236         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
1237                   err, errno))
1238                 goto disable_pmu;
1239
1240         /* find map fds */
1241         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1242         if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
1243                   "err %d errno %d\n", err, errno))
1244                 goto disable_pmu;
1245
1246         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1247         if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
1248                   "err %d errno %d\n", err, errno))
1249                 goto disable_pmu;
1250
1251         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1252         if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
1253                   err, errno))
1254                 goto disable_pmu;
1255
1256         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1257         if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1258                   "err %d errno %d\n", err, errno))
1259                 goto disable_pmu;
1260
1261         assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1262                == 0);
1263         assert(system("./urandom_read") == 0);
1264         /* disable stack trace collection */
1265         key = 0;
1266         val = 1;
1267         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1268
1269         /* for every element in stackid_hmap, we can find a corresponding one
1270          * in stackmap, and vise versa.
1271          */
1272         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1273         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1274                   "err %d errno %d\n", err, errno))
1275                 goto disable_pmu;
1276
1277         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1278         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1279                   "err %d errno %d\n", err, errno))
1280                 goto disable_pmu;
1281
1282         err = extract_build_id(buf, 256);
1283
1284         if (CHECK(err, "get build_id with readelf",
1285                   "err %d errno %d\n", err, errno))
1286                 goto disable_pmu;
1287
1288         err = bpf_map_get_next_key(stackmap_fd, NULL, &key);
1289         if (CHECK(err, "get_next_key from stackmap",
1290                   "err %d, errno %d\n", err, errno))
1291                 goto disable_pmu;
1292
1293         do {
1294                 char build_id[64];
1295
1296                 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs);
1297                 if (CHECK(err, "lookup_elem from stackmap",
1298                           "err %d, errno %d\n", err, errno))
1299                         goto disable_pmu;
1300                 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i)
1301                         if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID &&
1302                             id_offs[i].offset != 0) {
1303                                 for (j = 0; j < 20; ++j)
1304                                         sprintf(build_id + 2 * j, "%02x",
1305                                                 id_offs[i].build_id[j] & 0xff);
1306                                 if (strstr(buf, build_id) != NULL)
1307                                         build_id_matches = 1;
1308                         }
1309                 previous_key = key;
1310         } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1311
1312         /* stack_map_get_build_id_offset() is racy and sometimes can return
1313          * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID;
1314          * try it one more time.
1315          */
1316         if (build_id_matches < 1 && retry--) {
1317                 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1318                 close(pmu_fd);
1319                 bpf_object__close(obj);
1320                 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
1321                        __func__);
1322                 goto retry;
1323         }
1324
1325         if (CHECK(build_id_matches < 1, "build id match",
1326                   "Didn't find expected build ID from the map\n"))
1327                 goto disable_pmu;
1328
1329         stack_trace_len = PERF_MAX_STACK_DEPTH
1330                 * sizeof(struct bpf_stack_build_id);
1331         err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1332         CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1333               "err %d errno %d\n", err, errno);
1334
1335 disable_pmu:
1336         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1337
1338 close_pmu:
1339         close(pmu_fd);
1340
1341 close_prog:
1342         bpf_object__close(obj);
1343
1344 out:
1345         return;
1346 }
1347
1348 static void test_stacktrace_build_id_nmi(void)
1349 {
1350         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1351         const char *file = "./test_stacktrace_build_id.o";
1352         int err, pmu_fd, prog_fd;
1353         struct perf_event_attr attr = {
1354                 .sample_freq = 5000,
1355                 .freq = 1,
1356                 .type = PERF_TYPE_HARDWARE,
1357                 .config = PERF_COUNT_HW_CPU_CYCLES,
1358         };
1359         __u32 key, previous_key, val, duration = 0;
1360         struct bpf_object *obj;
1361         char buf[256];
1362         int i, j;
1363         struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
1364         int build_id_matches = 0;
1365         int retry = 1;
1366
1367 retry:
1368         err = bpf_prog_load(file, BPF_PROG_TYPE_PERF_EVENT, &obj, &prog_fd);
1369         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1370                 return;
1371
1372         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1373                          0 /* cpu 0 */, -1 /* group id */,
1374                          0 /* flags */);
1375         if (CHECK(pmu_fd < 0, "perf_event_open",
1376                   "err %d errno %d. Does the test host support PERF_COUNT_HW_CPU_CYCLES?\n",
1377                   pmu_fd, errno))
1378                 goto close_prog;
1379
1380         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1381         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
1382                   err, errno))
1383                 goto close_pmu;
1384
1385         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1386         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
1387                   err, errno))
1388                 goto disable_pmu;
1389
1390         /* find map fds */
1391         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1392         if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
1393                   "err %d errno %d\n", err, errno))
1394                 goto disable_pmu;
1395
1396         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1397         if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
1398                   "err %d errno %d\n", err, errno))
1399                 goto disable_pmu;
1400
1401         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1402         if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
1403                   err, errno))
1404                 goto disable_pmu;
1405
1406         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1407         if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1408                   "err %d errno %d\n", err, errno))
1409                 goto disable_pmu;
1410
1411         assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1412                == 0);
1413         assert(system("taskset 0x1 ./urandom_read 100000") == 0);
1414         /* disable stack trace collection */
1415         key = 0;
1416         val = 1;
1417         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1418
1419         /* for every element in stackid_hmap, we can find a corresponding one
1420          * in stackmap, and vise versa.
1421          */
1422         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1423         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1424                   "err %d errno %d\n", err, errno))
1425                 goto disable_pmu;
1426
1427         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1428         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1429                   "err %d errno %d\n", err, errno))
1430                 goto disable_pmu;
1431
1432         err = extract_build_id(buf, 256);
1433
1434         if (CHECK(err, "get build_id with readelf",
1435                   "err %d errno %d\n", err, errno))
1436                 goto disable_pmu;
1437
1438         err = bpf_map_get_next_key(stackmap_fd, NULL, &key);
1439         if (CHECK(err, "get_next_key from stackmap",
1440                   "err %d, errno %d\n", err, errno))
1441                 goto disable_pmu;
1442
1443         do {
1444                 char build_id[64];
1445
1446                 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs);
1447                 if (CHECK(err, "lookup_elem from stackmap",
1448                           "err %d, errno %d\n", err, errno))
1449                         goto disable_pmu;
1450                 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i)
1451                         if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID &&
1452                             id_offs[i].offset != 0) {
1453                                 for (j = 0; j < 20; ++j)
1454                                         sprintf(build_id + 2 * j, "%02x",
1455                                                 id_offs[i].build_id[j] & 0xff);
1456                                 if (strstr(buf, build_id) != NULL)
1457                                         build_id_matches = 1;
1458                         }
1459                 previous_key = key;
1460         } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1461
1462         /* stack_map_get_build_id_offset() is racy and sometimes can return
1463          * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID;
1464          * try it one more time.
1465          */
1466         if (build_id_matches < 1 && retry--) {
1467                 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1468                 close(pmu_fd);
1469                 bpf_object__close(obj);
1470                 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
1471                        __func__);
1472                 goto retry;
1473         }
1474
1475         if (CHECK(build_id_matches < 1, "build id match",
1476                   "Didn't find expected build ID from the map\n"))
1477                 goto disable_pmu;
1478
1479         /*
1480          * We intentionally skip compare_stack_ips(). This is because we
1481          * only support one in_nmi() ips-to-build_id translation per cpu
1482          * at any time, thus stack_amap here will always fallback to
1483          * BPF_STACK_BUILD_ID_IP;
1484          */
1485
1486 disable_pmu:
1487         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1488
1489 close_pmu:
1490         close(pmu_fd);
1491
1492 close_prog:
1493         bpf_object__close(obj);
1494 }
1495
1496 #define MAX_CNT_RAWTP   10ull
1497 #define MAX_STACK_RAWTP 100
1498 struct get_stack_trace_t {
1499         int pid;
1500         int kern_stack_size;
1501         int user_stack_size;
1502         int user_stack_buildid_size;
1503         __u64 kern_stack[MAX_STACK_RAWTP];
1504         __u64 user_stack[MAX_STACK_RAWTP];
1505         struct bpf_stack_build_id user_stack_buildid[MAX_STACK_RAWTP];
1506 };
1507
1508 static int get_stack_print_output(void *data, int size)
1509 {
1510         bool good_kern_stack = false, good_user_stack = false;
1511         const char *nonjit_func = "___bpf_prog_run";
1512         struct get_stack_trace_t *e = data;
1513         int i, num_stack;
1514         static __u64 cnt;
1515         struct ksym *ks;
1516
1517         cnt++;
1518
1519         if (size < sizeof(struct get_stack_trace_t)) {
1520                 __u64 *raw_data = data;
1521                 bool found = false;
1522
1523                 num_stack = size / sizeof(__u64);
1524                 /* If jit is enabled, we do not have a good way to
1525                  * verify the sanity of the kernel stack. So we
1526                  * just assume it is good if the stack is not empty.
1527                  * This could be improved in the future.
1528                  */
1529                 if (jit_enabled) {
1530                         found = num_stack > 0;
1531                 } else {
1532                         for (i = 0; i < num_stack; i++) {
1533                                 ks = ksym_search(raw_data[i]);
1534                                 if (strcmp(ks->name, nonjit_func) == 0) {
1535                                         found = true;
1536                                         break;
1537                                 }
1538                         }
1539                 }
1540                 if (found) {
1541                         good_kern_stack = true;
1542                         good_user_stack = true;
1543                 }
1544         } else {
1545                 num_stack = e->kern_stack_size / sizeof(__u64);
1546                 if (jit_enabled) {
1547                         good_kern_stack = num_stack > 0;
1548                 } else {
1549                         for (i = 0; i < num_stack; i++) {
1550                                 ks = ksym_search(e->kern_stack[i]);
1551                                 if (strcmp(ks->name, nonjit_func) == 0) {
1552                                         good_kern_stack = true;
1553                                         break;
1554                                 }
1555                         }
1556                 }
1557                 if (e->user_stack_size > 0 && e->user_stack_buildid_size > 0)
1558                         good_user_stack = true;
1559         }
1560         if (!good_kern_stack || !good_user_stack)
1561                 return LIBBPF_PERF_EVENT_ERROR;
1562
1563         if (cnt == MAX_CNT_RAWTP)
1564                 return LIBBPF_PERF_EVENT_DONE;
1565
1566         return LIBBPF_PERF_EVENT_CONT;
1567 }
1568
1569 static void test_get_stack_raw_tp(void)
1570 {
1571         const char *file = "./test_get_stack_rawtp.o";
1572         int i, efd, err, prog_fd, pmu_fd, perfmap_fd;
1573         struct perf_event_attr attr = {};
1574         struct timespec tv = {0, 10};
1575         __u32 key = 0, duration = 0;
1576         struct bpf_object *obj;
1577
1578         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1579         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1580                 return;
1581
1582         efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1583         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1584                 goto close_prog;
1585
1586         perfmap_fd = bpf_find_map(__func__, obj, "perfmap");
1587         if (CHECK(perfmap_fd < 0, "bpf_find_map", "err %d errno %d\n",
1588                   perfmap_fd, errno))
1589                 goto close_prog;
1590
1591         err = load_kallsyms();
1592         if (CHECK(err < 0, "load_kallsyms", "err %d errno %d\n", err, errno))
1593                 goto close_prog;
1594
1595         attr.sample_type = PERF_SAMPLE_RAW;
1596         attr.type = PERF_TYPE_SOFTWARE;
1597         attr.config = PERF_COUNT_SW_BPF_OUTPUT;
1598         pmu_fd = syscall(__NR_perf_event_open, &attr, getpid()/*pid*/, -1/*cpu*/,
1599                          -1/*group_fd*/, 0);
1600         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", pmu_fd,
1601                   errno))
1602                 goto close_prog;
1603
1604         err = bpf_map_update_elem(perfmap_fd, &key, &pmu_fd, BPF_ANY);
1605         if (CHECK(err < 0, "bpf_map_update_elem", "err %d errno %d\n", err,
1606                   errno))
1607                 goto close_prog;
1608
1609         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1610         if (CHECK(err < 0, "ioctl PERF_EVENT_IOC_ENABLE", "err %d errno %d\n",
1611                   err, errno))
1612                 goto close_prog;
1613
1614         err = perf_event_mmap(pmu_fd);
1615         if (CHECK(err < 0, "perf_event_mmap", "err %d errno %d\n", err, errno))
1616                 goto close_prog;
1617
1618         /* trigger some syscall action */
1619         for (i = 0; i < MAX_CNT_RAWTP; i++)
1620                 nanosleep(&tv, NULL);
1621
1622         err = perf_event_poller(pmu_fd, get_stack_print_output);
1623         if (CHECK(err < 0, "perf_event_poller", "err %d errno %d\n", err, errno))
1624                 goto close_prog;
1625
1626         goto close_prog_noerr;
1627 close_prog:
1628         error_cnt++;
1629 close_prog_noerr:
1630         bpf_object__close(obj);
1631 }
1632
1633 static void test_task_fd_query_rawtp(void)
1634 {
1635         const char *file = "./test_get_stack_rawtp.o";
1636         __u64 probe_offset, probe_addr;
1637         __u32 len, prog_id, fd_type;
1638         struct bpf_object *obj;
1639         int efd, err, prog_fd;
1640         __u32 duration = 0;
1641         char buf[256];
1642
1643         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1644         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1645                 return;
1646
1647         efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1648         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1649                 goto close_prog;
1650
1651         /* query (getpid(), efd) */
1652         len = sizeof(buf);
1653         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1654                                 &fd_type, &probe_offset, &probe_addr);
1655         if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err,
1656                   errno))
1657                 goto close_prog;
1658
1659         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1660               strcmp(buf, "sys_enter") == 0;
1661         if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n",
1662                   fd_type, buf))
1663                 goto close_prog;
1664
1665         /* test zero len */
1666         len = 0;
1667         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1668                                 &fd_type, &probe_offset, &probe_addr);
1669         if (CHECK(err < 0, "bpf_task_fd_query (len = 0)", "err %d errno %d\n",
1670                   err, errno))
1671                 goto close_prog;
1672         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1673               len == strlen("sys_enter");
1674         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1675                 goto close_prog;
1676
1677         /* test empty buffer */
1678         len = sizeof(buf);
1679         err = bpf_task_fd_query(getpid(), efd, 0, 0, &len, &prog_id,
1680                                 &fd_type, &probe_offset, &probe_addr);
1681         if (CHECK(err < 0, "bpf_task_fd_query (buf = 0)", "err %d errno %d\n",
1682                   err, errno))
1683                 goto close_prog;
1684         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1685               len == strlen("sys_enter");
1686         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1687                 goto close_prog;
1688
1689         /* test smaller buffer */
1690         len = 3;
1691         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1692                                 &fd_type, &probe_offset, &probe_addr);
1693         if (CHECK(err >= 0 || errno != ENOSPC, "bpf_task_fd_query (len = 3)",
1694                   "err %d errno %d\n", err, errno))
1695                 goto close_prog;
1696         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1697               len == strlen("sys_enter") &&
1698               strcmp(buf, "sy") == 0;
1699         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1700                 goto close_prog;
1701
1702         goto close_prog_noerr;
1703 close_prog:
1704         error_cnt++;
1705 close_prog_noerr:
1706         bpf_object__close(obj);
1707 }
1708
1709 static void test_task_fd_query_tp_core(const char *probe_name,
1710                                        const char *tp_name)
1711 {
1712         const char *file = "./test_tracepoint.o";
1713         int err, bytes, efd, prog_fd, pmu_fd;
1714         struct perf_event_attr attr = {};
1715         __u64 probe_offset, probe_addr;
1716         __u32 len, prog_id, fd_type;
1717         struct bpf_object *obj;
1718         __u32 duration = 0;
1719         char buf[256];
1720
1721         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1722         if (CHECK(err, "bpf_prog_load", "err %d errno %d\n", err, errno))
1723                 goto close_prog;
1724
1725         snprintf(buf, sizeof(buf),
1726                  "/sys/kernel/debug/tracing/events/%s/id", probe_name);
1727         efd = open(buf, O_RDONLY, 0);
1728         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1729                 goto close_prog;
1730         bytes = read(efd, buf, sizeof(buf));
1731         close(efd);
1732         if (CHECK(bytes <= 0 || bytes >= sizeof(buf), "read",
1733                   "bytes %d errno %d\n", bytes, errno))
1734                 goto close_prog;
1735
1736         attr.config = strtol(buf, NULL, 0);
1737         attr.type = PERF_TYPE_TRACEPOINT;
1738         attr.sample_type = PERF_SAMPLE_RAW;
1739         attr.sample_period = 1;
1740         attr.wakeup_events = 1;
1741         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1742                          0 /* cpu 0 */, -1 /* group id */,
1743                          0 /* flags */);
1744         if (CHECK(err, "perf_event_open", "err %d errno %d\n", err, errno))
1745                 goto close_pmu;
1746
1747         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1748         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", err,
1749                   errno))
1750                 goto close_pmu;
1751
1752         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1753         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", err,
1754                   errno))
1755                 goto close_pmu;
1756
1757         /* query (getpid(), pmu_fd) */
1758         len = sizeof(buf);
1759         err = bpf_task_fd_query(getpid(), pmu_fd, 0, buf, &len, &prog_id,
1760                                 &fd_type, &probe_offset, &probe_addr);
1761         if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err,
1762                   errno))
1763                 goto close_pmu;
1764
1765         err = (fd_type == BPF_FD_TYPE_TRACEPOINT) && !strcmp(buf, tp_name);
1766         if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n",
1767                   fd_type, buf))
1768                 goto close_pmu;
1769
1770         close(pmu_fd);
1771         goto close_prog_noerr;
1772
1773 close_pmu:
1774         close(pmu_fd);
1775 close_prog:
1776         error_cnt++;
1777 close_prog_noerr:
1778         bpf_object__close(obj);
1779 }
1780
1781 static void test_task_fd_query_tp(void)
1782 {
1783         test_task_fd_query_tp_core("sched/sched_switch",
1784                                    "sched_switch");
1785         test_task_fd_query_tp_core("syscalls/sys_enter_read",
1786                                    "sys_enter_read");
1787 }
1788
1789 static int libbpf_debug_print(enum libbpf_print_level level,
1790                               const char *format, va_list args)
1791 {
1792         if (level == LIBBPF_DEBUG)
1793                 return 0;
1794
1795         return vfprintf(stderr, format, args);
1796 }
1797
1798 static void test_reference_tracking()
1799 {
1800         const char *file = "./test_sk_lookup_kern.o";
1801         struct bpf_object *obj;
1802         struct bpf_program *prog;
1803         __u32 duration = 0;
1804         int err = 0;
1805
1806         obj = bpf_object__open(file);
1807         if (IS_ERR(obj)) {
1808                 error_cnt++;
1809                 return;
1810         }
1811
1812         bpf_object__for_each_program(prog, obj) {
1813                 const char *title;
1814
1815                 /* Ignore .text sections */
1816                 title = bpf_program__title(prog, false);
1817                 if (strstr(title, ".text") != NULL)
1818                         continue;
1819
1820                 bpf_program__set_type(prog, BPF_PROG_TYPE_SCHED_CLS);
1821
1822                 /* Expect verifier failure if test name has 'fail' */
1823                 if (strstr(title, "fail") != NULL) {
1824                         libbpf_set_print(NULL);
1825                         err = !bpf_program__load(prog, "GPL", 0);
1826                         libbpf_set_print(libbpf_debug_print);
1827                 } else {
1828                         err = bpf_program__load(prog, "GPL", 0);
1829                 }
1830                 CHECK(err, title, "\n");
1831         }
1832         bpf_object__close(obj);
1833 }
1834
1835 enum {
1836         QUEUE,
1837         STACK,
1838 };
1839
1840 static void test_queue_stack_map(int type)
1841 {
1842         const int MAP_SIZE = 32;
1843         __u32 vals[MAP_SIZE], duration, retval, size, val;
1844         int i, err, prog_fd, map_in_fd, map_out_fd;
1845         char file[32], buf[128];
1846         struct bpf_object *obj;
1847         struct iphdr *iph = (void *)buf + sizeof(struct ethhdr);
1848
1849         /* Fill test values to be used */
1850         for (i = 0; i < MAP_SIZE; i++)
1851                 vals[i] = rand();
1852
1853         if (type == QUEUE)
1854                 strncpy(file, "./test_queue_map.o", sizeof(file));
1855         else if (type == STACK)
1856                 strncpy(file, "./test_stack_map.o", sizeof(file));
1857         else
1858                 return;
1859
1860         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
1861         if (err) {
1862                 error_cnt++;
1863                 return;
1864         }
1865
1866         map_in_fd = bpf_find_map(__func__, obj, "map_in");
1867         if (map_in_fd < 0)
1868                 goto out;
1869
1870         map_out_fd = bpf_find_map(__func__, obj, "map_out");
1871         if (map_out_fd < 0)
1872                 goto out;
1873
1874         /* Push 32 elements to the input map */
1875         for (i = 0; i < MAP_SIZE; i++) {
1876                 err = bpf_map_update_elem(map_in_fd, NULL, &vals[i], 0);
1877                 if (err) {
1878                         error_cnt++;
1879                         goto out;
1880                 }
1881         }
1882
1883         /* The eBPF program pushes iph.saddr in the output map,
1884          * pops the input map and saves this value in iph.daddr
1885          */
1886         for (i = 0; i < MAP_SIZE; i++) {
1887                 if (type == QUEUE) {
1888                         val = vals[i];
1889                         pkt_v4.iph.saddr = vals[i] * 5;
1890                 } else if (type == STACK) {
1891                         val = vals[MAP_SIZE - 1 - i];
1892                         pkt_v4.iph.saddr = vals[MAP_SIZE - 1 - i] * 5;
1893                 }
1894
1895                 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
1896                                         buf, &size, &retval, &duration);
1897                 if (err || retval || size != sizeof(pkt_v4) ||
1898                     iph->daddr != val)
1899                         break;
1900         }
1901
1902         CHECK(err || retval || size != sizeof(pkt_v4) || iph->daddr != val,
1903               "bpf_map_pop_elem",
1904               "err %d errno %d retval %d size %d iph->daddr %u\n",
1905               err, errno, retval, size, iph->daddr);
1906
1907         /* Queue is empty, program should return TC_ACT_SHOT */
1908         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
1909                                 buf, &size, &retval, &duration);
1910         CHECK(err || retval != 2 /* TC_ACT_SHOT */|| size != sizeof(pkt_v4),
1911               "check-queue-stack-map-empty",
1912               "err %d errno %d retval %d size %d\n",
1913               err, errno, retval, size);
1914
1915         /* Check that the program pushed elements correctly */
1916         for (i = 0; i < MAP_SIZE; i++) {
1917                 err = bpf_map_lookup_and_delete_elem(map_out_fd, NULL, &val);
1918                 if (err || val != vals[i] * 5)
1919                         break;
1920         }
1921
1922         CHECK(i != MAP_SIZE && (err || val != vals[i] * 5),
1923               "bpf_map_push_elem", "err %d value %u\n", err, val);
1924
1925 out:
1926         pkt_v4.iph.saddr = 0;
1927         bpf_object__close(obj);
1928 }
1929
1930 #define CHECK_FLOW_KEYS(desc, got, expected)                            \
1931         CHECK(memcmp(&got, &expected, sizeof(got)) != 0,                \
1932               desc,                                                     \
1933               "nhoff=%u/%u "                                            \
1934               "thoff=%u/%u "                                            \
1935               "addr_proto=0x%x/0x%x "                                   \
1936               "is_frag=%u/%u "                                          \
1937               "is_first_frag=%u/%u "                                    \
1938               "is_encap=%u/%u "                                         \
1939               "n_proto=0x%x/0x%x "                                      \
1940               "sport=%u/%u "                                            \
1941               "dport=%u/%u\n",                                          \
1942               got.nhoff, expected.nhoff,                                \
1943               got.thoff, expected.thoff,                                \
1944               got.addr_proto, expected.addr_proto,                      \
1945               got.is_frag, expected.is_frag,                            \
1946               got.is_first_frag, expected.is_first_frag,                \
1947               got.is_encap, expected.is_encap,                          \
1948               got.n_proto, expected.n_proto,                            \
1949               got.sport, expected.sport,                                \
1950               got.dport, expected.dport)
1951
1952 static struct bpf_flow_keys pkt_v4_flow_keys = {
1953         .nhoff = 0,
1954         .thoff = sizeof(struct iphdr),
1955         .addr_proto = ETH_P_IP,
1956         .ip_proto = IPPROTO_TCP,
1957         .n_proto = __bpf_constant_htons(ETH_P_IP),
1958 };
1959
1960 static struct bpf_flow_keys pkt_v6_flow_keys = {
1961         .nhoff = 0,
1962         .thoff = sizeof(struct ipv6hdr),
1963         .addr_proto = ETH_P_IPV6,
1964         .ip_proto = IPPROTO_TCP,
1965         .n_proto = __bpf_constant_htons(ETH_P_IPV6),
1966 };
1967
1968 static void test_flow_dissector(void)
1969 {
1970         struct bpf_flow_keys flow_keys;
1971         struct bpf_object *obj;
1972         __u32 duration, retval;
1973         int err, prog_fd;
1974         __u32 size;
1975
1976         err = bpf_flow_load(&obj, "./bpf_flow.o", "flow_dissector",
1977                             "jmp_table", &prog_fd);
1978         if (err) {
1979                 error_cnt++;
1980                 return;
1981         }
1982
1983         err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4),
1984                                 &flow_keys, &size, &retval, &duration);
1985         CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv4",
1986               "err %d errno %d retval %d duration %d size %u/%lu\n",
1987               err, errno, retval, duration, size, sizeof(flow_keys));
1988         CHECK_FLOW_KEYS("ipv4_flow_keys", flow_keys, pkt_v4_flow_keys);
1989
1990         err = bpf_prog_test_run(prog_fd, 10, &pkt_v6, sizeof(pkt_v6),
1991                                 &flow_keys, &size, &retval, &duration);
1992         CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv6",
1993               "err %d errno %d retval %d duration %d size %u/%lu\n",
1994               err, errno, retval, duration, size, sizeof(flow_keys));
1995         CHECK_FLOW_KEYS("ipv6_flow_keys", flow_keys, pkt_v6_flow_keys);
1996
1997         bpf_object__close(obj);
1998 }
1999
2000 static void *test_spin_lock(void *arg)
2001 {
2002         __u32 duration, retval;
2003         int err, prog_fd = *(u32 *) arg;
2004
2005         err = bpf_prog_test_run(prog_fd, 10000, &pkt_v4, sizeof(pkt_v4),
2006                                 NULL, NULL, &retval, &duration);
2007         CHECK(err || retval, "",
2008               "err %d errno %d retval %d duration %d\n",
2009               err, errno, retval, duration);
2010         pthread_exit(arg);
2011 }
2012
2013 static void test_spinlock(void)
2014 {
2015         const char *file = "./test_spin_lock.o";
2016         pthread_t thread_id[4];
2017         struct bpf_object *obj;
2018         int prog_fd;
2019         int err = 0, i;
2020         void *ret;
2021
2022         err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd);
2023         if (err) {
2024                 printf("test_spin_lock:bpf_prog_load errno %d\n", errno);
2025                 goto close_prog;
2026         }
2027         for (i = 0; i < 4; i++)
2028                 assert(pthread_create(&thread_id[i], NULL,
2029                                       &test_spin_lock, &prog_fd) == 0);
2030         for (i = 0; i < 4; i++)
2031                 assert(pthread_join(thread_id[i], &ret) == 0 &&
2032                        ret == (void *)&prog_fd);
2033         goto close_prog_noerr;
2034 close_prog:
2035         error_cnt++;
2036 close_prog_noerr:
2037         bpf_object__close(obj);
2038 }
2039
2040 static void *parallel_map_access(void *arg)
2041 {
2042         int err, map_fd = *(u32 *) arg;
2043         int vars[17], i, j, rnd, key = 0;
2044
2045         for (i = 0; i < 10000; i++) {
2046                 err = bpf_map_lookup_elem_flags(map_fd, &key, vars, BPF_F_LOCK);
2047                 if (err) {
2048                         printf("lookup failed\n");
2049                         error_cnt++;
2050                         goto out;
2051                 }
2052                 if (vars[0] != 0) {
2053                         printf("lookup #%d var[0]=%d\n", i, vars[0]);
2054                         error_cnt++;
2055                         goto out;
2056                 }
2057                 rnd = vars[1];
2058                 for (j = 2; j < 17; j++) {
2059                         if (vars[j] == rnd)
2060                                 continue;
2061                         printf("lookup #%d var[1]=%d var[%d]=%d\n",
2062                                i, rnd, j, vars[j]);
2063                         error_cnt++;
2064                         goto out;
2065                 }
2066         }
2067 out:
2068         pthread_exit(arg);
2069 }
2070
2071 static void test_map_lock(void)
2072 {
2073         const char *file = "./test_map_lock.o";
2074         int prog_fd, map_fd[2], vars[17] = {};
2075         pthread_t thread_id[6];
2076         struct bpf_object *obj;
2077         int err = 0, key = 0, i;
2078         void *ret;
2079
2080         err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd);
2081         if (err) {
2082                 printf("test_map_lock:bpf_prog_load errno %d\n", errno);
2083                 goto close_prog;
2084         }
2085         map_fd[0] = bpf_find_map(__func__, obj, "hash_map");
2086         if (map_fd[0] < 0)
2087                 goto close_prog;
2088         map_fd[1] = bpf_find_map(__func__, obj, "array_map");
2089         if (map_fd[1] < 0)
2090                 goto close_prog;
2091
2092         bpf_map_update_elem(map_fd[0], &key, vars, BPF_F_LOCK);
2093
2094         for (i = 0; i < 4; i++)
2095                 assert(pthread_create(&thread_id[i], NULL,
2096                                       &test_spin_lock, &prog_fd) == 0);
2097         for (i = 4; i < 6; i++)
2098                 assert(pthread_create(&thread_id[i], NULL,
2099                                       &parallel_map_access, &map_fd[i - 4]) == 0);
2100         for (i = 0; i < 4; i++)
2101                 assert(pthread_join(thread_id[i], &ret) == 0 &&
2102                        ret == (void *)&prog_fd);
2103         for (i = 4; i < 6; i++)
2104                 assert(pthread_join(thread_id[i], &ret) == 0 &&
2105                        ret == (void *)&map_fd[i - 4]);
2106         goto close_prog_noerr;
2107 close_prog:
2108         error_cnt++;
2109 close_prog_noerr:
2110         bpf_object__close(obj);
2111 }
2112
2113 static void sigalrm_handler(int s) {}
2114 static struct sigaction sigalrm_action = {
2115         .sa_handler = sigalrm_handler,
2116 };
2117
2118 static void test_signal_pending(enum bpf_prog_type prog_type)
2119 {
2120         struct bpf_insn prog[4096];
2121         struct itimerval timeo = {
2122                 .it_value.tv_usec = 100000, /* 100ms */
2123         };
2124         __u32 duration, retval;
2125         int prog_fd;
2126         int err;
2127         int i;
2128
2129         for (i = 0; i < ARRAY_SIZE(prog); i++)
2130                 prog[i] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0);
2131         prog[ARRAY_SIZE(prog) - 1] = BPF_EXIT_INSN();
2132
2133         prog_fd = bpf_load_program(prog_type, prog, ARRAY_SIZE(prog),
2134                                    "GPL", 0, NULL, 0);
2135         CHECK(prog_fd < 0, "test-run", "errno %d\n", errno);
2136
2137         err = sigaction(SIGALRM, &sigalrm_action, NULL);
2138         CHECK(err, "test-run-signal-sigaction", "errno %d\n", errno);
2139
2140         err = setitimer(ITIMER_REAL, &timeo, NULL);
2141         CHECK(err, "test-run-signal-timer", "errno %d\n", errno);
2142
2143         err = bpf_prog_test_run(prog_fd, 0xffffffff, &pkt_v4, sizeof(pkt_v4),
2144                                 NULL, NULL, &retval, &duration);
2145         CHECK(duration > 500000000, /* 500ms */
2146               "test-run-signal-duration",
2147               "duration %dns > 500ms\n",
2148               duration);
2149
2150         signal(SIGALRM, SIG_DFL);
2151 }
2152
2153 int main(void)
2154 {
2155         srand(time(NULL));
2156
2157         jit_enabled = is_jit_enabled();
2158
2159         test_pkt_access();
2160         test_prog_run_xattr();
2161         test_xdp();
2162         test_xdp_adjust_tail();
2163         test_l4lb_all();
2164         test_xdp_noinline();
2165         test_tcp_estats();
2166         test_bpf_obj_id();
2167         test_pkt_md_access();
2168         test_obj_name();
2169         test_tp_attach_query();
2170         test_stacktrace_map();
2171         test_stacktrace_build_id();
2172         test_stacktrace_build_id_nmi();
2173         test_stacktrace_map_raw_tp();
2174         test_get_stack_raw_tp();
2175         test_task_fd_query_rawtp();
2176         test_task_fd_query_tp();
2177         test_reference_tracking();
2178         test_queue_stack_map(QUEUE);
2179         test_queue_stack_map(STACK);
2180         test_flow_dissector();
2181         test_spinlock();
2182         test_map_lock();
2183         test_signal_pending(BPF_PROG_TYPE_SOCKET_FILTER);
2184         test_signal_pending(BPF_PROG_TYPE_FLOW_DISSECTOR);
2185
2186         printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt);
2187         return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
2188 }