tools/testing/selftests/bpf/test_progs.c

   1 /* Copyright (c) 2017 Facebook
   2  *
   3  * This program is free software; you can redistribute it and/or
   4  * modify it under the terms of version 2 of the GNU General Public
   5  * License as published by the Free Software Foundation.
   6  */
   7 #include <stdio.h>
   8 #include <unistd.h>
   9 #include <errno.h>
  10 #include <string.h>
  11 #include <assert.h>
  12 #include <stdlib.h>
  13 #include <stdarg.h>
  14 #include <time.h>
  15
  16 #include <linux/types.h>
  17 typedef __u16 __sum16;
  18 #include <arpa/inet.h>
  19 #include <linux/if_ether.h>
  20 #include <linux/if_packet.h>
  21 #include <linux/ip.h>
  22 #include <linux/ipv6.h>
  23 #include <linux/tcp.h>
  24 #include <linux/filter.h>
  25 #include <linux/perf_event.h>
  26 #include <linux/unistd.h>
  27
  28 #include <sys/ioctl.h>
  29 #include <sys/wait.h>
  30 #include <sys/types.h>
  31 #include <fcntl.h>
  32 #include <pthread.h>
  33 #include <linux/bpf.h>
  34 #include <linux/err.h>
  35 #include <bpf/bpf.h>
  36 #include <bpf/libbpf.h>
  37
  38 #include "test_iptunnel_common.h"
  39 #include "bpf_util.h"
  40 #include "bpf_endian.h"
  41 #include "bpf_rlimit.h"
  42 #include "trace_helpers.h"
  43 #include "flow_dissector_load.h"
  44
  45 static int error_cnt, pass_cnt;
  46 static bool jit_enabled;
  47
  48 #define MAGIC_BYTES 123
  49
  50 /* ipv4 test vector */
  51 static struct {
  52         struct ethhdr eth;
  53         struct iphdr iph;
  54         struct tcphdr tcp;
  55 } __packed pkt_v4 = {
  56         .eth.h_proto = __bpf_constant_htons(ETH_P_IP),
  57         .iph.ihl = 5,
  58         .iph.protocol = IPPROTO_TCP,
  59         .iph.tot_len = __bpf_constant_htons(MAGIC_BYTES),
  60         .tcp.urg_ptr = 123,
  61         .tcp.doff = 5,
  62 };
  63
  64 /* ipv6 test vector */
  65 static struct {
  66         struct ethhdr eth;
  67         struct ipv6hdr iph;
  68         struct tcphdr tcp;
  69 } __packed pkt_v6 = {
  70         .eth.h_proto = __bpf_constant_htons(ETH_P_IPV6),
  71         .iph.nexthdr = IPPROTO_TCP,
  72         .iph.payload_len = __bpf_constant_htons(MAGIC_BYTES),
  73         .tcp.urg_ptr = 123,
  74         .tcp.doff = 5,
  75 };
  76
  77 #define _CHECK(condition, tag, duration, format...) ({                  \
  78         int __ret = !!(condition);                                      \
  79         if (__ret) {                                                    \
  80                 error_cnt++;                                            \
  81                 printf("%s:FAIL:%s ", __func__, tag);                   \
  82                 printf(format);                                         \
  83         } else {                                                        \
  84                 pass_cnt++;                                             \
  85                 printf("%s:PASS:%s %d nsec\n", __func__, tag, duration);\
  86         }                                                               \
  87         __ret;                                                          \
  88 })
  89
  90 #define CHECK(condition, tag, format...) \
  91         _CHECK(condition, tag, duration, format)
  92 #define CHECK_ATTR(condition, tag, format...) \
  93         _CHECK(condition, tag, tattr.duration, format)
  94
  95 static int bpf_find_map(const char *test, struct bpf_object *obj,
  96                         const char *name)
  97 {
  98         struct bpf_map *map;
  99
 100         map = bpf_object__find_map_by_name(obj, name);
 101         if (!map) {
 102                 printf("%s:FAIL:map '%s' not found\n", test, name);
 103                 error_cnt++;
 104                 return -1;
 105         }
 106         return bpf_map__fd(map);
 107 }
 108
 109 static void test_pkt_access(void)
 110 {
 111         const char *file = "./test_pkt_access.o";
 112         struct bpf_object *obj;
 113         __u32 duration, retval;
 114         int err, prog_fd;
 115
 116         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 117         if (err) {
 118                 error_cnt++;
 119                 return;
 120         }
 121
 122         err = bpf_prog_test_run(prog_fd, 100000, &pkt_v4, sizeof(pkt_v4),
 123                                 NULL, NULL, &retval, &duration);
 124         CHECK(err || retval, "ipv4",
 125               "err %d errno %d retval %d duration %d\n",
 126               err, errno, retval, duration);
 127
 128         err = bpf_prog_test_run(prog_fd, 100000, &pkt_v6, sizeof(pkt_v6),
 129                                 NULL, NULL, &retval, &duration);
 130         CHECK(err || retval, "ipv6",
 131               "err %d errno %d retval %d duration %d\n",
 132               err, errno, retval, duration);
 133         bpf_object__close(obj);
 134 }
 135
 136 static void test_prog_run_xattr(void)
 137 {
 138         const char *file = "./test_pkt_access.o";
 139         struct bpf_object *obj;
 140         char buf[10];
 141         int err;
 142         struct bpf_prog_test_run_attr tattr = {
 143                 .repeat = 1,
 144                 .data_in = &pkt_v4,
 145                 .data_size_in = sizeof(pkt_v4),
 146                 .data_out = buf,
 147                 .data_size_out = 5,
 148         };
 149
 150         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj,
 151                             &tattr.prog_fd);
 152         if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno))
 153                 return;
 154
 155         memset(buf, 0, sizeof(buf));
 156
 157         err = bpf_prog_test_run_xattr(&tattr);
 158         CHECK_ATTR(err != -1 || errno != ENOSPC || tattr.retval, "run",
 159               "err %d errno %d retval %d\n", err, errno, tattr.retval);
 160
 161         CHECK_ATTR(tattr.data_size_out != sizeof(pkt_v4), "data_size_out",
 162               "incorrect output size, want %lu have %u\n",
 163               sizeof(pkt_v4), tattr.data_size_out);
 164
 165         CHECK_ATTR(buf[5] != 0, "overflow",
 166               "BPF_PROG_TEST_RUN ignored size hint\n");
 167
 168         tattr.data_out = NULL;
 169         tattr.data_size_out = 0;
 170         errno = 0;
 171
 172         err = bpf_prog_test_run_xattr(&tattr);
 173         CHECK_ATTR(err || errno || tattr.retval, "run_no_output",
 174               "err %d errno %d retval %d\n", err, errno, tattr.retval);
 175
 176         tattr.data_size_out = 1;
 177         err = bpf_prog_test_run_xattr(&tattr);
 178         CHECK_ATTR(err != -EINVAL, "run_wrong_size_out", "err %d\n", err);
 179
 180         bpf_object__close(obj);
 181 }
 182
 183 static void test_xdp(void)
 184 {
 185         struct vip key4 = {.protocol = 6, .family = AF_INET};
 186         struct vip key6 = {.protocol = 6, .family = AF_INET6};
 187         struct iptnl_info value4 = {.family = AF_INET};
 188         struct iptnl_info value6 = {.family = AF_INET6};
 189         const char *file = "./test_xdp.o";
 190         struct bpf_object *obj;
 191         char buf[128];
 192         struct ipv6hdr *iph6 = (void *)buf + sizeof(struct ethhdr);
 193         struct iphdr *iph = (void *)buf + sizeof(struct ethhdr);
 194         __u32 duration, retval, size;
 195         int err, prog_fd, map_fd;
 196
 197         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 198         if (err) {
 199                 error_cnt++;
 200                 return;
 201         }
 202
 203         map_fd = bpf_find_map(__func__, obj, "vip2tnl");
 204         if (map_fd < 0)
 205                 goto out;
 206         bpf_map_update_elem(map_fd, &key4, &value4, 0);
 207         bpf_map_update_elem(map_fd, &key6, &value6, 0);
 208
 209         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
 210                                 buf, &size, &retval, &duration);
 211
 212         CHECK(err || retval != XDP_TX || size != 74 ||
 213               iph->protocol != IPPROTO_IPIP, "ipv4",
 214               "err %d errno %d retval %d size %d\n",
 215               err, errno, retval, size);
 216
 217         err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6),
 218                                 buf, &size, &retval, &duration);
 219         CHECK(err || retval != XDP_TX || size != 114 ||
 220               iph6->nexthdr != IPPROTO_IPV6, "ipv6",
 221               "err %d errno %d retval %d size %d\n",
 222               err, errno, retval, size);
 223 out:
 224         bpf_object__close(obj);
 225 }
 226
 227 static void test_xdp_adjust_tail(void)
 228 {
 229         const char *file = "./test_adjust_tail.o";
 230         struct bpf_object *obj;
 231         char buf[128];
 232         __u32 duration, retval, size;
 233         int err, prog_fd;
 234
 235         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 236         if (err) {
 237                 error_cnt++;
 238                 return;
 239         }
 240
 241         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
 242                                 buf, &size, &retval, &duration);
 243
 244         CHECK(err || retval != XDP_DROP,
 245               "ipv4", "err %d errno %d retval %d size %d\n",
 246               err, errno, retval, size);
 247
 248         err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6),
 249                                 buf, &size, &retval, &duration);
 250         CHECK(err || retval != XDP_TX || size != 54,
 251               "ipv6", "err %d errno %d retval %d size %d\n",
 252               err, errno, retval, size);
 253         bpf_object__close(obj);
 254 }
 255
 256
 257
 258 #define MAGIC_VAL 0x1234
 259 #define NUM_ITER 100000
 260 #define VIP_NUM 5
 261
 262 static void test_l4lb(const char *file)
 263 {
 264         unsigned int nr_cpus = bpf_num_possible_cpus();
 265         struct vip key = {.protocol = 6};
 266         struct vip_meta {
 267                 __u32 flags;
 268                 __u32 vip_num;
 269         } value = {.vip_num = VIP_NUM};
 270         __u32 stats_key = VIP_NUM;
 271         struct vip_stats {
 272                 __u64 bytes;
 273                 __u64 pkts;
 274         } stats[nr_cpus];
 275         struct real_definition {
 276                 union {
 277                         __be32 dst;
 278                         __be32 dstv6[4];
 279                 };
 280                 __u8 flags;
 281         } real_def = {.dst = MAGIC_VAL};
 282         __u32 ch_key = 11, real_num = 3;
 283         __u32 duration, retval, size;
 284         int err, i, prog_fd, map_fd;
 285         __u64 bytes = 0, pkts = 0;
 286         struct bpf_object *obj;
 287         char buf[128];
 288         u32 *magic = (u32 *)buf;
 289
 290         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 291         if (err) {
 292                 error_cnt++;
 293                 return;
 294         }
 295
 296         map_fd = bpf_find_map(__func__, obj, "vip_map");
 297         if (map_fd < 0)
 298                 goto out;
 299         bpf_map_update_elem(map_fd, &key, &value, 0);
 300
 301         map_fd = bpf_find_map(__func__, obj, "ch_rings");
 302         if (map_fd < 0)
 303                 goto out;
 304         bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
 305
 306         map_fd = bpf_find_map(__func__, obj, "reals");
 307         if (map_fd < 0)
 308                 goto out;
 309         bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
 310
 311         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
 312                                 buf, &size, &retval, &duration);
 313         CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 54 ||
 314               *magic != MAGIC_VAL, "ipv4",
 315               "err %d errno %d retval %d size %d magic %x\n",
 316               err, errno, retval, size, *magic);
 317
 318         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
 319                                 buf, &size, &retval, &duration);
 320         CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 74 ||
 321               *magic != MAGIC_VAL, "ipv6",
 322               "err %d errno %d retval %d size %d magic %x\n",
 323               err, errno, retval, size, *magic);
 324
 325         map_fd = bpf_find_map(__func__, obj, "stats");
 326         if (map_fd < 0)
 327                 goto out;
 328         bpf_map_lookup_elem(map_fd, &stats_key, stats);
 329         for (i = 0; i < nr_cpus; i++) {
 330                 bytes += stats[i].bytes;
 331                 pkts += stats[i].pkts;
 332         }
 333         if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
 334                 error_cnt++;
 335                 printf("test_l4lb:FAIL:stats %lld %lld\n", bytes, pkts);
 336         }
 337 out:
 338         bpf_object__close(obj);
 339 }
 340
 341 static void test_l4lb_all(void)
 342 {
 343         const char *file1 = "./test_l4lb.o";
 344         const char *file2 = "./test_l4lb_noinline.o";
 345
 346         test_l4lb(file1);
 347         test_l4lb(file2);
 348 }
 349
 350 static void test_xdp_noinline(void)
 351 {
 352         const char *file = "./test_xdp_noinline.o";
 353         unsigned int nr_cpus = bpf_num_possible_cpus();
 354         struct vip key = {.protocol = 6};
 355         struct vip_meta {
 356                 __u32 flags;
 357                 __u32 vip_num;
 358         } value = {.vip_num = VIP_NUM};
 359         __u32 stats_key = VIP_NUM;
 360         struct vip_stats {
 361                 __u64 bytes;
 362                 __u64 pkts;
 363         } stats[nr_cpus];
 364         struct real_definition {
 365                 union {
 366                         __be32 dst;
 367                         __be32 dstv6[4];
 368                 };
 369                 __u8 flags;
 370         } real_def = {.dst = MAGIC_VAL};
 371         __u32 ch_key = 11, real_num = 3;
 372         __u32 duration, retval, size;
 373         int err, i, prog_fd, map_fd;
 374         __u64 bytes = 0, pkts = 0;
 375         struct bpf_object *obj;
 376         char buf[128];
 377         u32 *magic = (u32 *)buf;
 378
 379         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 380         if (err) {
 381                 error_cnt++;
 382                 return;
 383         }
 384
 385         map_fd = bpf_find_map(__func__, obj, "vip_map");
 386         if (map_fd < 0)
 387                 goto out;
 388         bpf_map_update_elem(map_fd, &key, &value, 0);
 389
 390         map_fd = bpf_find_map(__func__, obj, "ch_rings");
 391         if (map_fd < 0)
 392                 goto out;
 393         bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
 394
 395         map_fd = bpf_find_map(__func__, obj, "reals");
 396         if (map_fd < 0)
 397                 goto out;
 398         bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
 399
 400         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
 401                                 buf, &size, &retval, &duration);
 402         CHECK(err || retval != 1 || size != 54 ||
 403               *magic != MAGIC_VAL, "ipv4",
 404               "err %d errno %d retval %d size %d magic %x\n",
 405               err, errno, retval, size, *magic);
 406
 407         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
 408                                 buf, &size, &retval, &duration);
 409         CHECK(err || retval != 1 || size != 74 ||
 410               *magic != MAGIC_VAL, "ipv6",
 411               "err %d errno %d retval %d size %d magic %x\n",
 412               err, errno, retval, size, *magic);
 413
 414         map_fd = bpf_find_map(__func__, obj, "stats");
 415         if (map_fd < 0)
 416                 goto out;
 417         bpf_map_lookup_elem(map_fd, &stats_key, stats);
 418         for (i = 0; i < nr_cpus; i++) {
 419                 bytes += stats[i].bytes;
 420                 pkts += stats[i].pkts;
 421         }
 422         if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
 423                 error_cnt++;
 424                 printf("test_xdp_noinline:FAIL:stats %lld %lld\n", bytes, pkts);
 425         }
 426 out:
 427         bpf_object__close(obj);
 428 }
 429
 430 static void test_tcp_estats(void)
 431 {
 432         const char *file = "./test_tcp_estats.o";
 433         int err, prog_fd;
 434         struct bpf_object *obj;
 435         __u32 duration = 0;
 436
 437         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
 438         CHECK(err, "", "err %d errno %d\n", err, errno);
 439         if (err) {
 440                 error_cnt++;
 441                 return;
 442         }
 443
 444         bpf_object__close(obj);
 445 }
 446
 447 static inline __u64 ptr_to_u64(const void *ptr)
 448 {
 449         return (__u64) (unsigned long) ptr;
 450 }
 451
 452 static bool is_jit_enabled(void)
 453 {
 454         const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
 455         bool enabled = false;
 456         int sysctl_fd;
 457
 458         sysctl_fd = open(jit_sysctl, 0, O_RDONLY);
 459         if (sysctl_fd != -1) {
 460                 char tmpc;
 461
 462                 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
 463                         enabled = (tmpc != '0');
 464                 close(sysctl_fd);
 465         }
 466
 467         return enabled;
 468 }
 469
 470 static void test_bpf_obj_id(void)
 471 {
 472         const __u64 array_magic_value = 0xfaceb00c;
 473         const __u32 array_key = 0;
 474         const int nr_iters = 2;
 475         const char *file = "./test_obj_id.o";
 476         const char *expected_prog_name = "test_obj_id";
 477         const char *expected_map_name = "test_map_id";
 478         const __u64 nsec_per_sec = 1000000000;
 479
 480         struct bpf_object *objs[nr_iters];
 481         int prog_fds[nr_iters], map_fds[nr_iters];
 482         /* +1 to test for the info_len returned by kernel */
 483         struct bpf_prog_info prog_infos[nr_iters + 1];
 484         struct bpf_map_info map_infos[nr_iters + 1];
 485         /* Each prog only uses one map. +1 to test nr_map_ids
 486          * returned by kernel.
 487          */
 488         __u32 map_ids[nr_iters + 1];
 489         char jited_insns[128], xlated_insns[128], zeros[128];
 490         __u32 i, next_id, info_len, nr_id_found, duration = 0;
 491         struct timespec real_time_ts, boot_time_ts;
 492         int err = 0;
 493         __u64 array_value;
 494         uid_t my_uid = getuid();
 495         time_t now, load_time;
 496
 497         err = bpf_prog_get_fd_by_id(0);
 498         CHECK(err >= 0 || errno != ENOENT,
 499               "get-fd-by-notexist-prog-id", "err %d errno %d\n", err, errno);
 500
 501         err = bpf_map_get_fd_by_id(0);
 502         CHECK(err >= 0 || errno != ENOENT,
 503               "get-fd-by-notexist-map-id", "err %d errno %d\n", err, errno);
 504
 505         for (i = 0; i < nr_iters; i++)
 506                 objs[i] = NULL;
 507
 508         /* Check bpf_obj_get_info_by_fd() */
 509         bzero(zeros, sizeof(zeros));
 510         for (i = 0; i < nr_iters; i++) {
 511                 now = time(NULL);
 512                 err = bpf_prog_load(file, BPF_PROG_TYPE_SOCKET_FILTER,
 513                                     &objs[i], &prog_fds[i]);
 514                 /* test_obj_id.o is a dumb prog. It should never fail
 515                  * to load.
 516                  */
 517                 if (err)
 518                         error_cnt++;
 519                 assert(!err);
 520
 521                 /* Insert a magic value to the map */
 522                 map_fds[i] = bpf_find_map(__func__, objs[i], "test_map_id");
 523                 assert(map_fds[i] >= 0);
 524                 err = bpf_map_update_elem(map_fds[i], &array_key,
 525                                           &array_magic_value, 0);
 526                 assert(!err);
 527
 528                 /* Check getting map info */
 529                 info_len = sizeof(struct bpf_map_info) * 2;
 530                 bzero(&map_infos[i], info_len);
 531                 err = bpf_obj_get_info_by_fd(map_fds[i], &map_infos[i],
 532                                              &info_len);
 533                 if (CHECK(err ||
 534                           map_infos[i].type != BPF_MAP_TYPE_ARRAY ||
 535                           map_infos[i].key_size != sizeof(__u32) ||
 536                           map_infos[i].value_size != sizeof(__u64) ||
 537                           map_infos[i].max_entries != 1 ||
 538                           map_infos[i].map_flags != 0 ||
 539                           info_len != sizeof(struct bpf_map_info) ||
 540                           strcmp((char *)map_infos[i].name, expected_map_name),
 541                           "get-map-info(fd)",
 542                           "err %d errno %d type %d(%d) info_len %u(%Zu) key_size %u value_size %u max_entries %u map_flags %X name %s(%s)\n",
 543                           err, errno,
 544                           map_infos[i].type, BPF_MAP_TYPE_ARRAY,
 545                           info_len, sizeof(struct bpf_map_info),
 546                           map_infos[i].key_size,
 547                           map_infos[i].value_size,
 548                           map_infos[i].max_entries,
 549                           map_infos[i].map_flags,
 550                           map_infos[i].name, expected_map_name))
 551                         goto done;
 552
 553                 /* Check getting prog info */
 554                 info_len = sizeof(struct bpf_prog_info) * 2;
 555                 bzero(&prog_infos[i], info_len);
 556                 bzero(jited_insns, sizeof(jited_insns));
 557                 bzero(xlated_insns, sizeof(xlated_insns));
 558                 prog_infos[i].jited_prog_insns = ptr_to_u64(jited_insns);
 559                 prog_infos[i].jited_prog_len = sizeof(jited_insns);
 560                 prog_infos[i].xlated_prog_insns = ptr_to_u64(xlated_insns);
 561                 prog_infos[i].xlated_prog_len = sizeof(xlated_insns);
 562                 prog_infos[i].map_ids = ptr_to_u64(map_ids + i);
 563                 prog_infos[i].nr_map_ids = 2;
 564                 err = clock_gettime(CLOCK_REALTIME, &real_time_ts);
 565                 assert(!err);
 566                 err = clock_gettime(CLOCK_BOOTTIME, &boot_time_ts);
 567                 assert(!err);
 568                 err = bpf_obj_get_info_by_fd(prog_fds[i], &prog_infos[i],
 569                                              &info_len);
 570                 load_time = (real_time_ts.tv_sec - boot_time_ts.tv_sec)
 571                         + (prog_infos[i].load_time / nsec_per_sec);
 572                 if (CHECK(err ||
 573                           prog_infos[i].type != BPF_PROG_TYPE_SOCKET_FILTER ||
 574                           info_len != sizeof(struct bpf_prog_info) ||
 575                           (jit_enabled && !prog_infos[i].jited_prog_len) ||
 576                           (jit_enabled &&
 577                            !memcmp(jited_insns, zeros, sizeof(zeros))) ||
 578                           !prog_infos[i].xlated_prog_len ||
 579                           !memcmp(xlated_insns, zeros, sizeof(zeros)) ||
 580                           load_time < now - 60 || load_time > now + 60 ||
 581                           prog_infos[i].created_by_uid != my_uid ||
 582                           prog_infos[i].nr_map_ids != 1 ||
 583                           *(int *)(long)prog_infos[i].map_ids != map_infos[i].id ||
 584                           strcmp((char *)prog_infos[i].name, expected_prog_name),
 585                           "get-prog-info(fd)",
 586                           "err %d errno %d i %d type %d(%d) info_len %u(%Zu) jit_enabled %d jited_prog_len %u xlated_prog_len %u jited_prog %d xlated_prog %d load_time %lu(%lu) uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) name %s(%s)\n",
 587                           err, errno, i,
 588                           prog_infos[i].type, BPF_PROG_TYPE_SOCKET_FILTER,
 589                           info_len, sizeof(struct bpf_prog_info),
 590                           jit_enabled,
 591                           prog_infos[i].jited_prog_len,
 592                           prog_infos[i].xlated_prog_len,
 593                           !!memcmp(jited_insns, zeros, sizeof(zeros)),
 594                           !!memcmp(xlated_insns, zeros, sizeof(zeros)),
 595                           load_time, now,
 596                           prog_infos[i].created_by_uid, my_uid,
 597                           prog_infos[i].nr_map_ids, 1,
 598                           *(int *)(long)prog_infos[i].map_ids, map_infos[i].id,
 599                           prog_infos[i].name, expected_prog_name))
 600                         goto done;
 601         }
 602
 603         /* Check bpf_prog_get_next_id() */
 604         nr_id_found = 0;
 605         next_id = 0;
 606         while (!bpf_prog_get_next_id(next_id, &next_id)) {
 607                 struct bpf_prog_info prog_info = {};
 608                 __u32 saved_map_id;
 609                 int prog_fd;
 610
 611                 info_len = sizeof(prog_info);
 612
 613                 prog_fd = bpf_prog_get_fd_by_id(next_id);
 614                 if (prog_fd < 0 && errno == ENOENT)
 615                         /* The bpf_prog is in the dead row */
 616                         continue;
 617                 if (CHECK(prog_fd < 0, "get-prog-fd(next_id)",
 618                           "prog_fd %d next_id %d errno %d\n",
 619                           prog_fd, next_id, errno))
 620                         break;
 621
 622                 for (i = 0; i < nr_iters; i++)
 623                         if (prog_infos[i].id == next_id)
 624                                 break;
 625
 626                 if (i == nr_iters)
 627                         continue;
 628
 629                 nr_id_found++;
 630
 631                 /* Negative test:
 632                  * prog_info.nr_map_ids = 1
 633                  * prog_info.map_ids = NULL
 634                  */
 635                 prog_info.nr_map_ids = 1;
 636                 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
 637                 if (CHECK(!err || errno != EFAULT,
 638                           "get-prog-fd-bad-nr-map-ids", "err %d errno %d(%d)",
 639                           err, errno, EFAULT))
 640                         break;
 641                 bzero(&prog_info, sizeof(prog_info));
 642                 info_len = sizeof(prog_info);
 643
 644                 saved_map_id = *(int *)((long)prog_infos[i].map_ids);
 645                 prog_info.map_ids = prog_infos[i].map_ids;
 646                 prog_info.nr_map_ids = 2;
 647                 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
 648                 prog_infos[i].jited_prog_insns = 0;
 649                 prog_infos[i].xlated_prog_insns = 0;
 650                 CHECK(err || info_len != sizeof(struct bpf_prog_info) ||
 651                       memcmp(&prog_info, &prog_infos[i], info_len) ||
 652                       *(int *)(long)prog_info.map_ids != saved_map_id,
 653                       "get-prog-info(next_id->fd)",
 654                       "err %d errno %d info_len %u(%Zu) memcmp %d map_id %u(%u)\n",
 655                       err, errno, info_len, sizeof(struct bpf_prog_info),
 656                       memcmp(&prog_info, &prog_infos[i], info_len),
 657                       *(int *)(long)prog_info.map_ids, saved_map_id);
 658                 close(prog_fd);
 659         }
 660         CHECK(nr_id_found != nr_iters,
 661               "check total prog id found by get_next_id",
 662               "nr_id_found %u(%u)\n",
 663               nr_id_found, nr_iters);
 664
 665         /* Check bpf_map_get_next_id() */
 666         nr_id_found = 0;
 667         next_id = 0;
 668         while (!bpf_map_get_next_id(next_id, &next_id)) {
 669                 struct bpf_map_info map_info = {};
 670                 int map_fd;
 671
 672                 info_len = sizeof(map_info);
 673
 674                 map_fd = bpf_map_get_fd_by_id(next_id);
 675                 if (map_fd < 0 && errno == ENOENT)
 676                         /* The bpf_map is in the dead row */
 677                         continue;
 678                 if (CHECK(map_fd < 0, "get-map-fd(next_id)",
 679                           "map_fd %d next_id %u errno %d\n",
 680                           map_fd, next_id, errno))
 681                         break;
 682
 683                 for (i = 0; i < nr_iters; i++)
 684                         if (map_infos[i].id == next_id)
 685                                 break;
 686
 687                 if (i == nr_iters)
 688                         continue;
 689
 690                 nr_id_found++;
 691
 692                 err = bpf_map_lookup_elem(map_fd, &array_key, &array_value);
 693                 assert(!err);
 694
 695                 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &info_len);
 696                 CHECK(err || info_len != sizeof(struct bpf_map_info) ||
 697                       memcmp(&map_info, &map_infos[i], info_len) ||
 698                       array_value != array_magic_value,
 699                       "check get-map-info(next_id->fd)",
 700                       "err %d errno %d info_len %u(%Zu) memcmp %d array_value %llu(%llu)\n",
 701                       err, errno, info_len, sizeof(struct bpf_map_info),
 702                       memcmp(&map_info, &map_infos[i], info_len),
 703                       array_value, array_magic_value);
 704
 705                 close(map_fd);
 706         }
 707         CHECK(nr_id_found != nr_iters,
 708               "check total map id found by get_next_id",
 709               "nr_id_found %u(%u)\n",
 710               nr_id_found, nr_iters);
 711
 712 done:
 713         for (i = 0; i < nr_iters; i++)
 714                 bpf_object__close(objs[i]);
 715 }
 716
 717 static void test_pkt_md_access(void)
 718 {
 719         const char *file = "./test_pkt_md_access.o";
 720         struct bpf_object *obj;
 721         __u32 duration, retval;
 722         int err, prog_fd;
 723
 724         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 725         if (err) {
 726                 error_cnt++;
 727                 return;
 728         }
 729
 730         err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4),
 731                                 NULL, NULL, &retval, &duration);
 732         CHECK(err || retval, "",
 733               "err %d errno %d retval %d duration %d\n",
 734               err, errno, retval, duration);
 735
 736         bpf_object__close(obj);
 737 }
 738
 739 static void test_obj_name(void)
 740 {
 741         struct {
 742                 const char *name;
 743                 int success;
 744                 int expected_errno;
 745         } tests[] = {
 746                 { "", 1, 0 },
 747                 { "_123456789ABCDE", 1, 0 },
 748                 { "_123456789ABCDEF", 0, EINVAL },
 749                 { "_123456789ABCD\n", 0, EINVAL },
 750         };
 751         struct bpf_insn prog[] = {
 752                 BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0),
 753                 BPF_EXIT_INSN(),
 754         };
 755         __u32 duration = 0;
 756         int i;
 757
 758         for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) {
 759                 size_t name_len = strlen(tests[i].name) + 1;
 760                 union bpf_attr attr;
 761                 size_t ncopy;
 762                 int fd;
 763
 764                 /* test different attr.prog_name during BPF_PROG_LOAD */
 765                 ncopy = name_len < sizeof(attr.prog_name) ?
 766                         name_len : sizeof(attr.prog_name);
 767                 bzero(&attr, sizeof(attr));
 768                 attr.prog_type = BPF_PROG_TYPE_SCHED_CLS;
 769                 attr.insn_cnt = 2;
 770                 attr.insns = ptr_to_u64(prog);
 771                 attr.license = ptr_to_u64("");
 772                 memcpy(attr.prog_name, tests[i].name, ncopy);
 773
 774                 fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr));
 775                 CHECK((tests[i].success && fd < 0) ||
 776                       (!tests[i].success && fd != -1) ||
 777                       (!tests[i].success && errno != tests[i].expected_errno),
 778                       "check-bpf-prog-name",
 779                       "fd %d(%d) errno %d(%d)\n",
 780                        fd, tests[i].success, errno, tests[i].expected_errno);
 781
 782                 if (fd != -1)
 783                         close(fd);
 784
 785                 /* test different attr.map_name during BPF_MAP_CREATE */
 786                 ncopy = name_len < sizeof(attr.map_name) ?
 787                         name_len : sizeof(attr.map_name);
 788                 bzero(&attr, sizeof(attr));
 789                 attr.map_type = BPF_MAP_TYPE_ARRAY;
 790                 attr.key_size = 4;
 791                 attr.value_size = 4;
 792                 attr.max_entries = 1;
 793                 attr.map_flags = 0;
 794                 memcpy(attr.map_name, tests[i].name, ncopy);
 795                 fd = syscall(__NR_bpf, BPF_MAP_CREATE, &attr, sizeof(attr));
 796                 CHECK((tests[i].success && fd < 0) ||
 797                       (!tests[i].success && fd != -1) ||
 798                       (!tests[i].success && errno != tests[i].expected_errno),
 799                       "check-bpf-map-name",
 800                       "fd %d(%d) errno %d(%d)\n",
 801                       fd, tests[i].success, errno, tests[i].expected_errno);
 802
 803                 if (fd != -1)
 804                         close(fd);
 805         }
 806 }
 807
 808 static void test_tp_attach_query(void)
 809 {
 810         const int num_progs = 3;
 811         int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs];
 812         __u32 duration = 0, info_len, saved_prog_ids[num_progs];
 813         const char *file = "./test_tracepoint.o";
 814         struct perf_event_query_bpf *query;
 815         struct perf_event_attr attr = {};
 816         struct bpf_object *obj[num_progs];
 817         struct bpf_prog_info prog_info;
 818         char buf[256];
 819
 820         snprintf(buf, sizeof(buf),
 821                  "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
 822         efd = open(buf, O_RDONLY, 0);
 823         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
 824                 return;
 825         bytes = read(efd, buf, sizeof(buf));
 826         close(efd);
 827         if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
 828                   "read", "bytes %d errno %d\n", bytes, errno))
 829                 return;
 830
 831         attr.config = strtol(buf, NULL, 0);
 832         attr.type = PERF_TYPE_TRACEPOINT;
 833         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
 834         attr.sample_period = 1;
 835         attr.wakeup_events = 1;
 836
 837         query = malloc(sizeof(*query) + sizeof(__u32) * num_progs);
 838         for (i = 0; i < num_progs; i++) {
 839                 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj[i],
 840                                     &prog_fd[i]);
 841                 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
 842                         goto cleanup1;
 843
 844                 bzero(&prog_info, sizeof(prog_info));
 845                 prog_info.jited_prog_len = 0;
 846                 prog_info.xlated_prog_len = 0;
 847                 prog_info.nr_map_ids = 0;
 848                 info_len = sizeof(prog_info);
 849                 err = bpf_obj_get_info_by_fd(prog_fd[i], &prog_info, &info_len);
 850                 if (CHECK(err, "bpf_obj_get_info_by_fd", "err %d errno %d\n",
 851                           err, errno))
 852                         goto cleanup1;
 853                 saved_prog_ids[i] = prog_info.id;
 854
 855                 pmu_fd[i] = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
 856                                     0 /* cpu 0 */, -1 /* group id */,
 857                                     0 /* flags */);
 858                 if (CHECK(pmu_fd[i] < 0, "perf_event_open", "err %d errno %d\n",
 859                           pmu_fd[i], errno))
 860                         goto cleanup2;
 861                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_ENABLE, 0);
 862                 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
 863                           err, errno))
 864                         goto cleanup3;
 865
 866                 if (i == 0) {
 867                         /* check NULL prog array query */
 868                         query->ids_len = num_progs;
 869                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 870                         if (CHECK(err || query->prog_cnt != 0,
 871                                   "perf_event_ioc_query_bpf",
 872                                   "err %d errno %d query->prog_cnt %u\n",
 873                                   err, errno, query->prog_cnt))
 874                                 goto cleanup3;
 875                 }
 876
 877                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_SET_BPF, prog_fd[i]);
 878                 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
 879                           err, errno))
 880                         goto cleanup3;
 881
 882                 if (i == 1) {
 883                         /* try to get # of programs only */
 884                         query->ids_len = 0;
 885                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 886                         if (CHECK(err || query->prog_cnt != 2,
 887                                   "perf_event_ioc_query_bpf",
 888                                   "err %d errno %d query->prog_cnt %u\n",
 889                                   err, errno, query->prog_cnt))
 890                                 goto cleanup3;
 891
 892                         /* try a few negative tests */
 893                         /* invalid query pointer */
 894                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF,
 895                                     (struct perf_event_query_bpf *)0x1);
 896                         if (CHECK(!err || errno != EFAULT,
 897                                   "perf_event_ioc_query_bpf",
 898                                   "err %d errno %d\n", err, errno))
 899                                 goto cleanup3;
 900
 901                         /* no enough space */
 902                         query->ids_len = 1;
 903                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 904                         if (CHECK(!err || errno != ENOSPC || query->prog_cnt != 2,
 905                                   "perf_event_ioc_query_bpf",
 906                                   "err %d errno %d query->prog_cnt %u\n",
 907                                   err, errno, query->prog_cnt))
 908                                 goto cleanup3;
 909                 }
 910
 911                 query->ids_len = num_progs;
 912                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 913                 if (CHECK(err || query->prog_cnt != (i + 1),
 914                           "perf_event_ioc_query_bpf",
 915                           "err %d errno %d query->prog_cnt %u\n",
 916                           err, errno, query->prog_cnt))
 917                         goto cleanup3;
 918                 for (j = 0; j < i + 1; j++)
 919                         if (CHECK(saved_prog_ids[j] != query->ids[j],
 920                                   "perf_event_ioc_query_bpf",
 921                                   "#%d saved_prog_id %x query prog_id %x\n",
 922                                   j, saved_prog_ids[j], query->ids[j]))
 923                                 goto cleanup3;
 924         }
 925
 926         i = num_progs - 1;
 927         for (; i >= 0; i--) {
 928  cleanup3:
 929                 ioctl(pmu_fd[i], PERF_EVENT_IOC_DISABLE);
 930  cleanup2:
 931                 close(pmu_fd[i]);
 932  cleanup1:
 933                 bpf_object__close(obj[i]);
 934         }
 935         free(query);
 936 }
 937
 938 static int compare_map_keys(int map1_fd, int map2_fd)
 939 {
 940         __u32 key, next_key;
 941         char val_buf[PERF_MAX_STACK_DEPTH *
 942                      sizeof(struct bpf_stack_build_id)];
 943         int err;
 944
 945         err = bpf_map_get_next_key(map1_fd, NULL, &key);
 946         if (err)
 947                 return err;
 948         err = bpf_map_lookup_elem(map2_fd, &key, val_buf);
 949         if (err)
 950                 return err;
 951
 952         while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) {
 953                 err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf);
 954                 if (err)
 955                         return err;
 956
 957                 key = next_key;
 958         }
 959         if (errno != ENOENT)
 960                 return -1;
 961
 962         return 0;
 963 }
 964
 965 static int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len)
 966 {
 967         __u32 key, next_key, *cur_key_p, *next_key_p;
 968         char *val_buf1, *val_buf2;
 969         int i, err = 0;
 970
 971         val_buf1 = malloc(stack_trace_len);
 972         val_buf2 = malloc(stack_trace_len);
 973         cur_key_p = NULL;
 974         next_key_p = &key;
 975         while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) {
 976                 err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1);
 977                 if (err)
 978                         goto out;
 979                 err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2);
 980                 if (err)
 981                         goto out;
 982                 for (i = 0; i < stack_trace_len; i++) {
 983                         if (val_buf1[i] != val_buf2[i]) {
 984                                 err = -1;
 985                                 goto out;
 986                         }
 987                 }
 988                 key = *next_key_p;
 989                 cur_key_p = &key;
 990                 next_key_p = &next_key;
 991         }
 992         if (errno != ENOENT)
 993                 err = -1;
 994
 995 out:
 996         free(val_buf1);
 997         free(val_buf2);
 998         return err;
 999 }
1000
1001 static void test_stacktrace_map()
1002 {
1003         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1004         const char *file = "./test_stacktrace_map.o";
1005         int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1006         struct perf_event_attr attr = {};
1007         __u32 key, val, duration = 0;
1008         struct bpf_object *obj;
1009         char buf[256];
1010
1011         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1012         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1013                 return;
1014
1015         /* Get the ID for the sched/sched_switch tracepoint */
1016         snprintf(buf, sizeof(buf),
1017                  "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
1018         efd = open(buf, O_RDONLY, 0);
1019         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1020                 goto close_prog;
1021
1022         bytes = read(efd, buf, sizeof(buf));
1023         close(efd);
1024         if (bytes <= 0 || bytes >= sizeof(buf))
1025                 goto close_prog;
1026
1027         /* Open the perf event and attach bpf progrram */
1028         attr.config = strtol(buf, NULL, 0);
1029         attr.type = PERF_TYPE_TRACEPOINT;
1030         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
1031         attr.sample_period = 1;
1032         attr.wakeup_events = 1;
1033         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1034                          0 /* cpu 0 */, -1 /* group id */,
1035                          0 /* flags */);
1036         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
1037                   pmu_fd, errno))
1038                 goto close_prog;
1039
1040         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1041         if (err)
1042                 goto disable_pmu;
1043
1044         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1045         if (err)
1046                 goto disable_pmu;
1047
1048         /* find map fds */
1049         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1050         if (control_map_fd < 0)
1051                 goto disable_pmu;
1052
1053         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1054         if (stackid_hmap_fd < 0)
1055                 goto disable_pmu;
1056
1057         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1058         if (stackmap_fd < 0)
1059                 goto disable_pmu;
1060
1061         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1062         if (stack_amap_fd < 0)
1063                 goto disable_pmu;
1064
1065         /* give some time for bpf program run */
1066         sleep(1);
1067
1068         /* disable stack trace collection */
1069         key = 0;
1070         val = 1;
1071         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1072
1073         /* for every element in stackid_hmap, we can find a corresponding one
1074          * in stackmap, and vise versa.
1075          */
1076         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1077         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1078                   "err %d errno %d\n", err, errno))
1079                 goto disable_pmu_noerr;
1080
1081         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1082         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1083                   "err %d errno %d\n", err, errno))
1084                 goto disable_pmu_noerr;
1085
1086         stack_trace_len = PERF_MAX_STACK_DEPTH * sizeof(__u64);
1087         err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1088         if (CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1089                   "err %d errno %d\n", err, errno))
1090                 goto disable_pmu_noerr;
1091
1092         goto disable_pmu_noerr;
1093 disable_pmu:
1094         error_cnt++;
1095 disable_pmu_noerr:
1096         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1097         close(pmu_fd);
1098 close_prog:
1099         bpf_object__close(obj);
1100 }
1101
1102 static void test_stacktrace_map_raw_tp()
1103 {
1104         int control_map_fd, stackid_hmap_fd, stackmap_fd;
1105         const char *file = "./test_stacktrace_map.o";
1106         int efd, err, prog_fd;
1107         __u32 key, val, duration = 0;
1108         struct bpf_object *obj;
1109
1110         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1111         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1112                 return;
1113
1114         efd = bpf_raw_tracepoint_open("sched_switch", prog_fd);
1115         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1116                 goto close_prog;
1117
1118         /* find map fds */
1119         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1120         if (control_map_fd < 0)
1121                 goto close_prog;
1122
1123         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1124         if (stackid_hmap_fd < 0)
1125                 goto close_prog;
1126
1127         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1128         if (stackmap_fd < 0)
1129                 goto close_prog;
1130
1131         /* give some time for bpf program run */
1132         sleep(1);
1133
1134         /* disable stack trace collection */
1135         key = 0;
1136         val = 1;
1137         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1138
1139         /* for every element in stackid_hmap, we can find a corresponding one
1140          * in stackmap, and vise versa.
1141          */
1142         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1143         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1144                   "err %d errno %d\n", err, errno))
1145                 goto close_prog;
1146
1147         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1148         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1149                   "err %d errno %d\n", err, errno))
1150                 goto close_prog;
1151
1152         goto close_prog_noerr;
1153 close_prog:
1154         error_cnt++;
1155 close_prog_noerr:
1156         bpf_object__close(obj);
1157 }
1158
1159 static int extract_build_id(char *build_id, size_t size)
1160 {
1161         FILE *fp;
1162         char *line = NULL;
1163         size_t len = 0;
1164
1165         fp = popen("readelf -n ./urandom_read | grep 'Build ID'", "r");
1166         if (fp == NULL)
1167                 return -1;
1168
1169         if (getline(&line, &len, fp) == -1)
1170                 goto err;
1171         fclose(fp);
1172
1173         if (len > size)
1174                 len = size;
1175         memcpy(build_id, line, len);
1176         build_id[len] = '\0';
1177         return 0;
1178 err:
1179         fclose(fp);
1180         return -1;
1181 }
1182
1183 static void test_stacktrace_build_id(void)
1184 {
1185         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1186         const char *file = "./test_stacktrace_build_id.o";
1187         int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1188         struct perf_event_attr attr = {};
1189         __u32 key, previous_key, val, duration = 0;
1190         struct bpf_object *obj;
1191         char buf[256];
1192         int i, j;
1193         struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
1194         int build_id_matches = 0;
1195         int retry = 1;
1196
1197 retry:
1198         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1199         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1200                 goto out;
1201
1202         /* Get the ID for the sched/sched_switch tracepoint */
1203         snprintf(buf, sizeof(buf),
1204                  "/sys/kernel/debug/tracing/events/random/urandom_read/id");
1205         efd = open(buf, O_RDONLY, 0);
1206         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1207                 goto close_prog;
1208
1209         bytes = read(efd, buf, sizeof(buf));
1210         close(efd);
1211         if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
1212                   "read", "bytes %d errno %d\n", bytes, errno))
1213                 goto close_prog;
1214
1215         /* Open the perf event and attach bpf progrram */
1216         attr.config = strtol(buf, NULL, 0);
1217         attr.type = PERF_TYPE_TRACEPOINT;
1218         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
1219         attr.sample_period = 1;
1220         attr.wakeup_events = 1;
1221         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1222                          0 /* cpu 0 */, -1 /* group id */,
1223                          0 /* flags */);
1224         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
1225                   pmu_fd, errno))
1226                 goto close_prog;
1227
1228         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1229         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
1230                   err, errno))
1231                 goto close_pmu;
1232
1233         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1234         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
1235                   err, errno))
1236                 goto disable_pmu;
1237
1238         /* find map fds */
1239         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1240         if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
1241                   "err %d errno %d\n", err, errno))
1242                 goto disable_pmu;
1243
1244         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1245         if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
1246                   "err %d errno %d\n", err, errno))
1247                 goto disable_pmu;
1248
1249         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1250         if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
1251                   err, errno))
1252                 goto disable_pmu;
1253
1254         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1255         if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1256                   "err %d errno %d\n", err, errno))
1257                 goto disable_pmu;
1258
1259         assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1260                == 0);
1261         assert(system("./urandom_read") == 0);
1262         /* disable stack trace collection */
1263         key = 0;
1264         val = 1;
1265         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1266
1267         /* for every element in stackid_hmap, we can find a corresponding one
1268          * in stackmap, and vise versa.
1269          */
1270         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1271         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1272                   "err %d errno %d\n", err, errno))
1273                 goto disable_pmu;
1274
1275         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1276         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1277                   "err %d errno %d\n", err, errno))
1278                 goto disable_pmu;
1279
1280         err = extract_build_id(buf, 256);
1281
1282         if (CHECK(err, "get build_id with readelf",
1283                   "err %d errno %d\n", err, errno))
1284                 goto disable_pmu;
1285
1286         err = bpf_map_get_next_key(stackmap_fd, NULL, &key);
1287         if (CHECK(err, "get_next_key from stackmap",
1288                   "err %d, errno %d\n", err, errno))
1289                 goto disable_pmu;
1290
1291         do {
1292                 char build_id[64];
1293
1294                 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs);
1295                 if (CHECK(err, "lookup_elem from stackmap",
1296                           "err %d, errno %d\n", err, errno))
1297                         goto disable_pmu;
1298                 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i)
1299                         if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID &&
1300                             id_offs[i].offset != 0) {
1301                                 for (j = 0; j < 20; ++j)
1302                                         sprintf(build_id + 2 * j, "%02x",
1303                                                 id_offs[i].build_id[j] & 0xff);
1304                                 if (strstr(buf, build_id) != NULL)
1305                                         build_id_matches = 1;
1306                         }
1307                 previous_key = key;
1308         } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1309
1310         /* stack_map_get_build_id_offset() is racy and sometimes can return
1311          * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID;
1312          * try it one more time.
1313          */
1314         if (build_id_matches < 1 && retry--) {
1315                 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1316                 close(pmu_fd);
1317                 bpf_object__close(obj);
1318                 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
1319                        __func__);
1320                 goto retry;
1321         }
1322
1323         if (CHECK(build_id_matches < 1, "build id match",
1324                   "Didn't find expected build ID from the map\n"))
1325                 goto disable_pmu;
1326
1327         stack_trace_len = PERF_MAX_STACK_DEPTH
1328                 * sizeof(struct bpf_stack_build_id);
1329         err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1330         CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1331               "err %d errno %d\n", err, errno);
1332
1333 disable_pmu:
1334         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1335
1336 close_pmu:
1337         close(pmu_fd);
1338
1339 close_prog:
1340         bpf_object__close(obj);
1341
1342 out:
1343         return;
1344 }
1345
1346 static void test_stacktrace_build_id_nmi(void)
1347 {
1348         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1349         const char *file = "./test_stacktrace_build_id.o";
1350         int err, pmu_fd, prog_fd;
1351         struct perf_event_attr attr = {
1352                 .sample_freq = 5000,
1353                 .freq = 1,
1354                 .type = PERF_TYPE_HARDWARE,
1355                 .config = PERF_COUNT_HW_CPU_CYCLES,
1356         };
1357         __u32 key, previous_key, val, duration = 0;
1358         struct bpf_object *obj;
1359         char buf[256];
1360         int i, j;
1361         struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
1362         int build_id_matches = 0;
1363         int retry = 1;
1364
1365 retry:
1366         err = bpf_prog_load(file, BPF_PROG_TYPE_PERF_EVENT, &obj, &prog_fd);
1367         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1368                 return;
1369
1370         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1371                          0 /* cpu 0 */, -1 /* group id */,
1372                          0 /* flags */);
1373         if (CHECK(pmu_fd < 0, "perf_event_open",
1374                   "err %d errno %d. Does the test host support PERF_COUNT_HW_CPU_CYCLES?\n",
1375                   pmu_fd, errno))
1376                 goto close_prog;
1377
1378         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1379         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
1380                   err, errno))
1381                 goto close_pmu;
1382
1383         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1384         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
1385                   err, errno))
1386                 goto disable_pmu;
1387
1388         /* find map fds */
1389         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1390         if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
1391                   "err %d errno %d\n", err, errno))
1392                 goto disable_pmu;
1393
1394         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1395         if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
1396                   "err %d errno %d\n", err, errno))
1397                 goto disable_pmu;
1398
1399         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1400         if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
1401                   err, errno))
1402                 goto disable_pmu;
1403
1404         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1405         if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1406                   "err %d errno %d\n", err, errno))
1407                 goto disable_pmu;
1408
1409         assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1410                == 0);
1411         assert(system("taskset 0x1 ./urandom_read 100000") == 0);
1412         /* disable stack trace collection */
1413         key = 0;
1414         val = 1;
1415         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1416
1417         /* for every element in stackid_hmap, we can find a corresponding one
1418          * in stackmap, and vise versa.
1419          */
1420         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1421         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1422                   "err %d errno %d\n", err, errno))
1423                 goto disable_pmu;
1424
1425         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1426         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1427                   "err %d errno %d\n", err, errno))
1428                 goto disable_pmu;
1429
1430         err = extract_build_id(buf, 256);
1431
1432         if (CHECK(err, "get build_id with readelf",
1433                   "err %d errno %d\n", err, errno))
1434                 goto disable_pmu;
1435
1436         err = bpf_map_get_next_key(stackmap_fd, NULL, &key);
1437         if (CHECK(err, "get_next_key from stackmap",
1438                   "err %d, errno %d\n", err, errno))
1439                 goto disable_pmu;
1440
1441         do {
1442                 char build_id[64];
1443
1444                 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs);
1445                 if (CHECK(err, "lookup_elem from stackmap",
1446                           "err %d, errno %d\n", err, errno))
1447                         goto disable_pmu;
1448                 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i)
1449                         if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID &&
1450                             id_offs[i].offset != 0) {
1451                                 for (j = 0; j < 20; ++j)
1452                                         sprintf(build_id + 2 * j, "%02x",
1453                                                 id_offs[i].build_id[j] & 0xff);
1454                                 if (strstr(buf, build_id) != NULL)
1455                                         build_id_matches = 1;
1456                         }
1457                 previous_key = key;
1458         } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1459
1460         /* stack_map_get_build_id_offset() is racy and sometimes can return
1461          * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID;
1462          * try it one more time.
1463          */
1464         if (build_id_matches < 1 && retry--) {
1465                 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1466                 close(pmu_fd);
1467                 bpf_object__close(obj);
1468                 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
1469                        __func__);
1470                 goto retry;
1471         }
1472
1473         if (CHECK(build_id_matches < 1, "build id match",
1474                   "Didn't find expected build ID from the map\n"))
1475                 goto disable_pmu;
1476
1477         /*
1478          * We intentionally skip compare_stack_ips(). This is because we
1479          * only support one in_nmi() ips-to-build_id translation per cpu
1480          * at any time, thus stack_amap here will always fallback to
1481          * BPF_STACK_BUILD_ID_IP;
1482          */
1483
1484 disable_pmu:
1485         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1486
1487 close_pmu:
1488         close(pmu_fd);
1489
1490 close_prog:
1491         bpf_object__close(obj);
1492 }
1493
1494 #define MAX_CNT_RAWTP   10ull
1495 #define MAX_STACK_RAWTP 100
1496 struct get_stack_trace_t {
1497         int pid;
1498         int kern_stack_size;
1499         int user_stack_size;
1500         int user_stack_buildid_size;
1501         __u64 kern_stack[MAX_STACK_RAWTP];
1502         __u64 user_stack[MAX_STACK_RAWTP];
1503         struct bpf_stack_build_id user_stack_buildid[MAX_STACK_RAWTP];
1504 };
1505
1506 static int get_stack_print_output(void *data, int size)
1507 {
1508         bool good_kern_stack = false, good_user_stack = false;
1509         const char *nonjit_func = "___bpf_prog_run";
1510         struct get_stack_trace_t *e = data;
1511         int i, num_stack;
1512         static __u64 cnt;
1513         struct ksym *ks;
1514
1515         cnt++;
1516
1517         if (size < sizeof(struct get_stack_trace_t)) {
1518                 __u64 *raw_data = data;
1519                 bool found = false;
1520
1521                 num_stack = size / sizeof(__u64);
1522                 /* If jit is enabled, we do not have a good way to
1523                  * verify the sanity of the kernel stack. So we
1524                  * just assume it is good if the stack is not empty.
1525                  * This could be improved in the future.
1526                  */
1527                 if (jit_enabled) {
1528                         found = num_stack > 0;
1529                 } else {
1530                         for (i = 0; i < num_stack; i++) {
1531                                 ks = ksym_search(raw_data[i]);
1532                                 if (strcmp(ks->name, nonjit_func) == 0) {
1533                                         found = true;
1534                                         break;
1535                                 }
1536                         }
1537                 }
1538                 if (found) {
1539                         good_kern_stack = true;
1540                         good_user_stack = true;
1541                 }
1542         } else {
1543                 num_stack = e->kern_stack_size / sizeof(__u64);
1544                 if (jit_enabled) {
1545                         good_kern_stack = num_stack > 0;
1546                 } else {
1547                         for (i = 0; i < num_stack; i++) {
1548                                 ks = ksym_search(e->kern_stack[i]);
1549                                 if (strcmp(ks->name, nonjit_func) == 0) {
1550                                         good_kern_stack = true;
1551                                         break;
1552                                 }
1553                         }
1554                 }
1555                 if (e->user_stack_size > 0 && e->user_stack_buildid_size > 0)
1556                         good_user_stack = true;
1557         }
1558         if (!good_kern_stack || !good_user_stack)
1559                 return LIBBPF_PERF_EVENT_ERROR;
1560
1561         if (cnt == MAX_CNT_RAWTP)
1562                 return LIBBPF_PERF_EVENT_DONE;
1563
1564         return LIBBPF_PERF_EVENT_CONT;
1565 }
1566
1567 static void test_get_stack_raw_tp(void)
1568 {
1569         const char *file = "./test_get_stack_rawtp.o";
1570         int i, efd, err, prog_fd, pmu_fd, perfmap_fd;
1571         struct perf_event_attr attr = {};
1572         struct timespec tv = {0, 10};
1573         __u32 key = 0, duration = 0;
1574         struct bpf_object *obj;
1575
1576         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1577         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1578                 return;
1579
1580         efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1581         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1582                 goto close_prog;
1583
1584         perfmap_fd = bpf_find_map(__func__, obj, "perfmap");
1585         if (CHECK(perfmap_fd < 0, "bpf_find_map", "err %d errno %d\n",
1586                   perfmap_fd, errno))
1587                 goto close_prog;
1588
1589         err = load_kallsyms();
1590         if (CHECK(err < 0, "load_kallsyms", "err %d errno %d\n", err, errno))
1591                 goto close_prog;
1592
1593         attr.sample_type = PERF_SAMPLE_RAW;
1594         attr.type = PERF_TYPE_SOFTWARE;
1595         attr.config = PERF_COUNT_SW_BPF_OUTPUT;
1596         pmu_fd = syscall(__NR_perf_event_open, &attr, getpid()/*pid*/, -1/*cpu*/,
1597                          -1/*group_fd*/, 0);
1598         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", pmu_fd,
1599                   errno))
1600                 goto close_prog;
1601
1602         err = bpf_map_update_elem(perfmap_fd, &key, &pmu_fd, BPF_ANY);
1603         if (CHECK(err < 0, "bpf_map_update_elem", "err %d errno %d\n", err,
1604                   errno))
1605                 goto close_prog;
1606
1607         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1608         if (CHECK(err < 0, "ioctl PERF_EVENT_IOC_ENABLE", "err %d errno %d\n",
1609                   err, errno))
1610                 goto close_prog;
1611
1612         err = perf_event_mmap(pmu_fd);
1613         if (CHECK(err < 0, "perf_event_mmap", "err %d errno %d\n", err, errno))
1614                 goto close_prog;
1615
1616         /* trigger some syscall action */
1617         for (i = 0; i < MAX_CNT_RAWTP; i++)
1618                 nanosleep(&tv, NULL);
1619
1620         err = perf_event_poller(pmu_fd, get_stack_print_output);
1621         if (CHECK(err < 0, "perf_event_poller", "err %d errno %d\n", err, errno))
1622                 goto close_prog;
1623
1624         goto close_prog_noerr;
1625 close_prog:
1626         error_cnt++;
1627 close_prog_noerr:
1628         bpf_object__close(obj);
1629 }
1630
1631 static void test_task_fd_query_rawtp(void)
1632 {
1633         const char *file = "./test_get_stack_rawtp.o";
1634         __u64 probe_offset, probe_addr;
1635         __u32 len, prog_id, fd_type;
1636         struct bpf_object *obj;
1637         int efd, err, prog_fd;
1638         __u32 duration = 0;
1639         char buf[256];
1640
1641         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1642         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1643                 return;
1644
1645         efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1646         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1647                 goto close_prog;
1648
1649         /* query (getpid(), efd) */
1650         len = sizeof(buf);
1651         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1652                                 &fd_type, &probe_offset, &probe_addr);
1653         if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err,
1654                   errno))
1655                 goto close_prog;
1656
1657         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1658               strcmp(buf, "sys_enter") == 0;
1659         if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n",
1660                   fd_type, buf))
1661                 goto close_prog;
1662
1663         /* test zero len */
1664         len = 0;
1665         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1666                                 &fd_type, &probe_offset, &probe_addr);
1667         if (CHECK(err < 0, "bpf_task_fd_query (len = 0)", "err %d errno %d\n",
1668                   err, errno))
1669                 goto close_prog;
1670         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1671               len == strlen("sys_enter");
1672         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1673                 goto close_prog;
1674
1675         /* test empty buffer */
1676         len = sizeof(buf);
1677         err = bpf_task_fd_query(getpid(), efd, 0, 0, &len, &prog_id,
1678                                 &fd_type, &probe_offset, &probe_addr);
1679         if (CHECK(err < 0, "bpf_task_fd_query (buf = 0)", "err %d errno %d\n",
1680                   err, errno))
1681                 goto close_prog;
1682         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1683               len == strlen("sys_enter");
1684         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1685                 goto close_prog;
1686
1687         /* test smaller buffer */
1688         len = 3;
1689         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1690                                 &fd_type, &probe_offset, &probe_addr);
1691         if (CHECK(err >= 0 || errno != ENOSPC, "bpf_task_fd_query (len = 3)",
1692                   "err %d errno %d\n", err, errno))
1693                 goto close_prog;
1694         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1695               len == strlen("sys_enter") &&
1696               strcmp(buf, "sy") == 0;
1697         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1698                 goto close_prog;
1699
1700         goto close_prog_noerr;
1701 close_prog:
1702         error_cnt++;
1703 close_prog_noerr:
1704         bpf_object__close(obj);
1705 }
1706
1707 static void test_task_fd_query_tp_core(const char *probe_name,
1708                                        const char *tp_name)
1709 {
1710         const char *file = "./test_tracepoint.o";
1711         int err, bytes, efd, prog_fd, pmu_fd;
1712         struct perf_event_attr attr = {};
1713         __u64 probe_offset, probe_addr;
1714         __u32 len, prog_id, fd_type;
1715         struct bpf_object *obj;
1716         __u32 duration = 0;
1717         char buf[256];
1718
1719         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1720         if (CHECK(err, "bpf_prog_load", "err %d errno %d\n", err, errno))
1721                 goto close_prog;
1722
1723         snprintf(buf, sizeof(buf),
1724                  "/sys/kernel/debug/tracing/events/%s/id", probe_name);
1725         efd = open(buf, O_RDONLY, 0);
1726         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1727                 goto close_prog;
1728         bytes = read(efd, buf, sizeof(buf));
1729         close(efd);
1730         if (CHECK(bytes <= 0 || bytes >= sizeof(buf), "read",
1731                   "bytes %d errno %d\n", bytes, errno))
1732                 goto close_prog;
1733
1734         attr.config = strtol(buf, NULL, 0);
1735         attr.type = PERF_TYPE_TRACEPOINT;
1736         attr.sample_type = PERF_SAMPLE_RAW;
1737         attr.sample_period = 1;
1738         attr.wakeup_events = 1;
1739         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1740                          0 /* cpu 0 */, -1 /* group id */,
1741                          0 /* flags */);
1742         if (CHECK(err, "perf_event_open", "err %d errno %d\n", err, errno))
1743                 goto close_pmu;
1744
1745         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1746         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", err,
1747                   errno))
1748                 goto close_pmu;
1749
1750         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1751         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", err,
1752                   errno))
1753                 goto close_pmu;
1754
1755         /* query (getpid(), pmu_fd) */
1756         len = sizeof(buf);
1757         err = bpf_task_fd_query(getpid(), pmu_fd, 0, buf, &len, &prog_id,
1758                                 &fd_type, &probe_offset, &probe_addr);
1759         if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err,
1760                   errno))
1761                 goto close_pmu;
1762
1763         err = (fd_type == BPF_FD_TYPE_TRACEPOINT) && !strcmp(buf, tp_name);
1764         if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n",
1765                   fd_type, buf))
1766                 goto close_pmu;
1767
1768         close(pmu_fd);
1769         goto close_prog_noerr;
1770
1771 close_pmu:
1772         close(pmu_fd);
1773 close_prog:
1774         error_cnt++;
1775 close_prog_noerr:
1776         bpf_object__close(obj);
1777 }
1778
1779 static void test_task_fd_query_tp(void)
1780 {
1781         test_task_fd_query_tp_core("sched/sched_switch",
1782                                    "sched_switch");
1783         test_task_fd_query_tp_core("syscalls/sys_enter_read",
1784                                    "sys_enter_read");
1785 }
1786
1787 static int libbpf_debug_print(enum libbpf_print_level level,
1788                               const char *format, va_list args)
1789 {
1790         if (level == LIBBPF_DEBUG)
1791                 return 0;
1792
1793         return vfprintf(stderr, format, args);
1794 }
1795
1796 static void test_reference_tracking()
1797 {
1798         const char *file = "./test_sk_lookup_kern.o";
1799         struct bpf_object *obj;
1800         struct bpf_program *prog;
1801         __u32 duration = 0;
1802         int err = 0;
1803
1804         obj = bpf_object__open(file);
1805         if (IS_ERR(obj)) {
1806                 error_cnt++;
1807                 return;
1808         }
1809
1810         bpf_object__for_each_program(prog, obj) {
1811                 const char *title;
1812
1813                 /* Ignore .text sections */
1814                 title = bpf_program__title(prog, false);
1815                 if (strstr(title, ".text") != NULL)
1816                         continue;
1817
1818                 bpf_program__set_type(prog, BPF_PROG_TYPE_SCHED_CLS);
1819
1820                 /* Expect verifier failure if test name has 'fail' */
1821                 if (strstr(title, "fail") != NULL) {
1822                         libbpf_set_print(NULL);
1823                         err = !bpf_program__load(prog, "GPL", 0);
1824                         libbpf_set_print(libbpf_debug_print);
1825                 } else {
1826                         err = bpf_program__load(prog, "GPL", 0);
1827                 }
1828                 CHECK(err, title, "\n");
1829         }
1830         bpf_object__close(obj);
1831 }
1832
1833 enum {
1834         QUEUE,
1835         STACK,
1836 };
1837
1838 static void test_queue_stack_map(int type)
1839 {
1840         const int MAP_SIZE = 32;
1841         __u32 vals[MAP_SIZE], duration, retval, size, val;
1842         int i, err, prog_fd, map_in_fd, map_out_fd;
1843         char file[32], buf[128];
1844         struct bpf_object *obj;
1845         struct iphdr *iph = (void *)buf + sizeof(struct ethhdr);
1846
1847         /* Fill test values to be used */
1848         for (i = 0; i < MAP_SIZE; i++)
1849                 vals[i] = rand();
1850
1851         if (type == QUEUE)
1852                 strncpy(file, "./test_queue_map.o", sizeof(file));
1853         else if (type == STACK)
1854                 strncpy(file, "./test_stack_map.o", sizeof(file));
1855         else
1856                 return;
1857
1858         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
1859         if (err) {
1860                 error_cnt++;
1861                 return;
1862         }
1863
1864         map_in_fd = bpf_find_map(__func__, obj, "map_in");
1865         if (map_in_fd < 0)
1866                 goto out;
1867
1868         map_out_fd = bpf_find_map(__func__, obj, "map_out");
1869         if (map_out_fd < 0)
1870                 goto out;
1871
1872         /* Push 32 elements to the input map */
1873         for (i = 0; i < MAP_SIZE; i++) {
1874                 err = bpf_map_update_elem(map_in_fd, NULL, &vals[i], 0);
1875                 if (err) {
1876                         error_cnt++;
1877                         goto out;
1878                 }
1879         }
1880
1881         /* The eBPF program pushes iph.saddr in the output map,
1882          * pops the input map and saves this value in iph.daddr
1883          */
1884         for (i = 0; i < MAP_SIZE; i++) {
1885                 if (type == QUEUE) {
1886                         val = vals[i];
1887                         pkt_v4.iph.saddr = vals[i] * 5;
1888                 } else if (type == STACK) {
1889                         val = vals[MAP_SIZE - 1 - i];
1890                         pkt_v4.iph.saddr = vals[MAP_SIZE - 1 - i] * 5;
1891                 }
1892
1893                 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
1894                                         buf, &size, &retval, &duration);
1895                 if (err || retval || size != sizeof(pkt_v4) ||
1896                     iph->daddr != val)
1897                         break;
1898         }
1899
1900         CHECK(err || retval || size != sizeof(pkt_v4) || iph->daddr != val,
1901               "bpf_map_pop_elem",
1902               "err %d errno %d retval %d size %d iph->daddr %u\n",
1903               err, errno, retval, size, iph->daddr);
1904
1905         /* Queue is empty, program should return TC_ACT_SHOT */
1906         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
1907                                 buf, &size, &retval, &duration);
1908         CHECK(err || retval != 2 /* TC_ACT_SHOT */|| size != sizeof(pkt_v4),
1909               "check-queue-stack-map-empty",
1910               "err %d errno %d retval %d size %d\n",
1911               err, errno, retval, size);
1912
1913         /* Check that the program pushed elements correctly */
1914         for (i = 0; i < MAP_SIZE; i++) {
1915                 err = bpf_map_lookup_and_delete_elem(map_out_fd, NULL, &val);
1916                 if (err || val != vals[i] * 5)
1917                         break;
1918         }
1919
1920         CHECK(i != MAP_SIZE && (err || val != vals[i] * 5),
1921               "bpf_map_push_elem", "err %d value %u\n", err, val);
1922
1923 out:
1924         pkt_v4.iph.saddr = 0;
1925         bpf_object__close(obj);
1926 }
1927
1928 #define CHECK_FLOW_KEYS(desc, got, expected)                            \
1929         CHECK(memcmp(&got, &expected, sizeof(got)) != 0,                \
1930               desc,                                                     \
1931               "nhoff=%u/%u "                                            \
1932               "thoff=%u/%u "                                            \
1933               "addr_proto=0x%x/0x%x "                                   \
1934               "is_frag=%u/%u "                                          \
1935               "is_first_frag=%u/%u "                                    \
1936               "is_encap=%u/%u "                                         \
1937               "n_proto=0x%x/0x%x "                                      \
1938               "sport=%u/%u "                                            \
1939               "dport=%u/%u\n",                                          \
1940               got.nhoff, expected.nhoff,                                \
1941               got.thoff, expected.thoff,                                \
1942               got.addr_proto, expected.addr_proto,                      \
1943               got.is_frag, expected.is_frag,                            \
1944               got.is_first_frag, expected.is_first_frag,                \
1945               got.is_encap, expected.is_encap,                          \
1946               got.n_proto, expected.n_proto,                            \
1947               got.sport, expected.sport,                                \
1948               got.dport, expected.dport)
1949
1950 static struct bpf_flow_keys pkt_v4_flow_keys = {
1951         .nhoff = 0,
1952         .thoff = sizeof(struct iphdr),
1953         .addr_proto = ETH_P_IP,
1954         .ip_proto = IPPROTO_TCP,
1955         .n_proto = bpf_htons(ETH_P_IP),
1956 };
1957
1958 static struct bpf_flow_keys pkt_v6_flow_keys = {
1959         .nhoff = 0,
1960         .thoff = sizeof(struct ipv6hdr),
1961         .addr_proto = ETH_P_IPV6,
1962         .ip_proto = IPPROTO_TCP,
1963         .n_proto = bpf_htons(ETH_P_IPV6),
1964 };
1965
1966 static void test_flow_dissector(void)
1967 {
1968         struct bpf_flow_keys flow_keys;
1969         struct bpf_object *obj;
1970         __u32 duration, retval;
1971         int err, prog_fd;
1972         __u32 size;
1973
1974         err = bpf_flow_load(&obj, "./bpf_flow.o", "flow_dissector",
1975                             "jmp_table", &prog_fd);
1976         if (err) {
1977                 error_cnt++;
1978                 return;
1979         }
1980
1981         err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4),
1982                                 &flow_keys, &size, &retval, &duration);
1983         CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv4",
1984               "err %d errno %d retval %d duration %d size %u/%lu\n",
1985               err, errno, retval, duration, size, sizeof(flow_keys));
1986         CHECK_FLOW_KEYS("ipv4_flow_keys", flow_keys, pkt_v4_flow_keys);
1987
1988         err = bpf_prog_test_run(prog_fd, 10, &pkt_v6, sizeof(pkt_v6),
1989                                 &flow_keys, &size, &retval, &duration);
1990         CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv6",
1991               "err %d errno %d retval %d duration %d size %u/%lu\n",
1992               err, errno, retval, duration, size, sizeof(flow_keys));
1993         CHECK_FLOW_KEYS("ipv6_flow_keys", flow_keys, pkt_v6_flow_keys);
1994
1995         bpf_object__close(obj);
1996 }
1997
1998 static void *test_spin_lock(void *arg)
1999 {
2000         __u32 duration, retval;
2001         int err, prog_fd = *(u32 *) arg;
2002
2003         err = bpf_prog_test_run(prog_fd, 10000, &pkt_v4, sizeof(pkt_v4),
2004                                 NULL, NULL, &retval, &duration);
2005         CHECK(err || retval, "",
2006               "err %d errno %d retval %d duration %d\n",
2007               err, errno, retval, duration);
2008         pthread_exit(arg);
2009 }
2010
2011 static void test_spinlock(void)
2012 {
2013         const char *file = "./test_spin_lock.o";
2014         pthread_t thread_id[4];
2015         struct bpf_object *obj;
2016         int prog_fd;
2017         int err = 0, i;
2018         void *ret;
2019
2020         err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd);
2021         if (err) {
2022                 printf("test_spin_lock:bpf_prog_load errno %d\n", errno);
2023                 goto close_prog;
2024         }
2025         for (i = 0; i < 4; i++)
2026                 assert(pthread_create(&thread_id[i], NULL,
2027                                       &test_spin_lock, &prog_fd) == 0);
2028         for (i = 0; i < 4; i++)
2029                 assert(pthread_join(thread_id[i], &ret) == 0 &&
2030                        ret == (void *)&prog_fd);
2031         goto close_prog_noerr;
2032 close_prog:
2033         error_cnt++;
2034 close_prog_noerr:
2035         bpf_object__close(obj);
2036 }
2037
2038 static void *parallel_map_access(void *arg)
2039 {
2040         int err, map_fd = *(u32 *) arg;
2041         int vars[17], i, j, rnd, key = 0;
2042
2043         for (i = 0; i < 10000; i++) {
2044                 err = bpf_map_lookup_elem_flags(map_fd, &key, vars, BPF_F_LOCK);
2045                 if (err) {
2046                         printf("lookup failed\n");
2047                         error_cnt++;
2048                         goto out;
2049                 }
2050                 if (vars[0] != 0) {
2051                         printf("lookup #%d var[0]=%d\n", i, vars[0]);
2052                         error_cnt++;
2053                         goto out;
2054                 }
2055                 rnd = vars[1];
2056                 for (j = 2; j < 17; j++) {
2057                         if (vars[j] == rnd)
2058                                 continue;
2059                         printf("lookup #%d var[1]=%d var[%d]=%d\n",
2060                                i, rnd, j, vars[j]);
2061                         error_cnt++;
2062                         goto out;
2063                 }
2064         }
2065 out:
2066         pthread_exit(arg);
2067 }
2068
2069 static void test_map_lock(void)
2070 {
2071         const char *file = "./test_map_lock.o";
2072         int prog_fd, map_fd[2], vars[17] = {};
2073         pthread_t thread_id[6];
2074         struct bpf_object *obj;
2075         int err = 0, key = 0, i;
2076         void *ret;
2077
2078         err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd);
2079         if (err) {
2080                 printf("test_map_lock:bpf_prog_load errno %d\n", errno);
2081                 goto close_prog;
2082         }
2083         map_fd[0] = bpf_find_map(__func__, obj, "hash_map");
2084         if (map_fd[0] < 0)
2085                 goto close_prog;
2086         map_fd[1] = bpf_find_map(__func__, obj, "array_map");
2087         if (map_fd[1] < 0)
2088                 goto close_prog;
2089
2090         bpf_map_update_elem(map_fd[0], &key, vars, BPF_F_LOCK);
2091
2092         for (i = 0; i < 4; i++)
2093                 assert(pthread_create(&thread_id[i], NULL,
2094                                       &test_spin_lock, &prog_fd) == 0);
2095         for (i = 4; i < 6; i++)
2096                 assert(pthread_create(&thread_id[i], NULL,
2097                                       &parallel_map_access, &map_fd[i - 4]) == 0);
2098         for (i = 0; i < 4; i++)
2099                 assert(pthread_join(thread_id[i], &ret) == 0 &&
2100                        ret == (void *)&prog_fd);
2101         for (i = 4; i < 6; i++)
2102                 assert(pthread_join(thread_id[i], &ret) == 0 &&
2103                        ret == (void *)&map_fd[i - 4]);
2104         goto close_prog_noerr;
2105 close_prog:
2106         error_cnt++;
2107 close_prog_noerr:
2108         bpf_object__close(obj);
2109 }
2110
2111 int main(void)
2112 {
2113         srand(time(NULL));
2114
2115         jit_enabled = is_jit_enabled();
2116
2117         test_pkt_access();
2118         test_prog_run_xattr();
2119         test_xdp();
2120         test_xdp_adjust_tail();
2121         test_l4lb_all();
2122         test_xdp_noinline();
2123         test_tcp_estats();
2124         test_bpf_obj_id();
2125         test_pkt_md_access();
2126         test_obj_name();
2127         test_tp_attach_query();
2128         test_stacktrace_map();
2129         test_stacktrace_build_id();
2130         test_stacktrace_build_id_nmi();
2131         test_stacktrace_map_raw_tp();
2132         test_get_stack_raw_tp();
2133         test_task_fd_query_rawtp();
2134         test_task_fd_query_tp();
2135         test_reference_tracking();
2136         test_queue_stack_map(QUEUE);
2137         test_queue_stack_map(STACK);
2138         test_flow_dissector();
2139         test_spinlock();
2140         test_map_lock();
2141
2142         printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt);
2143         return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
2144 }