tools/testing/selftests/bpf/test_progs.c

   1 /* Copyright (c) 2017 Facebook
   2  *
   3  * This program is free software; you can redistribute it and/or
   4  * modify it under the terms of version 2 of the GNU General Public
   5  * License as published by the Free Software Foundation.
   6  */
   7 #include <stdio.h>
   8 #include <unistd.h>
   9 #include <errno.h>
  10 #include <string.h>
  11 #include <assert.h>
  12 #include <stdlib.h>
  13 #include <time.h>
  14
  15 #include <linux/types.h>
  16 typedef __u16 __sum16;
  17 #include <arpa/inet.h>
  18 #include <linux/if_ether.h>
  19 #include <linux/if_packet.h>
  20 #include <linux/ip.h>
  21 #include <linux/ipv6.h>
  22 #include <linux/tcp.h>
  23 #include <linux/filter.h>
  24 #include <linux/perf_event.h>
  25 #include <linux/unistd.h>
  26
  27 #include <sys/ioctl.h>
  28 #include <sys/wait.h>
  29 #include <sys/types.h>
  30 #include <fcntl.h>
  31 #include <pthread.h>
  32 #include <linux/bpf.h>
  33 #include <linux/err.h>
  34 #include <bpf/bpf.h>
  35 #include <bpf/libbpf.h>
  36
  37 #include "test_iptunnel_common.h"
  38 #include "bpf_util.h"
  39 #include "bpf_endian.h"
  40 #include "bpf_rlimit.h"
  41 #include "trace_helpers.h"
  42 #include "flow_dissector_load.h"
  43
  44 static int error_cnt, pass_cnt;
  45 static bool jit_enabled;
  46
  47 #define MAGIC_BYTES 123
  48
  49 /* ipv4 test vector */
  50 static struct {
  51         struct ethhdr eth;
  52         struct iphdr iph;
  53         struct tcphdr tcp;
  54 } __packed pkt_v4 = {
  55         .eth.h_proto = __bpf_constant_htons(ETH_P_IP),
  56         .iph.ihl = 5,
  57         .iph.protocol = IPPROTO_TCP,
  58         .iph.tot_len = __bpf_constant_htons(MAGIC_BYTES),
  59         .tcp.urg_ptr = 123,
  60         .tcp.doff = 5,
  61 };
  62
  63 /* ipv6 test vector */
  64 static struct {
  65         struct ethhdr eth;
  66         struct ipv6hdr iph;
  67         struct tcphdr tcp;
  68 } __packed pkt_v6 = {
  69         .eth.h_proto = __bpf_constant_htons(ETH_P_IPV6),
  70         .iph.nexthdr = IPPROTO_TCP,
  71         .iph.payload_len = __bpf_constant_htons(MAGIC_BYTES),
  72         .tcp.urg_ptr = 123,
  73         .tcp.doff = 5,
  74 };
  75
  76 #define _CHECK(condition, tag, duration, format...) ({                  \
  77         int __ret = !!(condition);                                      \
  78         if (__ret) {                                                    \
  79                 error_cnt++;                                            \
  80                 printf("%s:FAIL:%s ", __func__, tag);                   \
  81                 printf(format);                                         \
  82         } else {                                                        \
  83                 pass_cnt++;                                             \
  84                 printf("%s:PASS:%s %d nsec\n", __func__, tag, duration);\
  85         }                                                               \
  86         __ret;                                                          \
  87 })
  88
  89 #define CHECK(condition, tag, format...) \
  90         _CHECK(condition, tag, duration, format)
  91 #define CHECK_ATTR(condition, tag, format...) \
  92         _CHECK(condition, tag, tattr.duration, format)
  93
  94 static int bpf_find_map(const char *test, struct bpf_object *obj,
  95                         const char *name)
  96 {
  97         struct bpf_map *map;
  98
  99         map = bpf_object__find_map_by_name(obj, name);
 100         if (!map) {
 101                 printf("%s:FAIL:map '%s' not found\n", test, name);
 102                 error_cnt++;
 103                 return -1;
 104         }
 105         return bpf_map__fd(map);
 106 }
 107
 108 static void test_pkt_access(void)
 109 {
 110         const char *file = "./test_pkt_access.o";
 111         struct bpf_object *obj;
 112         __u32 duration, retval;
 113         int err, prog_fd;
 114
 115         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 116         if (err) {
 117                 error_cnt++;
 118                 return;
 119         }
 120
 121         err = bpf_prog_test_run(prog_fd, 100000, &pkt_v4, sizeof(pkt_v4),
 122                                 NULL, NULL, &retval, &duration);
 123         CHECK(err || retval, "ipv4",
 124               "err %d errno %d retval %d duration %d\n",
 125               err, errno, retval, duration);
 126
 127         err = bpf_prog_test_run(prog_fd, 100000, &pkt_v6, sizeof(pkt_v6),
 128                                 NULL, NULL, &retval, &duration);
 129         CHECK(err || retval, "ipv6",
 130               "err %d errno %d retval %d duration %d\n",
 131               err, errno, retval, duration);
 132         bpf_object__close(obj);
 133 }
 134
 135 static void test_prog_run_xattr(void)
 136 {
 137         const char *file = "./test_pkt_access.o";
 138         struct bpf_object *obj;
 139         char buf[10];
 140         int err;
 141         struct bpf_prog_test_run_attr tattr = {
 142                 .repeat = 1,
 143                 .data_in = &pkt_v4,
 144                 .data_size_in = sizeof(pkt_v4),
 145                 .data_out = buf,
 146                 .data_size_out = 5,
 147         };
 148
 149         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj,
 150                             &tattr.prog_fd);
 151         if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno))
 152                 return;
 153
 154         memset(buf, 0, sizeof(buf));
 155
 156         err = bpf_prog_test_run_xattr(&tattr);
 157         CHECK_ATTR(err != -1 || errno != ENOSPC || tattr.retval, "run",
 158               "err %d errno %d retval %d\n", err, errno, tattr.retval);
 159
 160         CHECK_ATTR(tattr.data_size_out != sizeof(pkt_v4), "data_size_out",
 161               "incorrect output size, want %lu have %u\n",
 162               sizeof(pkt_v4), tattr.data_size_out);
 163
 164         CHECK_ATTR(buf[5] != 0, "overflow",
 165               "BPF_PROG_TEST_RUN ignored size hint\n");
 166
 167         tattr.data_out = NULL;
 168         tattr.data_size_out = 0;
 169         errno = 0;
 170
 171         err = bpf_prog_test_run_xattr(&tattr);
 172         CHECK_ATTR(err || errno || tattr.retval, "run_no_output",
 173               "err %d errno %d retval %d\n", err, errno, tattr.retval);
 174
 175         tattr.data_size_out = 1;
 176         err = bpf_prog_test_run_xattr(&tattr);
 177         CHECK_ATTR(err != -EINVAL, "run_wrong_size_out", "err %d\n", err);
 178
 179         bpf_object__close(obj);
 180 }
 181
 182 static void test_xdp(void)
 183 {
 184         struct vip key4 = {.protocol = 6, .family = AF_INET};
 185         struct vip key6 = {.protocol = 6, .family = AF_INET6};
 186         struct iptnl_info value4 = {.family = AF_INET};
 187         struct iptnl_info value6 = {.family = AF_INET6};
 188         const char *file = "./test_xdp.o";
 189         struct bpf_object *obj;
 190         char buf[128];
 191         struct ipv6hdr *iph6 = (void *)buf + sizeof(struct ethhdr);
 192         struct iphdr *iph = (void *)buf + sizeof(struct ethhdr);
 193         __u32 duration, retval, size;
 194         int err, prog_fd, map_fd;
 195
 196         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 197         if (err) {
 198                 error_cnt++;
 199                 return;
 200         }
 201
 202         map_fd = bpf_find_map(__func__, obj, "vip2tnl");
 203         if (map_fd < 0)
 204                 goto out;
 205         bpf_map_update_elem(map_fd, &key4, &value4, 0);
 206         bpf_map_update_elem(map_fd, &key6, &value6, 0);
 207
 208         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
 209                                 buf, &size, &retval, &duration);
 210
 211         CHECK(err || retval != XDP_TX || size != 74 ||
 212               iph->protocol != IPPROTO_IPIP, "ipv4",
 213               "err %d errno %d retval %d size %d\n",
 214               err, errno, retval, size);
 215
 216         err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6),
 217                                 buf, &size, &retval, &duration);
 218         CHECK(err || retval != XDP_TX || size != 114 ||
 219               iph6->nexthdr != IPPROTO_IPV6, "ipv6",
 220               "err %d errno %d retval %d size %d\n",
 221               err, errno, retval, size);
 222 out:
 223         bpf_object__close(obj);
 224 }
 225
 226 static void test_xdp_adjust_tail(void)
 227 {
 228         const char *file = "./test_adjust_tail.o";
 229         struct bpf_object *obj;
 230         char buf[128];
 231         __u32 duration, retval, size;
 232         int err, prog_fd;
 233
 234         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 235         if (err) {
 236                 error_cnt++;
 237                 return;
 238         }
 239
 240         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
 241                                 buf, &size, &retval, &duration);
 242
 243         CHECK(err || retval != XDP_DROP,
 244               "ipv4", "err %d errno %d retval %d size %d\n",
 245               err, errno, retval, size);
 246
 247         err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6),
 248                                 buf, &size, &retval, &duration);
 249         CHECK(err || retval != XDP_TX || size != 54,
 250               "ipv6", "err %d errno %d retval %d size %d\n",
 251               err, errno, retval, size);
 252         bpf_object__close(obj);
 253 }
 254
 255
 256
 257 #define MAGIC_VAL 0x1234
 258 #define NUM_ITER 100000
 259 #define VIP_NUM 5
 260
 261 static void test_l4lb(const char *file)
 262 {
 263         unsigned int nr_cpus = bpf_num_possible_cpus();
 264         struct vip key = {.protocol = 6};
 265         struct vip_meta {
 266                 __u32 flags;
 267                 __u32 vip_num;
 268         } value = {.vip_num = VIP_NUM};
 269         __u32 stats_key = VIP_NUM;
 270         struct vip_stats {
 271                 __u64 bytes;
 272                 __u64 pkts;
 273         } stats[nr_cpus];
 274         struct real_definition {
 275                 union {
 276                         __be32 dst;
 277                         __be32 dstv6[4];
 278                 };
 279                 __u8 flags;
 280         } real_def = {.dst = MAGIC_VAL};
 281         __u32 ch_key = 11, real_num = 3;
 282         __u32 duration, retval, size;
 283         int err, i, prog_fd, map_fd;
 284         __u64 bytes = 0, pkts = 0;
 285         struct bpf_object *obj;
 286         char buf[128];
 287         u32 *magic = (u32 *)buf;
 288
 289         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 290         if (err) {
 291                 error_cnt++;
 292                 return;
 293         }
 294
 295         map_fd = bpf_find_map(__func__, obj, "vip_map");
 296         if (map_fd < 0)
 297                 goto out;
 298         bpf_map_update_elem(map_fd, &key, &value, 0);
 299
 300         map_fd = bpf_find_map(__func__, obj, "ch_rings");
 301         if (map_fd < 0)
 302                 goto out;
 303         bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
 304
 305         map_fd = bpf_find_map(__func__, obj, "reals");
 306         if (map_fd < 0)
 307                 goto out;
 308         bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
 309
 310         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
 311                                 buf, &size, &retval, &duration);
 312         CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 54 ||
 313               *magic != MAGIC_VAL, "ipv4",
 314               "err %d errno %d retval %d size %d magic %x\n",
 315               err, errno, retval, size, *magic);
 316
 317         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
 318                                 buf, &size, &retval, &duration);
 319         CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 74 ||
 320               *magic != MAGIC_VAL, "ipv6",
 321               "err %d errno %d retval %d size %d magic %x\n",
 322               err, errno, retval, size, *magic);
 323
 324         map_fd = bpf_find_map(__func__, obj, "stats");
 325         if (map_fd < 0)
 326                 goto out;
 327         bpf_map_lookup_elem(map_fd, &stats_key, stats);
 328         for (i = 0; i < nr_cpus; i++) {
 329                 bytes += stats[i].bytes;
 330                 pkts += stats[i].pkts;
 331         }
 332         if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
 333                 error_cnt++;
 334                 printf("test_l4lb:FAIL:stats %lld %lld\n", bytes, pkts);
 335         }
 336 out:
 337         bpf_object__close(obj);
 338 }
 339
 340 static void test_l4lb_all(void)
 341 {
 342         const char *file1 = "./test_l4lb.o";
 343         const char *file2 = "./test_l4lb_noinline.o";
 344
 345         test_l4lb(file1);
 346         test_l4lb(file2);
 347 }
 348
 349 static void test_xdp_noinline(void)
 350 {
 351         const char *file = "./test_xdp_noinline.o";
 352         unsigned int nr_cpus = bpf_num_possible_cpus();
 353         struct vip key = {.protocol = 6};
 354         struct vip_meta {
 355                 __u32 flags;
 356                 __u32 vip_num;
 357         } value = {.vip_num = VIP_NUM};
 358         __u32 stats_key = VIP_NUM;
 359         struct vip_stats {
 360                 __u64 bytes;
 361                 __u64 pkts;
 362         } stats[nr_cpus];
 363         struct real_definition {
 364                 union {
 365                         __be32 dst;
 366                         __be32 dstv6[4];
 367                 };
 368                 __u8 flags;
 369         } real_def = {.dst = MAGIC_VAL};
 370         __u32 ch_key = 11, real_num = 3;
 371         __u32 duration, retval, size;
 372         int err, i, prog_fd, map_fd;
 373         __u64 bytes = 0, pkts = 0;
 374         struct bpf_object *obj;
 375         char buf[128];
 376         u32 *magic = (u32 *)buf;
 377
 378         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 379         if (err) {
 380                 error_cnt++;
 381                 return;
 382         }
 383
 384         map_fd = bpf_find_map(__func__, obj, "vip_map");
 385         if (map_fd < 0)
 386                 goto out;
 387         bpf_map_update_elem(map_fd, &key, &value, 0);
 388
 389         map_fd = bpf_find_map(__func__, obj, "ch_rings");
 390         if (map_fd < 0)
 391                 goto out;
 392         bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
 393
 394         map_fd = bpf_find_map(__func__, obj, "reals");
 395         if (map_fd < 0)
 396                 goto out;
 397         bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
 398
 399         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
 400                                 buf, &size, &retval, &duration);
 401         CHECK(err || retval != 1 || size != 54 ||
 402               *magic != MAGIC_VAL, "ipv4",
 403               "err %d errno %d retval %d size %d magic %x\n",
 404               err, errno, retval, size, *magic);
 405
 406         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
 407                                 buf, &size, &retval, &duration);
 408         CHECK(err || retval != 1 || size != 74 ||
 409               *magic != MAGIC_VAL, "ipv6",
 410               "err %d errno %d retval %d size %d magic %x\n",
 411               err, errno, retval, size, *magic);
 412
 413         map_fd = bpf_find_map(__func__, obj, "stats");
 414         if (map_fd < 0)
 415                 goto out;
 416         bpf_map_lookup_elem(map_fd, &stats_key, stats);
 417         for (i = 0; i < nr_cpus; i++) {
 418                 bytes += stats[i].bytes;
 419                 pkts += stats[i].pkts;
 420         }
 421         if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
 422                 error_cnt++;
 423                 printf("test_xdp_noinline:FAIL:stats %lld %lld\n", bytes, pkts);
 424         }
 425 out:
 426         bpf_object__close(obj);
 427 }
 428
 429 static void test_tcp_estats(void)
 430 {
 431         const char *file = "./test_tcp_estats.o";
 432         int err, prog_fd;
 433         struct bpf_object *obj;
 434         __u32 duration = 0;
 435
 436         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
 437         CHECK(err, "", "err %d errno %d\n", err, errno);
 438         if (err) {
 439                 error_cnt++;
 440                 return;
 441         }
 442
 443         bpf_object__close(obj);
 444 }
 445
 446 static inline __u64 ptr_to_u64(const void *ptr)
 447 {
 448         return (__u64) (unsigned long) ptr;
 449 }
 450
 451 static bool is_jit_enabled(void)
 452 {
 453         const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
 454         bool enabled = false;
 455         int sysctl_fd;
 456
 457         sysctl_fd = open(jit_sysctl, 0, O_RDONLY);
 458         if (sysctl_fd != -1) {
 459                 char tmpc;
 460
 461                 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
 462                         enabled = (tmpc != '0');
 463                 close(sysctl_fd);
 464         }
 465
 466         return enabled;
 467 }
 468
 469 static void test_bpf_obj_id(void)
 470 {
 471         const __u64 array_magic_value = 0xfaceb00c;
 472         const __u32 array_key = 0;
 473         const int nr_iters = 2;
 474         const char *file = "./test_obj_id.o";
 475         const char *expected_prog_name = "test_obj_id";
 476         const char *expected_map_name = "test_map_id";
 477         const __u64 nsec_per_sec = 1000000000;
 478
 479         struct bpf_object *objs[nr_iters];
 480         int prog_fds[nr_iters], map_fds[nr_iters];
 481         /* +1 to test for the info_len returned by kernel */
 482         struct bpf_prog_info prog_infos[nr_iters + 1];
 483         struct bpf_map_info map_infos[nr_iters + 1];
 484         /* Each prog only uses one map. +1 to test nr_map_ids
 485          * returned by kernel.
 486          */
 487         __u32 map_ids[nr_iters + 1];
 488         char jited_insns[128], xlated_insns[128], zeros[128];
 489         __u32 i, next_id, info_len, nr_id_found, duration = 0;
 490         struct timespec real_time_ts, boot_time_ts;
 491         int err = 0;
 492         __u64 array_value;
 493         uid_t my_uid = getuid();
 494         time_t now, load_time;
 495
 496         err = bpf_prog_get_fd_by_id(0);
 497         CHECK(err >= 0 || errno != ENOENT,
 498               "get-fd-by-notexist-prog-id", "err %d errno %d\n", err, errno);
 499
 500         err = bpf_map_get_fd_by_id(0);
 501         CHECK(err >= 0 || errno != ENOENT,
 502               "get-fd-by-notexist-map-id", "err %d errno %d\n", err, errno);
 503
 504         for (i = 0; i < nr_iters; i++)
 505                 objs[i] = NULL;
 506
 507         /* Check bpf_obj_get_info_by_fd() */
 508         bzero(zeros, sizeof(zeros));
 509         for (i = 0; i < nr_iters; i++) {
 510                 now = time(NULL);
 511                 err = bpf_prog_load(file, BPF_PROG_TYPE_SOCKET_FILTER,
 512                                     &objs[i], &prog_fds[i]);
 513                 /* test_obj_id.o is a dumb prog. It should never fail
 514                  * to load.
 515                  */
 516                 if (err)
 517                         error_cnt++;
 518                 assert(!err);
 519
 520                 /* Insert a magic value to the map */
 521                 map_fds[i] = bpf_find_map(__func__, objs[i], "test_map_id");
 522                 assert(map_fds[i] >= 0);
 523                 err = bpf_map_update_elem(map_fds[i], &array_key,
 524                                           &array_magic_value, 0);
 525                 assert(!err);
 526
 527                 /* Check getting map info */
 528                 info_len = sizeof(struct bpf_map_info) * 2;
 529                 bzero(&map_infos[i], info_len);
 530                 err = bpf_obj_get_info_by_fd(map_fds[i], &map_infos[i],
 531                                              &info_len);
 532                 if (CHECK(err ||
 533                           map_infos[i].type != BPF_MAP_TYPE_ARRAY ||
 534                           map_infos[i].key_size != sizeof(__u32) ||
 535                           map_infos[i].value_size != sizeof(__u64) ||
 536                           map_infos[i].max_entries != 1 ||
 537                           map_infos[i].map_flags != 0 ||
 538                           info_len != sizeof(struct bpf_map_info) ||
 539                           strcmp((char *)map_infos[i].name, expected_map_name),
 540                           "get-map-info(fd)",
 541                           "err %d errno %d type %d(%d) info_len %u(%Zu) key_size %u value_size %u max_entries %u map_flags %X name %s(%s)\n",
 542                           err, errno,
 543                           map_infos[i].type, BPF_MAP_TYPE_ARRAY,
 544                           info_len, sizeof(struct bpf_map_info),
 545                           map_infos[i].key_size,
 546                           map_infos[i].value_size,
 547                           map_infos[i].max_entries,
 548                           map_infos[i].map_flags,
 549                           map_infos[i].name, expected_map_name))
 550                         goto done;
 551
 552                 /* Check getting prog info */
 553                 info_len = sizeof(struct bpf_prog_info) * 2;
 554                 bzero(&prog_infos[i], info_len);
 555                 bzero(jited_insns, sizeof(jited_insns));
 556                 bzero(xlated_insns, sizeof(xlated_insns));
 557                 prog_infos[i].jited_prog_insns = ptr_to_u64(jited_insns);
 558                 prog_infos[i].jited_prog_len = sizeof(jited_insns);
 559                 prog_infos[i].xlated_prog_insns = ptr_to_u64(xlated_insns);
 560                 prog_infos[i].xlated_prog_len = sizeof(xlated_insns);
 561                 prog_infos[i].map_ids = ptr_to_u64(map_ids + i);
 562                 prog_infos[i].nr_map_ids = 2;
 563                 err = clock_gettime(CLOCK_REALTIME, &real_time_ts);
 564                 assert(!err);
 565                 err = clock_gettime(CLOCK_BOOTTIME, &boot_time_ts);
 566                 assert(!err);
 567                 err = bpf_obj_get_info_by_fd(prog_fds[i], &prog_infos[i],
 568                                              &info_len);
 569                 load_time = (real_time_ts.tv_sec - boot_time_ts.tv_sec)
 570                         + (prog_infos[i].load_time / nsec_per_sec);
 571                 if (CHECK(err ||
 572                           prog_infos[i].type != BPF_PROG_TYPE_SOCKET_FILTER ||
 573                           info_len != sizeof(struct bpf_prog_info) ||
 574                           (jit_enabled && !prog_infos[i].jited_prog_len) ||
 575                           (jit_enabled &&
 576                            !memcmp(jited_insns, zeros, sizeof(zeros))) ||
 577                           !prog_infos[i].xlated_prog_len ||
 578                           !memcmp(xlated_insns, zeros, sizeof(zeros)) ||
 579                           load_time < now - 60 || load_time > now + 60 ||
 580                           prog_infos[i].created_by_uid != my_uid ||
 581                           prog_infos[i].nr_map_ids != 1 ||
 582                           *(int *)(long)prog_infos[i].map_ids != map_infos[i].id ||
 583                           strcmp((char *)prog_infos[i].name, expected_prog_name),
 584                           "get-prog-info(fd)",
 585                           "err %d errno %d i %d type %d(%d) info_len %u(%Zu) jit_enabled %d jited_prog_len %u xlated_prog_len %u jited_prog %d xlated_prog %d load_time %lu(%lu) uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) name %s(%s)\n",
 586                           err, errno, i,
 587                           prog_infos[i].type, BPF_PROG_TYPE_SOCKET_FILTER,
 588                           info_len, sizeof(struct bpf_prog_info),
 589                           jit_enabled,
 590                           prog_infos[i].jited_prog_len,
 591                           prog_infos[i].xlated_prog_len,
 592                           !!memcmp(jited_insns, zeros, sizeof(zeros)),
 593                           !!memcmp(xlated_insns, zeros, sizeof(zeros)),
 594                           load_time, now,
 595                           prog_infos[i].created_by_uid, my_uid,
 596                           prog_infos[i].nr_map_ids, 1,
 597                           *(int *)(long)prog_infos[i].map_ids, map_infos[i].id,
 598                           prog_infos[i].name, expected_prog_name))
 599                         goto done;
 600         }
 601
 602         /* Check bpf_prog_get_next_id() */
 603         nr_id_found = 0;
 604         next_id = 0;
 605         while (!bpf_prog_get_next_id(next_id, &next_id)) {
 606                 struct bpf_prog_info prog_info = {};
 607                 __u32 saved_map_id;
 608                 int prog_fd;
 609
 610                 info_len = sizeof(prog_info);
 611
 612                 prog_fd = bpf_prog_get_fd_by_id(next_id);
 613                 if (prog_fd < 0 && errno == ENOENT)
 614                         /* The bpf_prog is in the dead row */
 615                         continue;
 616                 if (CHECK(prog_fd < 0, "get-prog-fd(next_id)",
 617                           "prog_fd %d next_id %d errno %d\n",
 618                           prog_fd, next_id, errno))
 619                         break;
 620
 621                 for (i = 0; i < nr_iters; i++)
 622                         if (prog_infos[i].id == next_id)
 623                                 break;
 624
 625                 if (i == nr_iters)
 626                         continue;
 627
 628                 nr_id_found++;
 629
 630                 /* Negative test:
 631                  * prog_info.nr_map_ids = 1
 632                  * prog_info.map_ids = NULL
 633                  */
 634                 prog_info.nr_map_ids = 1;
 635                 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
 636                 if (CHECK(!err || errno != EFAULT,
 637                           "get-prog-fd-bad-nr-map-ids", "err %d errno %d(%d)",
 638                           err, errno, EFAULT))
 639                         break;
 640                 bzero(&prog_info, sizeof(prog_info));
 641                 info_len = sizeof(prog_info);
 642
 643                 saved_map_id = *(int *)((long)prog_infos[i].map_ids);
 644                 prog_info.map_ids = prog_infos[i].map_ids;
 645                 prog_info.nr_map_ids = 2;
 646                 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
 647                 prog_infos[i].jited_prog_insns = 0;
 648                 prog_infos[i].xlated_prog_insns = 0;
 649                 CHECK(err || info_len != sizeof(struct bpf_prog_info) ||
 650                       memcmp(&prog_info, &prog_infos[i], info_len) ||
 651                       *(int *)(long)prog_info.map_ids != saved_map_id,
 652                       "get-prog-info(next_id->fd)",
 653                       "err %d errno %d info_len %u(%Zu) memcmp %d map_id %u(%u)\n",
 654                       err, errno, info_len, sizeof(struct bpf_prog_info),
 655                       memcmp(&prog_info, &prog_infos[i], info_len),
 656                       *(int *)(long)prog_info.map_ids, saved_map_id);
 657                 close(prog_fd);
 658         }
 659         CHECK(nr_id_found != nr_iters,
 660               "check total prog id found by get_next_id",
 661               "nr_id_found %u(%u)\n",
 662               nr_id_found, nr_iters);
 663
 664         /* Check bpf_map_get_next_id() */
 665         nr_id_found = 0;
 666         next_id = 0;
 667         while (!bpf_map_get_next_id(next_id, &next_id)) {
 668                 struct bpf_map_info map_info = {};
 669                 int map_fd;
 670
 671                 info_len = sizeof(map_info);
 672
 673                 map_fd = bpf_map_get_fd_by_id(next_id);
 674                 if (map_fd < 0 && errno == ENOENT)
 675                         /* The bpf_map is in the dead row */
 676                         continue;
 677                 if (CHECK(map_fd < 0, "get-map-fd(next_id)",
 678                           "map_fd %d next_id %u errno %d\n",
 679                           map_fd, next_id, errno))
 680                         break;
 681
 682                 for (i = 0; i < nr_iters; i++)
 683                         if (map_infos[i].id == next_id)
 684                                 break;
 685
 686                 if (i == nr_iters)
 687                         continue;
 688
 689                 nr_id_found++;
 690
 691                 err = bpf_map_lookup_elem(map_fd, &array_key, &array_value);
 692                 assert(!err);
 693
 694                 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &info_len);
 695                 CHECK(err || info_len != sizeof(struct bpf_map_info) ||
 696                       memcmp(&map_info, &map_infos[i], info_len) ||
 697                       array_value != array_magic_value,
 698                       "check get-map-info(next_id->fd)",
 699                       "err %d errno %d info_len %u(%Zu) memcmp %d array_value %llu(%llu)\n",
 700                       err, errno, info_len, sizeof(struct bpf_map_info),
 701                       memcmp(&map_info, &map_infos[i], info_len),
 702                       array_value, array_magic_value);
 703
 704                 close(map_fd);
 705         }
 706         CHECK(nr_id_found != nr_iters,
 707               "check total map id found by get_next_id",
 708               "nr_id_found %u(%u)\n",
 709               nr_id_found, nr_iters);
 710
 711 done:
 712         for (i = 0; i < nr_iters; i++)
 713                 bpf_object__close(objs[i]);
 714 }
 715
 716 static void test_pkt_md_access(void)
 717 {
 718         const char *file = "./test_pkt_md_access.o";
 719         struct bpf_object *obj;
 720         __u32 duration, retval;
 721         int err, prog_fd;
 722
 723         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 724         if (err) {
 725                 error_cnt++;
 726                 return;
 727         }
 728
 729         err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4),
 730                                 NULL, NULL, &retval, &duration);
 731         CHECK(err || retval, "",
 732               "err %d errno %d retval %d duration %d\n",
 733               err, errno, retval, duration);
 734
 735         bpf_object__close(obj);
 736 }
 737
 738 static void test_obj_name(void)
 739 {
 740         struct {
 741                 const char *name;
 742                 int success;
 743                 int expected_errno;
 744         } tests[] = {
 745                 { "", 1, 0 },
 746                 { "_123456789ABCDE", 1, 0 },
 747                 { "_123456789ABCDEF", 0, EINVAL },
 748                 { "_123456789ABCD\n", 0, EINVAL },
 749         };
 750         struct bpf_insn prog[] = {
 751                 BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0),
 752                 BPF_EXIT_INSN(),
 753         };
 754         __u32 duration = 0;
 755         int i;
 756
 757         for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) {
 758                 size_t name_len = strlen(tests[i].name) + 1;
 759                 union bpf_attr attr;
 760                 size_t ncopy;
 761                 int fd;
 762
 763                 /* test different attr.prog_name during BPF_PROG_LOAD */
 764                 ncopy = name_len < sizeof(attr.prog_name) ?
 765                         name_len : sizeof(attr.prog_name);
 766                 bzero(&attr, sizeof(attr));
 767                 attr.prog_type = BPF_PROG_TYPE_SCHED_CLS;
 768                 attr.insn_cnt = 2;
 769                 attr.insns = ptr_to_u64(prog);
 770                 attr.license = ptr_to_u64("");
 771                 memcpy(attr.prog_name, tests[i].name, ncopy);
 772
 773                 fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr));
 774                 CHECK((tests[i].success && fd < 0) ||
 775                       (!tests[i].success && fd != -1) ||
 776                       (!tests[i].success && errno != tests[i].expected_errno),
 777                       "check-bpf-prog-name",
 778                       "fd %d(%d) errno %d(%d)\n",
 779                        fd, tests[i].success, errno, tests[i].expected_errno);
 780
 781                 if (fd != -1)
 782                         close(fd);
 783
 784                 /* test different attr.map_name during BPF_MAP_CREATE */
 785                 ncopy = name_len < sizeof(attr.map_name) ?
 786                         name_len : sizeof(attr.map_name);
 787                 bzero(&attr, sizeof(attr));
 788                 attr.map_type = BPF_MAP_TYPE_ARRAY;
 789                 attr.key_size = 4;
 790                 attr.value_size = 4;
 791                 attr.max_entries = 1;
 792                 attr.map_flags = 0;
 793                 memcpy(attr.map_name, tests[i].name, ncopy);
 794                 fd = syscall(__NR_bpf, BPF_MAP_CREATE, &attr, sizeof(attr));
 795                 CHECK((tests[i].success && fd < 0) ||
 796                       (!tests[i].success && fd != -1) ||
 797                       (!tests[i].success && errno != tests[i].expected_errno),
 798                       "check-bpf-map-name",
 799                       "fd %d(%d) errno %d(%d)\n",
 800                       fd, tests[i].success, errno, tests[i].expected_errno);
 801
 802                 if (fd != -1)
 803                         close(fd);
 804         }
 805 }
 806
 807 static void test_tp_attach_query(void)
 808 {
 809         const int num_progs = 3;
 810         int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs];
 811         __u32 duration = 0, info_len, saved_prog_ids[num_progs];
 812         const char *file = "./test_tracepoint.o";
 813         struct perf_event_query_bpf *query;
 814         struct perf_event_attr attr = {};
 815         struct bpf_object *obj[num_progs];
 816         struct bpf_prog_info prog_info;
 817         char buf[256];
 818
 819         snprintf(buf, sizeof(buf),
 820                  "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
 821         efd = open(buf, O_RDONLY, 0);
 822         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
 823                 return;
 824         bytes = read(efd, buf, sizeof(buf));
 825         close(efd);
 826         if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
 827                   "read", "bytes %d errno %d\n", bytes, errno))
 828                 return;
 829
 830         attr.config = strtol(buf, NULL, 0);
 831         attr.type = PERF_TYPE_TRACEPOINT;
 832         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
 833         attr.sample_period = 1;
 834         attr.wakeup_events = 1;
 835
 836         query = malloc(sizeof(*query) + sizeof(__u32) * num_progs);
 837         for (i = 0; i < num_progs; i++) {
 838                 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj[i],
 839                                     &prog_fd[i]);
 840                 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
 841                         goto cleanup1;
 842
 843                 bzero(&prog_info, sizeof(prog_info));
 844                 prog_info.jited_prog_len = 0;
 845                 prog_info.xlated_prog_len = 0;
 846                 prog_info.nr_map_ids = 0;
 847                 info_len = sizeof(prog_info);
 848                 err = bpf_obj_get_info_by_fd(prog_fd[i], &prog_info, &info_len);
 849                 if (CHECK(err, "bpf_obj_get_info_by_fd", "err %d errno %d\n",
 850                           err, errno))
 851                         goto cleanup1;
 852                 saved_prog_ids[i] = prog_info.id;
 853
 854                 pmu_fd[i] = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
 855                                     0 /* cpu 0 */, -1 /* group id */,
 856                                     0 /* flags */);
 857                 if (CHECK(pmu_fd[i] < 0, "perf_event_open", "err %d errno %d\n",
 858                           pmu_fd[i], errno))
 859                         goto cleanup2;
 860                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_ENABLE, 0);
 861                 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
 862                           err, errno))
 863                         goto cleanup3;
 864
 865                 if (i == 0) {
 866                         /* check NULL prog array query */
 867                         query->ids_len = num_progs;
 868                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 869                         if (CHECK(err || query->prog_cnt != 0,
 870                                   "perf_event_ioc_query_bpf",
 871                                   "err %d errno %d query->prog_cnt %u\n",
 872                                   err, errno, query->prog_cnt))
 873                                 goto cleanup3;
 874                 }
 875
 876                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_SET_BPF, prog_fd[i]);
 877                 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
 878                           err, errno))
 879                         goto cleanup3;
 880
 881                 if (i == 1) {
 882                         /* try to get # of programs only */
 883                         query->ids_len = 0;
 884                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 885                         if (CHECK(err || query->prog_cnt != 2,
 886                                   "perf_event_ioc_query_bpf",
 887                                   "err %d errno %d query->prog_cnt %u\n",
 888                                   err, errno, query->prog_cnt))
 889                                 goto cleanup3;
 890
 891                         /* try a few negative tests */
 892                         /* invalid query pointer */
 893                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF,
 894                                     (struct perf_event_query_bpf *)0x1);
 895                         if (CHECK(!err || errno != EFAULT,
 896                                   "perf_event_ioc_query_bpf",
 897                                   "err %d errno %d\n", err, errno))
 898                                 goto cleanup3;
 899
 900                         /* no enough space */
 901                         query->ids_len = 1;
 902                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 903                         if (CHECK(!err || errno != ENOSPC || query->prog_cnt != 2,
 904                                   "perf_event_ioc_query_bpf",
 905                                   "err %d errno %d query->prog_cnt %u\n",
 906                                   err, errno, query->prog_cnt))
 907                                 goto cleanup3;
 908                 }
 909
 910                 query->ids_len = num_progs;
 911                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 912                 if (CHECK(err || query->prog_cnt != (i + 1),
 913                           "perf_event_ioc_query_bpf",
 914                           "err %d errno %d query->prog_cnt %u\n",
 915                           err, errno, query->prog_cnt))
 916                         goto cleanup3;
 917                 for (j = 0; j < i + 1; j++)
 918                         if (CHECK(saved_prog_ids[j] != query->ids[j],
 919                                   "perf_event_ioc_query_bpf",
 920                                   "#%d saved_prog_id %x query prog_id %x\n",
 921                                   j, saved_prog_ids[j], query->ids[j]))
 922                                 goto cleanup3;
 923         }
 924
 925         i = num_progs - 1;
 926         for (; i >= 0; i--) {
 927  cleanup3:
 928                 ioctl(pmu_fd[i], PERF_EVENT_IOC_DISABLE);
 929  cleanup2:
 930                 close(pmu_fd[i]);
 931  cleanup1:
 932                 bpf_object__close(obj[i]);
 933         }
 934         free(query);
 935 }
 936
 937 static int compare_map_keys(int map1_fd, int map2_fd)
 938 {
 939         __u32 key, next_key;
 940         char val_buf[PERF_MAX_STACK_DEPTH *
 941                      sizeof(struct bpf_stack_build_id)];
 942         int err;
 943
 944         err = bpf_map_get_next_key(map1_fd, NULL, &key);
 945         if (err)
 946                 return err;
 947         err = bpf_map_lookup_elem(map2_fd, &key, val_buf);
 948         if (err)
 949                 return err;
 950
 951         while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) {
 952                 err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf);
 953                 if (err)
 954                         return err;
 955
 956                 key = next_key;
 957         }
 958         if (errno != ENOENT)
 959                 return -1;
 960
 961         return 0;
 962 }
 963
 964 static int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len)
 965 {
 966         __u32 key, next_key, *cur_key_p, *next_key_p;
 967         char *val_buf1, *val_buf2;
 968         int i, err = 0;
 969
 970         val_buf1 = malloc(stack_trace_len);
 971         val_buf2 = malloc(stack_trace_len);
 972         cur_key_p = NULL;
 973         next_key_p = &key;
 974         while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) {
 975                 err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1);
 976                 if (err)
 977                         goto out;
 978                 err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2);
 979                 if (err)
 980                         goto out;
 981                 for (i = 0; i < stack_trace_len; i++) {
 982                         if (val_buf1[i] != val_buf2[i]) {
 983                                 err = -1;
 984                                 goto out;
 985                         }
 986                 }
 987                 key = *next_key_p;
 988                 cur_key_p = &key;
 989                 next_key_p = &next_key;
 990         }
 991         if (errno != ENOENT)
 992                 err = -1;
 993
 994 out:
 995         free(val_buf1);
 996         free(val_buf2);
 997         return err;
 998 }
 999
1000 static void test_stacktrace_map()
1001 {
1002         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1003         const char *file = "./test_stacktrace_map.o";
1004         int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1005         struct perf_event_attr attr = {};
1006         __u32 key, val, duration = 0;
1007         struct bpf_object *obj;
1008         char buf[256];
1009
1010         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1011         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1012                 return;
1013
1014         /* Get the ID for the sched/sched_switch tracepoint */
1015         snprintf(buf, sizeof(buf),
1016                  "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
1017         efd = open(buf, O_RDONLY, 0);
1018         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1019                 goto close_prog;
1020
1021         bytes = read(efd, buf, sizeof(buf));
1022         close(efd);
1023         if (bytes <= 0 || bytes >= sizeof(buf))
1024                 goto close_prog;
1025
1026         /* Open the perf event and attach bpf progrram */
1027         attr.config = strtol(buf, NULL, 0);
1028         attr.type = PERF_TYPE_TRACEPOINT;
1029         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
1030         attr.sample_period = 1;
1031         attr.wakeup_events = 1;
1032         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1033                          0 /* cpu 0 */, -1 /* group id */,
1034                          0 /* flags */);
1035         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
1036                   pmu_fd, errno))
1037                 goto close_prog;
1038
1039         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1040         if (err)
1041                 goto disable_pmu;
1042
1043         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1044         if (err)
1045                 goto disable_pmu;
1046
1047         /* find map fds */
1048         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1049         if (control_map_fd < 0)
1050                 goto disable_pmu;
1051
1052         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1053         if (stackid_hmap_fd < 0)
1054                 goto disable_pmu;
1055
1056         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1057         if (stackmap_fd < 0)
1058                 goto disable_pmu;
1059
1060         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1061         if (stack_amap_fd < 0)
1062                 goto disable_pmu;
1063
1064         /* give some time for bpf program run */
1065         sleep(1);
1066
1067         /* disable stack trace collection */
1068         key = 0;
1069         val = 1;
1070         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1071
1072         /* for every element in stackid_hmap, we can find a corresponding one
1073          * in stackmap, and vise versa.
1074          */
1075         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1076         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1077                   "err %d errno %d\n", err, errno))
1078                 goto disable_pmu_noerr;
1079
1080         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1081         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1082                   "err %d errno %d\n", err, errno))
1083                 goto disable_pmu_noerr;
1084
1085         stack_trace_len = PERF_MAX_STACK_DEPTH * sizeof(__u64);
1086         err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1087         if (CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1088                   "err %d errno %d\n", err, errno))
1089                 goto disable_pmu_noerr;
1090
1091         goto disable_pmu_noerr;
1092 disable_pmu:
1093         error_cnt++;
1094 disable_pmu_noerr:
1095         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1096         close(pmu_fd);
1097 close_prog:
1098         bpf_object__close(obj);
1099 }
1100
1101 static void test_stacktrace_map_raw_tp()
1102 {
1103         int control_map_fd, stackid_hmap_fd, stackmap_fd;
1104         const char *file = "./test_stacktrace_map.o";
1105         int efd, err, prog_fd;
1106         __u32 key, val, duration = 0;
1107         struct bpf_object *obj;
1108
1109         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1110         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1111                 return;
1112
1113         efd = bpf_raw_tracepoint_open("sched_switch", prog_fd);
1114         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1115                 goto close_prog;
1116
1117         /* find map fds */
1118         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1119         if (control_map_fd < 0)
1120                 goto close_prog;
1121
1122         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1123         if (stackid_hmap_fd < 0)
1124                 goto close_prog;
1125
1126         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1127         if (stackmap_fd < 0)
1128                 goto close_prog;
1129
1130         /* give some time for bpf program run */
1131         sleep(1);
1132
1133         /* disable stack trace collection */
1134         key = 0;
1135         val = 1;
1136         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1137
1138         /* for every element in stackid_hmap, we can find a corresponding one
1139          * in stackmap, and vise versa.
1140          */
1141         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1142         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1143                   "err %d errno %d\n", err, errno))
1144                 goto close_prog;
1145
1146         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1147         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1148                   "err %d errno %d\n", err, errno))
1149                 goto close_prog;
1150
1151         goto close_prog_noerr;
1152 close_prog:
1153         error_cnt++;
1154 close_prog_noerr:
1155         bpf_object__close(obj);
1156 }
1157
1158 static int extract_build_id(char *build_id, size_t size)
1159 {
1160         FILE *fp;
1161         char *line = NULL;
1162         size_t len = 0;
1163
1164         fp = popen("readelf -n ./urandom_read | grep 'Build ID'", "r");
1165         if (fp == NULL)
1166                 return -1;
1167
1168         if (getline(&line, &len, fp) == -1)
1169                 goto err;
1170         fclose(fp);
1171
1172         if (len > size)
1173                 len = size;
1174         memcpy(build_id, line, len);
1175         build_id[len] = '\0';
1176         return 0;
1177 err:
1178         fclose(fp);
1179         return -1;
1180 }
1181
1182 static void test_stacktrace_build_id(void)
1183 {
1184         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1185         const char *file = "./test_stacktrace_build_id.o";
1186         int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1187         struct perf_event_attr attr = {};
1188         __u32 key, previous_key, val, duration = 0;
1189         struct bpf_object *obj;
1190         char buf[256];
1191         int i, j;
1192         struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
1193         int build_id_matches = 0;
1194         int retry = 1;
1195
1196 retry:
1197         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1198         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1199                 goto out;
1200
1201         /* Get the ID for the sched/sched_switch tracepoint */
1202         snprintf(buf, sizeof(buf),
1203                  "/sys/kernel/debug/tracing/events/random/urandom_read/id");
1204         efd = open(buf, O_RDONLY, 0);
1205         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1206                 goto close_prog;
1207
1208         bytes = read(efd, buf, sizeof(buf));
1209         close(efd);
1210         if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
1211                   "read", "bytes %d errno %d\n", bytes, errno))
1212                 goto close_prog;
1213
1214         /* Open the perf event and attach bpf progrram */
1215         attr.config = strtol(buf, NULL, 0);
1216         attr.type = PERF_TYPE_TRACEPOINT;
1217         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
1218         attr.sample_period = 1;
1219         attr.wakeup_events = 1;
1220         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1221                          0 /* cpu 0 */, -1 /* group id */,
1222                          0 /* flags */);
1223         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
1224                   pmu_fd, errno))
1225                 goto close_prog;
1226
1227         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1228         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
1229                   err, errno))
1230                 goto close_pmu;
1231
1232         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1233         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
1234                   err, errno))
1235                 goto disable_pmu;
1236
1237         /* find map fds */
1238         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1239         if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
1240                   "err %d errno %d\n", err, errno))
1241                 goto disable_pmu;
1242
1243         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1244         if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
1245                   "err %d errno %d\n", err, errno))
1246                 goto disable_pmu;
1247
1248         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1249         if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
1250                   err, errno))
1251                 goto disable_pmu;
1252
1253         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1254         if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1255                   "err %d errno %d\n", err, errno))
1256                 goto disable_pmu;
1257
1258         assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1259                == 0);
1260         assert(system("./urandom_read") == 0);
1261         /* disable stack trace collection */
1262         key = 0;
1263         val = 1;
1264         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1265
1266         /* for every element in stackid_hmap, we can find a corresponding one
1267          * in stackmap, and vise versa.
1268          */
1269         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1270         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1271                   "err %d errno %d\n", err, errno))
1272                 goto disable_pmu;
1273
1274         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1275         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1276                   "err %d errno %d\n", err, errno))
1277                 goto disable_pmu;
1278
1279         err = extract_build_id(buf, 256);
1280
1281         if (CHECK(err, "get build_id with readelf",
1282                   "err %d errno %d\n", err, errno))
1283                 goto disable_pmu;
1284
1285         err = bpf_map_get_next_key(stackmap_fd, NULL, &key);
1286         if (CHECK(err, "get_next_key from stackmap",
1287                   "err %d, errno %d\n", err, errno))
1288                 goto disable_pmu;
1289
1290         do {
1291                 char build_id[64];
1292
1293                 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs);
1294                 if (CHECK(err, "lookup_elem from stackmap",
1295                           "err %d, errno %d\n", err, errno))
1296                         goto disable_pmu;
1297                 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i)
1298                         if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID &&
1299                             id_offs[i].offset != 0) {
1300                                 for (j = 0; j < 20; ++j)
1301                                         sprintf(build_id + 2 * j, "%02x",
1302                                                 id_offs[i].build_id[j] & 0xff);
1303                                 if (strstr(buf, build_id) != NULL)
1304                                         build_id_matches = 1;
1305                         }
1306                 previous_key = key;
1307         } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1308
1309         /* stack_map_get_build_id_offset() is racy and sometimes can return
1310          * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID;
1311          * try it one more time.
1312          */
1313         if (build_id_matches < 1 && retry--) {
1314                 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1315                 close(pmu_fd);
1316                 bpf_object__close(obj);
1317                 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
1318                        __func__);
1319                 goto retry;
1320         }
1321
1322         if (CHECK(build_id_matches < 1, "build id match",
1323                   "Didn't find expected build ID from the map\n"))
1324                 goto disable_pmu;
1325
1326         stack_trace_len = PERF_MAX_STACK_DEPTH
1327                 * sizeof(struct bpf_stack_build_id);
1328         err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1329         CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1330               "err %d errno %d\n", err, errno);
1331
1332 disable_pmu:
1333         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1334
1335 close_pmu:
1336         close(pmu_fd);
1337
1338 close_prog:
1339         bpf_object__close(obj);
1340
1341 out:
1342         return;
1343 }
1344
1345 static void test_stacktrace_build_id_nmi(void)
1346 {
1347         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1348         const char *file = "./test_stacktrace_build_id.o";
1349         int err, pmu_fd, prog_fd;
1350         struct perf_event_attr attr = {
1351                 .sample_freq = 5000,
1352                 .freq = 1,
1353                 .type = PERF_TYPE_HARDWARE,
1354                 .config = PERF_COUNT_HW_CPU_CYCLES,
1355         };
1356         __u32 key, previous_key, val, duration = 0;
1357         struct bpf_object *obj;
1358         char buf[256];
1359         int i, j;
1360         struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
1361         int build_id_matches = 0;
1362         int retry = 1;
1363
1364 retry:
1365         err = bpf_prog_load(file, BPF_PROG_TYPE_PERF_EVENT, &obj, &prog_fd);
1366         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1367                 return;
1368
1369         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1370                          0 /* cpu 0 */, -1 /* group id */,
1371                          0 /* flags */);
1372         if (CHECK(pmu_fd < 0, "perf_event_open",
1373                   "err %d errno %d. Does the test host support PERF_COUNT_HW_CPU_CYCLES?\n",
1374                   pmu_fd, errno))
1375                 goto close_prog;
1376
1377         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1378         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
1379                   err, errno))
1380                 goto close_pmu;
1381
1382         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1383         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
1384                   err, errno))
1385                 goto disable_pmu;
1386
1387         /* find map fds */
1388         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1389         if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
1390                   "err %d errno %d\n", err, errno))
1391                 goto disable_pmu;
1392
1393         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1394         if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
1395                   "err %d errno %d\n", err, errno))
1396                 goto disable_pmu;
1397
1398         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1399         if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
1400                   err, errno))
1401                 goto disable_pmu;
1402
1403         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1404         if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1405                   "err %d errno %d\n", err, errno))
1406                 goto disable_pmu;
1407
1408         assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1409                == 0);
1410         assert(system("taskset 0x1 ./urandom_read 100000") == 0);
1411         /* disable stack trace collection */
1412         key = 0;
1413         val = 1;
1414         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1415
1416         /* for every element in stackid_hmap, we can find a corresponding one
1417          * in stackmap, and vise versa.
1418          */
1419         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1420         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1421                   "err %d errno %d\n", err, errno))
1422                 goto disable_pmu;
1423
1424         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1425         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1426                   "err %d errno %d\n", err, errno))
1427                 goto disable_pmu;
1428
1429         err = extract_build_id(buf, 256);
1430
1431         if (CHECK(err, "get build_id with readelf",
1432                   "err %d errno %d\n", err, errno))
1433                 goto disable_pmu;
1434
1435         err = bpf_map_get_next_key(stackmap_fd, NULL, &key);
1436         if (CHECK(err, "get_next_key from stackmap",
1437                   "err %d, errno %d\n", err, errno))
1438                 goto disable_pmu;
1439
1440         do {
1441                 char build_id[64];
1442
1443                 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs);
1444                 if (CHECK(err, "lookup_elem from stackmap",
1445                           "err %d, errno %d\n", err, errno))
1446                         goto disable_pmu;
1447                 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i)
1448                         if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID &&
1449                             id_offs[i].offset != 0) {
1450                                 for (j = 0; j < 20; ++j)
1451                                         sprintf(build_id + 2 * j, "%02x",
1452                                                 id_offs[i].build_id[j] & 0xff);
1453                                 if (strstr(buf, build_id) != NULL)
1454                                         build_id_matches = 1;
1455                         }
1456                 previous_key = key;
1457         } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1458
1459         /* stack_map_get_build_id_offset() is racy and sometimes can return
1460          * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID;
1461          * try it one more time.
1462          */
1463         if (build_id_matches < 1 && retry--) {
1464                 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1465                 close(pmu_fd);
1466                 bpf_object__close(obj);
1467                 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
1468                        __func__);
1469                 goto retry;
1470         }
1471
1472         if (CHECK(build_id_matches < 1, "build id match",
1473                   "Didn't find expected build ID from the map\n"))
1474                 goto disable_pmu;
1475
1476         /*
1477          * We intentionally skip compare_stack_ips(). This is because we
1478          * only support one in_nmi() ips-to-build_id translation per cpu
1479          * at any time, thus stack_amap here will always fallback to
1480          * BPF_STACK_BUILD_ID_IP;
1481          */
1482
1483 disable_pmu:
1484         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1485
1486 close_pmu:
1487         close(pmu_fd);
1488
1489 close_prog:
1490         bpf_object__close(obj);
1491 }
1492
1493 #define MAX_CNT_RAWTP   10ull
1494 #define MAX_STACK_RAWTP 100
1495 struct get_stack_trace_t {
1496         int pid;
1497         int kern_stack_size;
1498         int user_stack_size;
1499         int user_stack_buildid_size;
1500         __u64 kern_stack[MAX_STACK_RAWTP];
1501         __u64 user_stack[MAX_STACK_RAWTP];
1502         struct bpf_stack_build_id user_stack_buildid[MAX_STACK_RAWTP];
1503 };
1504
1505 static int get_stack_print_output(void *data, int size)
1506 {
1507         bool good_kern_stack = false, good_user_stack = false;
1508         const char *nonjit_func = "___bpf_prog_run";
1509         struct get_stack_trace_t *e = data;
1510         int i, num_stack;
1511         static __u64 cnt;
1512         struct ksym *ks;
1513
1514         cnt++;
1515
1516         if (size < sizeof(struct get_stack_trace_t)) {
1517                 __u64 *raw_data = data;
1518                 bool found = false;
1519
1520                 num_stack = size / sizeof(__u64);
1521                 /* If jit is enabled, we do not have a good way to
1522                  * verify the sanity of the kernel stack. So we
1523                  * just assume it is good if the stack is not empty.
1524                  * This could be improved in the future.
1525                  */
1526                 if (jit_enabled) {
1527                         found = num_stack > 0;
1528                 } else {
1529                         for (i = 0; i < num_stack; i++) {
1530                                 ks = ksym_search(raw_data[i]);
1531                                 if (strcmp(ks->name, nonjit_func) == 0) {
1532                                         found = true;
1533                                         break;
1534                                 }
1535                         }
1536                 }
1537                 if (found) {
1538                         good_kern_stack = true;
1539                         good_user_stack = true;
1540                 }
1541         } else {
1542                 num_stack = e->kern_stack_size / sizeof(__u64);
1543                 if (jit_enabled) {
1544                         good_kern_stack = num_stack > 0;
1545                 } else {
1546                         for (i = 0; i < num_stack; i++) {
1547                                 ks = ksym_search(e->kern_stack[i]);
1548                                 if (strcmp(ks->name, nonjit_func) == 0) {
1549                                         good_kern_stack = true;
1550                                         break;
1551                                 }
1552                         }
1553                 }
1554                 if (e->user_stack_size > 0 && e->user_stack_buildid_size > 0)
1555                         good_user_stack = true;
1556         }
1557         if (!good_kern_stack || !good_user_stack)
1558                 return LIBBPF_PERF_EVENT_ERROR;
1559
1560         if (cnt == MAX_CNT_RAWTP)
1561                 return LIBBPF_PERF_EVENT_DONE;
1562
1563         return LIBBPF_PERF_EVENT_CONT;
1564 }
1565
1566 static void test_get_stack_raw_tp(void)
1567 {
1568         const char *file = "./test_get_stack_rawtp.o";
1569         int i, efd, err, prog_fd, pmu_fd, perfmap_fd;
1570         struct perf_event_attr attr = {};
1571         struct timespec tv = {0, 10};
1572         __u32 key = 0, duration = 0;
1573         struct bpf_object *obj;
1574
1575         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1576         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1577                 return;
1578
1579         efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1580         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1581                 goto close_prog;
1582
1583         perfmap_fd = bpf_find_map(__func__, obj, "perfmap");
1584         if (CHECK(perfmap_fd < 0, "bpf_find_map", "err %d errno %d\n",
1585                   perfmap_fd, errno))
1586                 goto close_prog;
1587
1588         err = load_kallsyms();
1589         if (CHECK(err < 0, "load_kallsyms", "err %d errno %d\n", err, errno))
1590                 goto close_prog;
1591
1592         attr.sample_type = PERF_SAMPLE_RAW;
1593         attr.type = PERF_TYPE_SOFTWARE;
1594         attr.config = PERF_COUNT_SW_BPF_OUTPUT;
1595         pmu_fd = syscall(__NR_perf_event_open, &attr, getpid()/*pid*/, -1/*cpu*/,
1596                          -1/*group_fd*/, 0);
1597         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", pmu_fd,
1598                   errno))
1599                 goto close_prog;
1600
1601         err = bpf_map_update_elem(perfmap_fd, &key, &pmu_fd, BPF_ANY);
1602         if (CHECK(err < 0, "bpf_map_update_elem", "err %d errno %d\n", err,
1603                   errno))
1604                 goto close_prog;
1605
1606         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1607         if (CHECK(err < 0, "ioctl PERF_EVENT_IOC_ENABLE", "err %d errno %d\n",
1608                   err, errno))
1609                 goto close_prog;
1610
1611         err = perf_event_mmap(pmu_fd);
1612         if (CHECK(err < 0, "perf_event_mmap", "err %d errno %d\n", err, errno))
1613                 goto close_prog;
1614
1615         /* trigger some syscall action */
1616         for (i = 0; i < MAX_CNT_RAWTP; i++)
1617                 nanosleep(&tv, NULL);
1618
1619         err = perf_event_poller(pmu_fd, get_stack_print_output);
1620         if (CHECK(err < 0, "perf_event_poller", "err %d errno %d\n", err, errno))
1621                 goto close_prog;
1622
1623         goto close_prog_noerr;
1624 close_prog:
1625         error_cnt++;
1626 close_prog_noerr:
1627         bpf_object__close(obj);
1628 }
1629
1630 static void test_task_fd_query_rawtp(void)
1631 {
1632         const char *file = "./test_get_stack_rawtp.o";
1633         __u64 probe_offset, probe_addr;
1634         __u32 len, prog_id, fd_type;
1635         struct bpf_object *obj;
1636         int efd, err, prog_fd;
1637         __u32 duration = 0;
1638         char buf[256];
1639
1640         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1641         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1642                 return;
1643
1644         efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1645         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1646                 goto close_prog;
1647
1648         /* query (getpid(), efd) */
1649         len = sizeof(buf);
1650         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1651                                 &fd_type, &probe_offset, &probe_addr);
1652         if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err,
1653                   errno))
1654                 goto close_prog;
1655
1656         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1657               strcmp(buf, "sys_enter") == 0;
1658         if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n",
1659                   fd_type, buf))
1660                 goto close_prog;
1661
1662         /* test zero len */
1663         len = 0;
1664         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1665                                 &fd_type, &probe_offset, &probe_addr);
1666         if (CHECK(err < 0, "bpf_task_fd_query (len = 0)", "err %d errno %d\n",
1667                   err, errno))
1668                 goto close_prog;
1669         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1670               len == strlen("sys_enter");
1671         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1672                 goto close_prog;
1673
1674         /* test empty buffer */
1675         len = sizeof(buf);
1676         err = bpf_task_fd_query(getpid(), efd, 0, 0, &len, &prog_id,
1677                                 &fd_type, &probe_offset, &probe_addr);
1678         if (CHECK(err < 0, "bpf_task_fd_query (buf = 0)", "err %d errno %d\n",
1679                   err, errno))
1680                 goto close_prog;
1681         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1682               len == strlen("sys_enter");
1683         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1684                 goto close_prog;
1685
1686         /* test smaller buffer */
1687         len = 3;
1688         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1689                                 &fd_type, &probe_offset, &probe_addr);
1690         if (CHECK(err >= 0 || errno != ENOSPC, "bpf_task_fd_query (len = 3)",
1691                   "err %d errno %d\n", err, errno))
1692                 goto close_prog;
1693         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1694               len == strlen("sys_enter") &&
1695               strcmp(buf, "sy") == 0;
1696         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1697                 goto close_prog;
1698
1699         goto close_prog_noerr;
1700 close_prog:
1701         error_cnt++;
1702 close_prog_noerr:
1703         bpf_object__close(obj);
1704 }
1705
1706 static void test_task_fd_query_tp_core(const char *probe_name,
1707                                        const char *tp_name)
1708 {
1709         const char *file = "./test_tracepoint.o";
1710         int err, bytes, efd, prog_fd, pmu_fd;
1711         struct perf_event_attr attr = {};
1712         __u64 probe_offset, probe_addr;
1713         __u32 len, prog_id, fd_type;
1714         struct bpf_object *obj;
1715         __u32 duration = 0;
1716         char buf[256];
1717
1718         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1719         if (CHECK(err, "bpf_prog_load", "err %d errno %d\n", err, errno))
1720                 goto close_prog;
1721
1722         snprintf(buf, sizeof(buf),
1723                  "/sys/kernel/debug/tracing/events/%s/id", probe_name);
1724         efd = open(buf, O_RDONLY, 0);
1725         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1726                 goto close_prog;
1727         bytes = read(efd, buf, sizeof(buf));
1728         close(efd);
1729         if (CHECK(bytes <= 0 || bytes >= sizeof(buf), "read",
1730                   "bytes %d errno %d\n", bytes, errno))
1731                 goto close_prog;
1732
1733         attr.config = strtol(buf, NULL, 0);
1734         attr.type = PERF_TYPE_TRACEPOINT;
1735         attr.sample_type = PERF_SAMPLE_RAW;
1736         attr.sample_period = 1;
1737         attr.wakeup_events = 1;
1738         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1739                          0 /* cpu 0 */, -1 /* group id */,
1740                          0 /* flags */);
1741         if (CHECK(err, "perf_event_open", "err %d errno %d\n", err, errno))
1742                 goto close_pmu;
1743
1744         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1745         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", err,
1746                   errno))
1747                 goto close_pmu;
1748
1749         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1750         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", err,
1751                   errno))
1752                 goto close_pmu;
1753
1754         /* query (getpid(), pmu_fd) */
1755         len = sizeof(buf);
1756         err = bpf_task_fd_query(getpid(), pmu_fd, 0, buf, &len, &prog_id,
1757                                 &fd_type, &probe_offset, &probe_addr);
1758         if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err,
1759                   errno))
1760                 goto close_pmu;
1761
1762         err = (fd_type == BPF_FD_TYPE_TRACEPOINT) && !strcmp(buf, tp_name);
1763         if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n",
1764                   fd_type, buf))
1765                 goto close_pmu;
1766
1767         close(pmu_fd);
1768         goto close_prog_noerr;
1769
1770 close_pmu:
1771         close(pmu_fd);
1772 close_prog:
1773         error_cnt++;
1774 close_prog_noerr:
1775         bpf_object__close(obj);
1776 }
1777
1778 static void test_task_fd_query_tp(void)
1779 {
1780         test_task_fd_query_tp_core("sched/sched_switch",
1781                                    "sched_switch");
1782         test_task_fd_query_tp_core("syscalls/sys_enter_read",
1783                                    "sys_enter_read");
1784 }
1785
1786 static void test_reference_tracking()
1787 {
1788         const char *file = "./test_sk_lookup_kern.o";
1789         struct bpf_object *obj;
1790         struct bpf_program *prog;
1791         __u32 duration = 0;
1792         int err = 0;
1793
1794         obj = bpf_object__open(file);
1795         if (IS_ERR(obj)) {
1796                 error_cnt++;
1797                 return;
1798         }
1799
1800         bpf_object__for_each_program(prog, obj) {
1801                 const char *title;
1802
1803                 /* Ignore .text sections */
1804                 title = bpf_program__title(prog, false);
1805                 if (strstr(title, ".text") != NULL)
1806                         continue;
1807
1808                 bpf_program__set_type(prog, BPF_PROG_TYPE_SCHED_CLS);
1809
1810                 /* Expect verifier failure if test name has 'fail' */
1811                 if (strstr(title, "fail") != NULL) {
1812                         libbpf_set_print(NULL, NULL, NULL);
1813                         err = !bpf_program__load(prog, "GPL", 0);
1814                         libbpf_set_print(printf, printf, NULL);
1815                 } else {
1816                         err = bpf_program__load(prog, "GPL", 0);
1817                 }
1818                 CHECK(err, title, "\n");
1819         }
1820         bpf_object__close(obj);
1821 }
1822
1823 enum {
1824         QUEUE,
1825         STACK,
1826 };
1827
1828 static void test_queue_stack_map(int type)
1829 {
1830         const int MAP_SIZE = 32;
1831         __u32 vals[MAP_SIZE], duration, retval, size, val;
1832         int i, err, prog_fd, map_in_fd, map_out_fd;
1833         char file[32], buf[128];
1834         struct bpf_object *obj;
1835         struct iphdr *iph = (void *)buf + sizeof(struct ethhdr);
1836
1837         /* Fill test values to be used */
1838         for (i = 0; i < MAP_SIZE; i++)
1839                 vals[i] = rand();
1840
1841         if (type == QUEUE)
1842                 strncpy(file, "./test_queue_map.o", sizeof(file));
1843         else if (type == STACK)
1844                 strncpy(file, "./test_stack_map.o", sizeof(file));
1845         else
1846                 return;
1847
1848         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
1849         if (err) {
1850                 error_cnt++;
1851                 return;
1852         }
1853
1854         map_in_fd = bpf_find_map(__func__, obj, "map_in");
1855         if (map_in_fd < 0)
1856                 goto out;
1857
1858         map_out_fd = bpf_find_map(__func__, obj, "map_out");
1859         if (map_out_fd < 0)
1860                 goto out;
1861
1862         /* Push 32 elements to the input map */
1863         for (i = 0; i < MAP_SIZE; i++) {
1864                 err = bpf_map_update_elem(map_in_fd, NULL, &vals[i], 0);
1865                 if (err) {
1866                         error_cnt++;
1867                         goto out;
1868                 }
1869         }
1870
1871         /* The eBPF program pushes iph.saddr in the output map,
1872          * pops the input map and saves this value in iph.daddr
1873          */
1874         for (i = 0; i < MAP_SIZE; i++) {
1875                 if (type == QUEUE) {
1876                         val = vals[i];
1877                         pkt_v4.iph.saddr = vals[i] * 5;
1878                 } else if (type == STACK) {
1879                         val = vals[MAP_SIZE - 1 - i];
1880                         pkt_v4.iph.saddr = vals[MAP_SIZE - 1 - i] * 5;
1881                 }
1882
1883                 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
1884                                         buf, &size, &retval, &duration);
1885                 if (err || retval || size != sizeof(pkt_v4) ||
1886                     iph->daddr != val)
1887                         break;
1888         }
1889
1890         CHECK(err || retval || size != sizeof(pkt_v4) || iph->daddr != val,
1891               "bpf_map_pop_elem",
1892               "err %d errno %d retval %d size %d iph->daddr %u\n",
1893               err, errno, retval, size, iph->daddr);
1894
1895         /* Queue is empty, program should return TC_ACT_SHOT */
1896         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
1897                                 buf, &size, &retval, &duration);
1898         CHECK(err || retval != 2 /* TC_ACT_SHOT */|| size != sizeof(pkt_v4),
1899               "check-queue-stack-map-empty",
1900               "err %d errno %d retval %d size %d\n",
1901               err, errno, retval, size);
1902
1903         /* Check that the program pushed elements correctly */
1904         for (i = 0; i < MAP_SIZE; i++) {
1905                 err = bpf_map_lookup_and_delete_elem(map_out_fd, NULL, &val);
1906                 if (err || val != vals[i] * 5)
1907                         break;
1908         }
1909
1910         CHECK(i != MAP_SIZE && (err || val != vals[i] * 5),
1911               "bpf_map_push_elem", "err %d value %u\n", err, val);
1912
1913 out:
1914         pkt_v4.iph.saddr = 0;
1915         bpf_object__close(obj);
1916 }
1917
1918 #define CHECK_FLOW_KEYS(desc, got, expected)                            \
1919         CHECK(memcmp(&got, &expected, sizeof(got)) != 0,                \
1920               desc,                                                     \
1921               "nhoff=%u/%u "                                            \
1922               "thoff=%u/%u "                                            \
1923               "addr_proto=0x%x/0x%x "                                   \
1924               "is_frag=%u/%u "                                          \
1925               "is_first_frag=%u/%u "                                    \
1926               "is_encap=%u/%u "                                         \
1927               "n_proto=0x%x/0x%x "                                      \
1928               "sport=%u/%u "                                            \
1929               "dport=%u/%u\n",                                          \
1930               got.nhoff, expected.nhoff,                                \
1931               got.thoff, expected.thoff,                                \
1932               got.addr_proto, expected.addr_proto,                      \
1933               got.is_frag, expected.is_frag,                            \
1934               got.is_first_frag, expected.is_first_frag,                \
1935               got.is_encap, expected.is_encap,                          \
1936               got.n_proto, expected.n_proto,                            \
1937               got.sport, expected.sport,                                \
1938               got.dport, expected.dport)
1939
1940 static struct bpf_flow_keys pkt_v4_flow_keys = {
1941         .nhoff = 0,
1942         .thoff = sizeof(struct iphdr),
1943         .addr_proto = ETH_P_IP,
1944         .ip_proto = IPPROTO_TCP,
1945         .n_proto = bpf_htons(ETH_P_IP),
1946 };
1947
1948 static struct bpf_flow_keys pkt_v6_flow_keys = {
1949         .nhoff = 0,
1950         .thoff = sizeof(struct ipv6hdr),
1951         .addr_proto = ETH_P_IPV6,
1952         .ip_proto = IPPROTO_TCP,
1953         .n_proto = bpf_htons(ETH_P_IPV6),
1954 };
1955
1956 static void test_flow_dissector(void)
1957 {
1958         struct bpf_flow_keys flow_keys;
1959         struct bpf_object *obj;
1960         __u32 duration, retval;
1961         int err, prog_fd;
1962         __u32 size;
1963
1964         err = bpf_flow_load(&obj, "./bpf_flow.o", "flow_dissector",
1965                             "jmp_table", &prog_fd);
1966         if (err) {
1967                 error_cnt++;
1968                 return;
1969         }
1970
1971         err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4),
1972                                 &flow_keys, &size, &retval, &duration);
1973         CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv4",
1974               "err %d errno %d retval %d duration %d size %u/%lu\n",
1975               err, errno, retval, duration, size, sizeof(flow_keys));
1976         CHECK_FLOW_KEYS("ipv4_flow_keys", flow_keys, pkt_v4_flow_keys);
1977
1978         err = bpf_prog_test_run(prog_fd, 10, &pkt_v6, sizeof(pkt_v6),
1979                                 &flow_keys, &size, &retval, &duration);
1980         CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv6",
1981               "err %d errno %d retval %d duration %d size %u/%lu\n",
1982               err, errno, retval, duration, size, sizeof(flow_keys));
1983         CHECK_FLOW_KEYS("ipv6_flow_keys", flow_keys, pkt_v6_flow_keys);
1984
1985         bpf_object__close(obj);
1986 }
1987
1988 static void *test_spin_lock(void *arg)
1989 {
1990         __u32 duration, retval;
1991         int err, prog_fd = *(u32 *) arg;
1992
1993         err = bpf_prog_test_run(prog_fd, 10000, &pkt_v4, sizeof(pkt_v4),
1994                                 NULL, NULL, &retval, &duration);
1995         CHECK(err || retval, "",
1996               "err %d errno %d retval %d duration %d\n",
1997               err, errno, retval, duration);
1998         pthread_exit(arg);
1999 }
2000
2001 static void test_spinlock(void)
2002 {
2003         const char *file = "./test_spin_lock.o";
2004         pthread_t thread_id[4];
2005         struct bpf_object *obj;
2006         int prog_fd;
2007         int err = 0, i;
2008         void *ret;
2009
2010         err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd);
2011         if (err) {
2012                 printf("test_spin_lock:bpf_prog_load errno %d\n", errno);
2013                 goto close_prog;
2014         }
2015         for (i = 0; i < 4; i++)
2016                 assert(pthread_create(&thread_id[i], NULL,
2017                                       &test_spin_lock, &prog_fd) == 0);
2018         for (i = 0; i < 4; i++)
2019                 assert(pthread_join(thread_id[i], &ret) == 0 &&
2020                        ret == (void *)&prog_fd);
2021         goto close_prog_noerr;
2022 close_prog:
2023         error_cnt++;
2024 close_prog_noerr:
2025         bpf_object__close(obj);
2026 }
2027
2028 int main(void)
2029 {
2030         srand(time(NULL));
2031
2032         jit_enabled = is_jit_enabled();
2033
2034         test_pkt_access();
2035         test_prog_run_xattr();
2036         test_xdp();
2037         test_xdp_adjust_tail();
2038         test_l4lb_all();
2039         test_xdp_noinline();
2040         test_tcp_estats();
2041         test_bpf_obj_id();
2042         test_pkt_md_access();
2043         test_obj_name();
2044         test_tp_attach_query();
2045         test_stacktrace_map();
2046         test_stacktrace_build_id();
2047         test_stacktrace_build_id_nmi();
2048         test_stacktrace_map_raw_tp();
2049         test_get_stack_raw_tp();
2050         test_task_fd_query_rawtp();
2051         test_task_fd_query_tp();
2052         test_reference_tracking();
2053         test_queue_stack_map(QUEUE);
2054         test_queue_stack_map(STACK);
2055         test_flow_dissector();
2056         test_spinlock();
2057
2058         printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt);
2059         return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
2060 }