tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  * Copyright (C) 2019 Isovalent, Inc.
  11  */
  12
  13 #ifndef _GNU_SOURCE
  14 #define _GNU_SOURCE
  15 #endif
  16 #include <stdlib.h>
  17 #include <stdio.h>
  18 #include <stdarg.h>
  19 #include <libgen.h>
  20 #include <inttypes.h>
  21 #include <limits.h>
  22 #include <string.h>
  23 #include <unistd.h>
  24 #include <endian.h>
  25 #include <fcntl.h>
  26 #include <errno.h>
  27 #include <ctype.h>
  28 #include <asm/unistd.h>
  29 #include <linux/err.h>
  30 #include <linux/kernel.h>
  31 #include <linux/bpf.h>
  32 #include <linux/btf.h>
  33 #include <linux/filter.h>
  34 #include <linux/limits.h>
  35 #include <linux/perf_event.h>
  36 #include <linux/ring_buffer.h>
  37 #include <linux/version.h>
  38 #include <sys/epoll.h>
  39 #include <sys/ioctl.h>
  40 #include <sys/mman.h>
  41 #include <sys/stat.h>
  42 #include <sys/types.h>
  43 #include <sys/vfs.h>
  44 #include <sys/utsname.h>
  45 #include <sys/resource.h>
  46 #include <libelf.h>
  47 #include <gelf.h>
  48 #include <zlib.h>
  49
  50 #include "libbpf.h"
  51 #include "bpf.h"
  52 #include "btf.h"
  53 #include "str_error.h"
  54 #include "libbpf_internal.h"
  55 #include "hashmap.h"
  56 #include "bpf_gen_internal.h"
  57
  58 #ifndef BPF_FS_MAGIC
  59 #define BPF_FS_MAGIC            0xcafe4a11
  60 #endif
  61
  62 #define BPF_INSN_SZ (sizeof(struct bpf_insn))
  63
  64 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
  65  * compilation if user enables corresponding warning. Disable it explicitly.
  66  */
  67 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
  68
  69 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  70
  71 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
  72 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
  73
  74 static const char * const attach_type_name[] = {
  75         [BPF_CGROUP_INET_INGRESS]       = "cgroup_inet_ingress",
  76         [BPF_CGROUP_INET_EGRESS]        = "cgroup_inet_egress",
  77         [BPF_CGROUP_INET_SOCK_CREATE]   = "cgroup_inet_sock_create",
  78         [BPF_CGROUP_INET_SOCK_RELEASE]  = "cgroup_inet_sock_release",
  79         [BPF_CGROUP_SOCK_OPS]           = "cgroup_sock_ops",
  80         [BPF_CGROUP_DEVICE]             = "cgroup_device",
  81         [BPF_CGROUP_INET4_BIND]         = "cgroup_inet4_bind",
  82         [BPF_CGROUP_INET6_BIND]         = "cgroup_inet6_bind",
  83         [BPF_CGROUP_INET4_CONNECT]      = "cgroup_inet4_connect",
  84         [BPF_CGROUP_INET6_CONNECT]      = "cgroup_inet6_connect",
  85         [BPF_CGROUP_INET4_POST_BIND]    = "cgroup_inet4_post_bind",
  86         [BPF_CGROUP_INET6_POST_BIND]    = "cgroup_inet6_post_bind",
  87         [BPF_CGROUP_INET4_GETPEERNAME]  = "cgroup_inet4_getpeername",
  88         [BPF_CGROUP_INET6_GETPEERNAME]  = "cgroup_inet6_getpeername",
  89         [BPF_CGROUP_INET4_GETSOCKNAME]  = "cgroup_inet4_getsockname",
  90         [BPF_CGROUP_INET6_GETSOCKNAME]  = "cgroup_inet6_getsockname",
  91         [BPF_CGROUP_UDP4_SENDMSG]       = "cgroup_udp4_sendmsg",
  92         [BPF_CGROUP_UDP6_SENDMSG]       = "cgroup_udp6_sendmsg",
  93         [BPF_CGROUP_SYSCTL]             = "cgroup_sysctl",
  94         [BPF_CGROUP_UDP4_RECVMSG]       = "cgroup_udp4_recvmsg",
  95         [BPF_CGROUP_UDP6_RECVMSG]       = "cgroup_udp6_recvmsg",
  96         [BPF_CGROUP_GETSOCKOPT]         = "cgroup_getsockopt",
  97         [BPF_CGROUP_SETSOCKOPT]         = "cgroup_setsockopt",
  98         [BPF_SK_SKB_STREAM_PARSER]      = "sk_skb_stream_parser",
  99         [BPF_SK_SKB_STREAM_VERDICT]     = "sk_skb_stream_verdict",
 100         [BPF_SK_SKB_VERDICT]            = "sk_skb_verdict",
 101         [BPF_SK_MSG_VERDICT]            = "sk_msg_verdict",
 102         [BPF_LIRC_MODE2]                = "lirc_mode2",
 103         [BPF_FLOW_DISSECTOR]            = "flow_dissector",
 104         [BPF_TRACE_RAW_TP]              = "trace_raw_tp",
 105         [BPF_TRACE_FENTRY]              = "trace_fentry",
 106         [BPF_TRACE_FEXIT]               = "trace_fexit",
 107         [BPF_MODIFY_RETURN]             = "modify_return",
 108         [BPF_LSM_MAC]                   = "lsm_mac",
 109         [BPF_LSM_CGROUP]                = "lsm_cgroup",
 110         [BPF_SK_LOOKUP]                 = "sk_lookup",
 111         [BPF_TRACE_ITER]                = "trace_iter",
 112         [BPF_XDP_DEVMAP]                = "xdp_devmap",
 113         [BPF_XDP_CPUMAP]                = "xdp_cpumap",
 114         [BPF_XDP]                       = "xdp",
 115         [BPF_SK_REUSEPORT_SELECT]       = "sk_reuseport_select",
 116         [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE]    = "sk_reuseport_select_or_migrate",
 117         [BPF_PERF_EVENT]                = "perf_event",
 118         [BPF_TRACE_KPROBE_MULTI]        = "trace_kprobe_multi",
 119 };
 120
 121 static const char * const link_type_name[] = {
 122         [BPF_LINK_TYPE_UNSPEC]                  = "unspec",
 123         [BPF_LINK_TYPE_RAW_TRACEPOINT]          = "raw_tracepoint",
 124         [BPF_LINK_TYPE_TRACING]                 = "tracing",
 125         [BPF_LINK_TYPE_CGROUP]                  = "cgroup",
 126         [BPF_LINK_TYPE_ITER]                    = "iter",
 127         [BPF_LINK_TYPE_NETNS]                   = "netns",
 128         [BPF_LINK_TYPE_XDP]                     = "xdp",
 129         [BPF_LINK_TYPE_PERF_EVENT]              = "perf_event",
 130         [BPF_LINK_TYPE_KPROBE_MULTI]            = "kprobe_multi",
 131         [BPF_LINK_TYPE_STRUCT_OPS]              = "struct_ops",
 132 };
 133
 134 static const char * const map_type_name[] = {
 135         [BPF_MAP_TYPE_UNSPEC]                   = "unspec",
 136         [BPF_MAP_TYPE_HASH]                     = "hash",
 137         [BPF_MAP_TYPE_ARRAY]                    = "array",
 138         [BPF_MAP_TYPE_PROG_ARRAY]               = "prog_array",
 139         [BPF_MAP_TYPE_PERF_EVENT_ARRAY]         = "perf_event_array",
 140         [BPF_MAP_TYPE_PERCPU_HASH]              = "percpu_hash",
 141         [BPF_MAP_TYPE_PERCPU_ARRAY]             = "percpu_array",
 142         [BPF_MAP_TYPE_STACK_TRACE]              = "stack_trace",
 143         [BPF_MAP_TYPE_CGROUP_ARRAY]             = "cgroup_array",
 144         [BPF_MAP_TYPE_LRU_HASH]                 = "lru_hash",
 145         [BPF_MAP_TYPE_LRU_PERCPU_HASH]          = "lru_percpu_hash",
 146         [BPF_MAP_TYPE_LPM_TRIE]                 = "lpm_trie",
 147         [BPF_MAP_TYPE_ARRAY_OF_MAPS]            = "array_of_maps",
 148         [BPF_MAP_TYPE_HASH_OF_MAPS]             = "hash_of_maps",
 149         [BPF_MAP_TYPE_DEVMAP]                   = "devmap",
 150         [BPF_MAP_TYPE_DEVMAP_HASH]              = "devmap_hash",
 151         [BPF_MAP_TYPE_SOCKMAP]                  = "sockmap",
 152         [BPF_MAP_TYPE_CPUMAP]                   = "cpumap",
 153         [BPF_MAP_TYPE_XSKMAP]                   = "xskmap",
 154         [BPF_MAP_TYPE_SOCKHASH]                 = "sockhash",
 155         [BPF_MAP_TYPE_CGROUP_STORAGE]           = "cgroup_storage",
 156         [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY]      = "reuseport_sockarray",
 157         [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE]    = "percpu_cgroup_storage",
 158         [BPF_MAP_TYPE_QUEUE]                    = "queue",
 159         [BPF_MAP_TYPE_STACK]                    = "stack",
 160         [BPF_MAP_TYPE_SK_STORAGE]               = "sk_storage",
 161         [BPF_MAP_TYPE_STRUCT_OPS]               = "struct_ops",
 162         [BPF_MAP_TYPE_RINGBUF]                  = "ringbuf",
 163         [BPF_MAP_TYPE_INODE_STORAGE]            = "inode_storage",
 164         [BPF_MAP_TYPE_TASK_STORAGE]             = "task_storage",
 165         [BPF_MAP_TYPE_BLOOM_FILTER]             = "bloom_filter",
 166 };
 167
 168 static const char * const prog_type_name[] = {
 169         [BPF_PROG_TYPE_UNSPEC]                  = "unspec",
 170         [BPF_PROG_TYPE_SOCKET_FILTER]           = "socket_filter",
 171         [BPF_PROG_TYPE_KPROBE]                  = "kprobe",
 172         [BPF_PROG_TYPE_SCHED_CLS]               = "sched_cls",
 173         [BPF_PROG_TYPE_SCHED_ACT]               = "sched_act",
 174         [BPF_PROG_TYPE_TRACEPOINT]              = "tracepoint",
 175         [BPF_PROG_TYPE_XDP]                     = "xdp",
 176         [BPF_PROG_TYPE_PERF_EVENT]              = "perf_event",
 177         [BPF_PROG_TYPE_CGROUP_SKB]              = "cgroup_skb",
 178         [BPF_PROG_TYPE_CGROUP_SOCK]             = "cgroup_sock",
 179         [BPF_PROG_TYPE_LWT_IN]                  = "lwt_in",
 180         [BPF_PROG_TYPE_LWT_OUT]                 = "lwt_out",
 181         [BPF_PROG_TYPE_LWT_XMIT]                = "lwt_xmit",
 182         [BPF_PROG_TYPE_SOCK_OPS]                = "sock_ops",
 183         [BPF_PROG_TYPE_SK_SKB]                  = "sk_skb",
 184         [BPF_PROG_TYPE_CGROUP_DEVICE]           = "cgroup_device",
 185         [BPF_PROG_TYPE_SK_MSG]                  = "sk_msg",
 186         [BPF_PROG_TYPE_RAW_TRACEPOINT]          = "raw_tracepoint",
 187         [BPF_PROG_TYPE_CGROUP_SOCK_ADDR]        = "cgroup_sock_addr",
 188         [BPF_PROG_TYPE_LWT_SEG6LOCAL]           = "lwt_seg6local",
 189         [BPF_PROG_TYPE_LIRC_MODE2]              = "lirc_mode2",
 190         [BPF_PROG_TYPE_SK_REUSEPORT]            = "sk_reuseport",
 191         [BPF_PROG_TYPE_FLOW_DISSECTOR]          = "flow_dissector",
 192         [BPF_PROG_TYPE_CGROUP_SYSCTL]           = "cgroup_sysctl",
 193         [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
 194         [BPF_PROG_TYPE_CGROUP_SOCKOPT]          = "cgroup_sockopt",
 195         [BPF_PROG_TYPE_TRACING]                 = "tracing",
 196         [BPF_PROG_TYPE_STRUCT_OPS]              = "struct_ops",
 197         [BPF_PROG_TYPE_EXT]                     = "ext",
 198         [BPF_PROG_TYPE_LSM]                     = "lsm",
 199         [BPF_PROG_TYPE_SK_LOOKUP]               = "sk_lookup",
 200         [BPF_PROG_TYPE_SYSCALL]                 = "syscall",
 201 };
 202
 203 static int __base_pr(enum libbpf_print_level level, const char *format,
 204                      va_list args)
 205 {
 206         if (level == LIBBPF_DEBUG)
 207                 return 0;
 208
 209         return vfprintf(stderr, format, args);
 210 }
 211
 212 static libbpf_print_fn_t __libbpf_pr = __base_pr;
 213
 214 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
 215 {
 216         libbpf_print_fn_t old_print_fn = __libbpf_pr;
 217
 218         __libbpf_pr = fn;
 219         return old_print_fn;
 220 }
 221
 222 __printf(2, 3)
 223 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
 224 {
 225         va_list args;
 226         int old_errno;
 227
 228         if (!__libbpf_pr)
 229                 return;
 230
 231         old_errno = errno;
 232
 233         va_start(args, format);
 234         __libbpf_pr(level, format, args);
 235         va_end(args);
 236
 237         errno = old_errno;
 238 }
 239
 240 static void pr_perm_msg(int err)
 241 {
 242         struct rlimit limit;
 243         char buf[100];
 244
 245         if (err != -EPERM || geteuid() != 0)
 246                 return;
 247
 248         err = getrlimit(RLIMIT_MEMLOCK, &limit);
 249         if (err)
 250                 return;
 251
 252         if (limit.rlim_cur == RLIM_INFINITY)
 253                 return;
 254
 255         if (limit.rlim_cur < 1024)
 256                 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
 257         else if (limit.rlim_cur < 1024*1024)
 258                 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
 259         else
 260                 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
 261
 262         pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
 263                 buf);
 264 }
 265
 266 #define STRERR_BUFSIZE  128
 267
 268 /* Copied from tools/perf/util/util.h */
 269 #ifndef zfree
 270 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 271 #endif
 272
 273 #ifndef zclose
 274 # define zclose(fd) ({                  \
 275         int ___err = 0;                 \
 276         if ((fd) >= 0)                  \
 277                 ___err = close((fd));   \
 278         fd = -1;                        \
 279         ___err; })
 280 #endif
 281
 282 static inline __u64 ptr_to_u64(const void *ptr)
 283 {
 284         return (__u64) (unsigned long) ptr;
 285 }
 286
 287 int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
 288 {
 289         /* as of v1.0 libbpf_set_strict_mode() is a no-op */
 290         return 0;
 291 }
 292
 293 __u32 libbpf_major_version(void)
 294 {
 295         return LIBBPF_MAJOR_VERSION;
 296 }
 297
 298 __u32 libbpf_minor_version(void)
 299 {
 300         return LIBBPF_MINOR_VERSION;
 301 }
 302
 303 const char *libbpf_version_string(void)
 304 {
 305 #define __S(X) #X
 306 #define _S(X) __S(X)
 307         return  "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);
 308 #undef _S
 309 #undef __S
 310 }
 311
 312 enum reloc_type {
 313         RELO_LD64,
 314         RELO_CALL,
 315         RELO_DATA,
 316         RELO_EXTERN_VAR,
 317         RELO_EXTERN_FUNC,
 318         RELO_SUBPROG_ADDR,
 319         RELO_CORE,
 320 };
 321
 322 struct reloc_desc {
 323         enum reloc_type type;
 324         int insn_idx;
 325         union {
 326                 const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */
 327                 struct {
 328                         int map_idx;
 329                         int sym_off;
 330                 };
 331         };
 332 };
 333
 334 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
 335 enum sec_def_flags {
 336         SEC_NONE = 0,
 337         /* expected_attach_type is optional, if kernel doesn't support that */
 338         SEC_EXP_ATTACH_OPT = 1,
 339         /* legacy, only used by libbpf_get_type_names() and
 340          * libbpf_attach_type_by_name(), not used by libbpf itself at all.
 341          * This used to be associated with cgroup (and few other) BPF programs
 342          * that were attachable through BPF_PROG_ATTACH command. Pretty
 343          * meaningless nowadays, though.
 344          */
 345         SEC_ATTACHABLE = 2,
 346         SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT,
 347         /* attachment target is specified through BTF ID in either kernel or
 348          * other BPF program's BTF object */
 349         SEC_ATTACH_BTF = 4,
 350         /* BPF program type allows sleeping/blocking in kernel */
 351         SEC_SLEEPABLE = 8,
 352         /* BPF program support non-linear XDP buffer */
 353         SEC_XDP_FRAGS = 16,
 354 };
 355
 356 struct bpf_sec_def {
 357         char *sec;
 358         enum bpf_prog_type prog_type;
 359         enum bpf_attach_type expected_attach_type;
 360         long cookie;
 361         int handler_id;
 362
 363         libbpf_prog_setup_fn_t prog_setup_fn;
 364         libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;
 365         libbpf_prog_attach_fn_t prog_attach_fn;
 366 };
 367
 368 /*
 369  * bpf_prog should be a better name but it has been used in
 370  * linux/filter.h.
 371  */
 372 struct bpf_program {
 373         char *name;
 374         char *sec_name;
 375         size_t sec_idx;
 376         const struct bpf_sec_def *sec_def;
 377         /* this program's instruction offset (in number of instructions)
 378          * within its containing ELF section
 379          */
 380         size_t sec_insn_off;
 381         /* number of original instructions in ELF section belonging to this
 382          * program, not taking into account subprogram instructions possible
 383          * appended later during relocation
 384          */
 385         size_t sec_insn_cnt;
 386         /* Offset (in number of instructions) of the start of instruction
 387          * belonging to this BPF program  within its containing main BPF
 388          * program. For the entry-point (main) BPF program, this is always
 389          * zero. For a sub-program, this gets reset before each of main BPF
 390          * programs are processed and relocated and is used to determined
 391          * whether sub-program was already appended to the main program, and
 392          * if yes, at which instruction offset.
 393          */
 394         size_t sub_insn_off;
 395
 396         /* instructions that belong to BPF program; insns[0] is located at
 397          * sec_insn_off instruction within its ELF section in ELF file, so
 398          * when mapping ELF file instruction index to the local instruction,
 399          * one needs to subtract sec_insn_off; and vice versa.
 400          */
 401         struct bpf_insn *insns;
 402         /* actual number of instruction in this BPF program's image; for
 403          * entry-point BPF programs this includes the size of main program
 404          * itself plus all the used sub-programs, appended at the end
 405          */
 406         size_t insns_cnt;
 407
 408         struct reloc_desc *reloc_desc;
 409         int nr_reloc;
 410
 411         /* BPF verifier log settings */
 412         char *log_buf;
 413         size_t log_size;
 414         __u32 log_level;
 415
 416         struct bpf_object *obj;
 417
 418         int fd;
 419         bool autoload;
 420         bool mark_btf_static;
 421         enum bpf_prog_type type;
 422         enum bpf_attach_type expected_attach_type;
 423
 424         int prog_ifindex;
 425         __u32 attach_btf_obj_fd;
 426         __u32 attach_btf_id;
 427         __u32 attach_prog_fd;
 428
 429         void *func_info;
 430         __u32 func_info_rec_size;
 431         __u32 func_info_cnt;
 432
 433         void *line_info;
 434         __u32 line_info_rec_size;
 435         __u32 line_info_cnt;
 436         __u32 prog_flags;
 437 };
 438
 439 struct bpf_struct_ops {
 440         const char *tname;
 441         const struct btf_type *type;
 442         struct bpf_program **progs;
 443         __u32 *kern_func_off;
 444         /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
 445         void *data;
 446         /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
 447          *      btf_vmlinux's format.
 448          * struct bpf_struct_ops_tcp_congestion_ops {
 449          *      [... some other kernel fields ...]
 450          *      struct tcp_congestion_ops data;
 451          * }
 452          * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
 453          * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
 454          * from "data".
 455          */
 456         void *kern_vdata;
 457         __u32 type_id;
 458 };
 459
 460 #define DATA_SEC ".data"
 461 #define BSS_SEC ".bss"
 462 #define RODATA_SEC ".rodata"
 463 #define KCONFIG_SEC ".kconfig"
 464 #define KSYMS_SEC ".ksyms"
 465 #define STRUCT_OPS_SEC ".struct_ops"
 466
 467 enum libbpf_map_type {
 468         LIBBPF_MAP_UNSPEC,
 469         LIBBPF_MAP_DATA,
 470         LIBBPF_MAP_BSS,
 471         LIBBPF_MAP_RODATA,
 472         LIBBPF_MAP_KCONFIG,
 473 };
 474
 475 struct bpf_map_def {
 476         unsigned int type;
 477         unsigned int key_size;
 478         unsigned int value_size;
 479         unsigned int max_entries;
 480         unsigned int map_flags;
 481 };
 482
 483 struct bpf_map {
 484         struct bpf_object *obj;
 485         char *name;
 486         /* real_name is defined for special internal maps (.rodata*,
 487          * .data*, .bss, .kconfig) and preserves their original ELF section
 488          * name. This is important to be be able to find corresponding BTF
 489          * DATASEC information.
 490          */
 491         char *real_name;
 492         int fd;
 493         int sec_idx;
 494         size_t sec_offset;
 495         int map_ifindex;
 496         int inner_map_fd;
 497         struct bpf_map_def def;
 498         __u32 numa_node;
 499         __u32 btf_var_idx;
 500         __u32 btf_key_type_id;
 501         __u32 btf_value_type_id;
 502         __u32 btf_vmlinux_value_type_id;
 503         enum libbpf_map_type libbpf_type;
 504         void *mmaped;
 505         struct bpf_struct_ops *st_ops;
 506         struct bpf_map *inner_map;
 507         void **init_slots;
 508         int init_slots_sz;
 509         char *pin_path;
 510         bool pinned;
 511         bool reused;
 512         bool autocreate;
 513         __u64 map_extra;
 514 };
 515
 516 enum extern_type {
 517         EXT_UNKNOWN,
 518         EXT_KCFG,
 519         EXT_KSYM,
 520 };
 521
 522 enum kcfg_type {
 523         KCFG_UNKNOWN,
 524         KCFG_CHAR,
 525         KCFG_BOOL,
 526         KCFG_INT,
 527         KCFG_TRISTATE,
 528         KCFG_CHAR_ARR,
 529 };
 530
 531 struct extern_desc {
 532         enum extern_type type;
 533         int sym_idx;
 534         int btf_id;
 535         int sec_btf_id;
 536         const char *name;
 537         bool is_set;
 538         bool is_weak;
 539         union {
 540                 struct {
 541                         enum kcfg_type type;
 542                         int sz;
 543                         int align;
 544                         int data_off;
 545                         bool is_signed;
 546                 } kcfg;
 547                 struct {
 548                         unsigned long long addr;
 549
 550                         /* target btf_id of the corresponding kernel var. */
 551                         int kernel_btf_obj_fd;
 552                         int kernel_btf_id;
 553
 554                         /* local btf_id of the ksym extern's type. */
 555                         __u32 type_id;
 556                         /* BTF fd index to be patched in for insn->off, this is
 557                          * 0 for vmlinux BTF, index in obj->fd_array for module
 558                          * BTF
 559                          */
 560                         __s16 btf_fd_idx;
 561                 } ksym;
 562         };
 563 };
 564
 565 struct module_btf {
 566         struct btf *btf;
 567         char *name;
 568         __u32 id;
 569         int fd;
 570         int fd_array_idx;
 571 };
 572
 573 enum sec_type {
 574         SEC_UNUSED = 0,
 575         SEC_RELO,
 576         SEC_BSS,
 577         SEC_DATA,
 578         SEC_RODATA,
 579 };
 580
 581 struct elf_sec_desc {
 582         enum sec_type sec_type;
 583         Elf64_Shdr *shdr;
 584         Elf_Data *data;
 585 };
 586
 587 struct elf_state {
 588         int fd;
 589         const void *obj_buf;
 590         size_t obj_buf_sz;
 591         Elf *elf;
 592         Elf64_Ehdr *ehdr;
 593         Elf_Data *symbols;
 594         Elf_Data *st_ops_data;
 595         size_t shstrndx; /* section index for section name strings */
 596         size_t strtabidx;
 597         struct elf_sec_desc *secs;
 598         int sec_cnt;
 599         int btf_maps_shndx;
 600         __u32 btf_maps_sec_btf_id;
 601         int text_shndx;
 602         int symbols_shndx;
 603         int st_ops_shndx;
 604 };
 605
 606 struct usdt_manager;
 607
 608 struct bpf_object {
 609         char name[BPF_OBJ_NAME_LEN];
 610         char license[64];
 611         __u32 kern_version;
 612
 613         struct bpf_program *programs;
 614         size_t nr_programs;
 615         struct bpf_map *maps;
 616         size_t nr_maps;
 617         size_t maps_cap;
 618
 619         char *kconfig;
 620         struct extern_desc *externs;
 621         int nr_extern;
 622         int kconfig_map_idx;
 623
 624         bool loaded;
 625         bool has_subcalls;
 626         bool has_rodata;
 627
 628         struct bpf_gen *gen_loader;
 629
 630         /* Information when doing ELF related work. Only valid if efile.elf is not NULL */
 631         struct elf_state efile;
 632
 633         struct btf *btf;
 634         struct btf_ext *btf_ext;
 635
 636         /* Parse and load BTF vmlinux if any of the programs in the object need
 637          * it at load time.
 638          */
 639         struct btf *btf_vmlinux;
 640         /* Path to the custom BTF to be used for BPF CO-RE relocations as an
 641          * override for vmlinux BTF.
 642          */
 643         char *btf_custom_path;
 644         /* vmlinux BTF override for CO-RE relocations */
 645         struct btf *btf_vmlinux_override;
 646         /* Lazily initialized kernel module BTFs */
 647         struct module_btf *btf_modules;
 648         bool btf_modules_loaded;
 649         size_t btf_module_cnt;
 650         size_t btf_module_cap;
 651
 652         /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */
 653         char *log_buf;
 654         size_t log_size;
 655         __u32 log_level;
 656
 657         int *fd_array;
 658         size_t fd_array_cap;
 659         size_t fd_array_cnt;
 660
 661         struct usdt_manager *usdt_man;
 662
 663         char path[];
 664 };
 665
 666 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
 667 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
 668 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
 669 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
 670 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);
 671 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
 672 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
 673 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);
 674 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx);
 675
 676 void bpf_program__unload(struct bpf_program *prog)
 677 {
 678         if (!prog)
 679                 return;
 680
 681         zclose(prog->fd);
 682
 683         zfree(&prog->func_info);
 684         zfree(&prog->line_info);
 685 }
 686
 687 static void bpf_program__exit(struct bpf_program *prog)
 688 {
 689         if (!prog)
 690                 return;
 691
 692         bpf_program__unload(prog);
 693         zfree(&prog->name);
 694         zfree(&prog->sec_name);
 695         zfree(&prog->insns);
 696         zfree(&prog->reloc_desc);
 697
 698         prog->nr_reloc = 0;
 699         prog->insns_cnt = 0;
 700         prog->sec_idx = -1;
 701 }
 702
 703 static bool insn_is_subprog_call(const struct bpf_insn *insn)
 704 {
 705         return BPF_CLASS(insn->code) == BPF_JMP &&
 706                BPF_OP(insn->code) == BPF_CALL &&
 707                BPF_SRC(insn->code) == BPF_K &&
 708                insn->src_reg == BPF_PSEUDO_CALL &&
 709                insn->dst_reg == 0 &&
 710                insn->off == 0;
 711 }
 712
 713 static bool is_call_insn(const struct bpf_insn *insn)
 714 {
 715         return insn->code == (BPF_JMP | BPF_CALL);
 716 }
 717
 718 static bool insn_is_pseudo_func(struct bpf_insn *insn)
 719 {
 720         return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
 721 }
 722
 723 static int
 724 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
 725                       const char *name, size_t sec_idx, const char *sec_name,
 726                       size_t sec_off, void *insn_data, size_t insn_data_sz)
 727 {
 728         if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
 729                 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
 730                         sec_name, name, sec_off, insn_data_sz);
 731                 return -EINVAL;
 732         }
 733
 734         memset(prog, 0, sizeof(*prog));
 735         prog->obj = obj;
 736
 737         prog->sec_idx = sec_idx;
 738         prog->sec_insn_off = sec_off / BPF_INSN_SZ;
 739         prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
 740         /* insns_cnt can later be increased by appending used subprograms */
 741         prog->insns_cnt = prog->sec_insn_cnt;
 742
 743         prog->type = BPF_PROG_TYPE_UNSPEC;
 744         prog->fd = -1;
 745
 746         /* libbpf's convention for SEC("?abc...") is that it's just like
 747          * SEC("abc...") but the corresponding bpf_program starts out with
 748          * autoload set to false.
 749          */
 750         if (sec_name[0] == '?') {
 751                 prog->autoload = false;
 752                 /* from now on forget there was ? in section name */
 753                 sec_name++;
 754         } else {
 755                 prog->autoload = true;
 756         }
 757
 758         /* inherit object's log_level */
 759         prog->log_level = obj->log_level;
 760
 761         prog->sec_name = strdup(sec_name);
 762         if (!prog->sec_name)
 763                 goto errout;
 764
 765         prog->name = strdup(name);
 766         if (!prog->name)
 767                 goto errout;
 768
 769         prog->insns = malloc(insn_data_sz);
 770         if (!prog->insns)
 771                 goto errout;
 772         memcpy(prog->insns, insn_data, insn_data_sz);
 773
 774         return 0;
 775 errout:
 776         pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
 777         bpf_program__exit(prog);
 778         return -ENOMEM;
 779 }
 780
 781 static int
 782 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
 783                          const char *sec_name, int sec_idx)
 784 {
 785         Elf_Data *symbols = obj->efile.symbols;
 786         struct bpf_program *prog, *progs;
 787         void *data = sec_data->d_buf;
 788         size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
 789         int nr_progs, err, i;
 790         const char *name;
 791         Elf64_Sym *sym;
 792
 793         progs = obj->programs;
 794         nr_progs = obj->nr_programs;
 795         nr_syms = symbols->d_size / sizeof(Elf64_Sym);
 796         sec_off = 0;
 797
 798         for (i = 0; i < nr_syms; i++) {
 799                 sym = elf_sym_by_idx(obj, i);
 800
 801                 if (sym->st_shndx != sec_idx)
 802                         continue;
 803                 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
 804                         continue;
 805
 806                 prog_sz = sym->st_size;
 807                 sec_off = sym->st_value;
 808
 809                 name = elf_sym_str(obj, sym->st_name);
 810                 if (!name) {
 811                         pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
 812                                 sec_name, sec_off);
 813                         return -LIBBPF_ERRNO__FORMAT;
 814                 }
 815
 816                 if (sec_off + prog_sz > sec_sz) {
 817                         pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
 818                                 sec_name, sec_off);
 819                         return -LIBBPF_ERRNO__FORMAT;
 820                 }
 821
 822                 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
 823                         pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);
 824                         return -ENOTSUP;
 825                 }
 826
 827                 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
 828                          sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
 829
 830                 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
 831                 if (!progs) {
 832                         /*
 833                          * In this case the original obj->programs
 834                          * is still valid, so don't need special treat for
 835                          * bpf_close_object().
 836                          */
 837                         pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
 838                                 sec_name, name);
 839                         return -ENOMEM;
 840                 }
 841                 obj->programs = progs;
 842
 843                 prog = &progs[nr_progs];
 844
 845                 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
 846                                             sec_off, data + sec_off, prog_sz);
 847                 if (err)
 848                         return err;
 849
 850                 /* if function is a global/weak symbol, but has restricted
 851                  * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC
 852                  * as static to enable more permissive BPF verification mode
 853                  * with more outside context available to BPF verifier
 854                  */
 855                 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL
 856                     && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
 857                         || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
 858                         prog->mark_btf_static = true;
 859
 860                 nr_progs++;
 861                 obj->nr_programs = nr_progs;
 862         }
 863
 864         return 0;
 865 }
 866
 867 __u32 get_kernel_version(void)
 868 {
 869         /* On Ubuntu LINUX_VERSION_CODE doesn't correspond to info.release,
 870          * but Ubuntu provides /proc/version_signature file, as described at
 871          * https://ubuntu.com/kernel, with an example contents below, which we
 872          * can use to get a proper LINUX_VERSION_CODE.
 873          *
 874          *   Ubuntu 5.4.0-12.15-generic 5.4.8
 875          *
 876          * In the above, 5.4.8 is what kernel is actually expecting, while
 877          * uname() call will return 5.4.0 in info.release.
 878          */
 879         const char *ubuntu_kver_file = "/proc/version_signature";
 880         __u32 major, minor, patch;
 881         struct utsname info;
 882
 883         if (access(ubuntu_kver_file, R_OK) == 0) {
 884                 FILE *f;
 885
 886                 f = fopen(ubuntu_kver_file, "r");
 887                 if (f) {
 888                         if (fscanf(f, "%*s %*s %d.%d.%d\n", &major, &minor, &patch) == 3) {
 889                                 fclose(f);
 890                                 return KERNEL_VERSION(major, minor, patch);
 891                         }
 892                         fclose(f);
 893                 }
 894                 /* something went wrong, fall back to uname() approach */
 895         }
 896
 897         uname(&info);
 898         if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
 899                 return 0;
 900         return KERNEL_VERSION(major, minor, patch);
 901 }
 902
 903 static const struct btf_member *
 904 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
 905 {
 906         struct btf_member *m;
 907         int i;
 908
 909         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 910                 if (btf_member_bit_offset(t, i) == bit_offset)
 911                         return m;
 912         }
 913
 914         return NULL;
 915 }
 916
 917 static const struct btf_member *
 918 find_member_by_name(const struct btf *btf, const struct btf_type *t,
 919                     const char *name)
 920 {
 921         struct btf_member *m;
 922         int i;
 923
 924         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 925                 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
 926                         return m;
 927         }
 928
 929         return NULL;
 930 }
 931
 932 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
 933 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
 934                                    const char *name, __u32 kind);
 935
 936 static int
 937 find_struct_ops_kern_types(const struct btf *btf, const char *tname,
 938                            const struct btf_type **type, __u32 *type_id,
 939                            const struct btf_type **vtype, __u32 *vtype_id,
 940                            const struct btf_member **data_member)
 941 {
 942         const struct btf_type *kern_type, *kern_vtype;
 943         const struct btf_member *kern_data_member;
 944         __s32 kern_vtype_id, kern_type_id;
 945         __u32 i;
 946
 947         kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
 948         if (kern_type_id < 0) {
 949                 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
 950                         tname);
 951                 return kern_type_id;
 952         }
 953         kern_type = btf__type_by_id(btf, kern_type_id);
 954
 955         /* Find the corresponding "map_value" type that will be used
 956          * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,
 957          * find "struct bpf_struct_ops_tcp_congestion_ops" from the
 958          * btf_vmlinux.
 959          */
 960         kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
 961                                                 tname, BTF_KIND_STRUCT);
 962         if (kern_vtype_id < 0) {
 963                 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
 964                         STRUCT_OPS_VALUE_PREFIX, tname);
 965                 return kern_vtype_id;
 966         }
 967         kern_vtype = btf__type_by_id(btf, kern_vtype_id);
 968
 969         /* Find "struct tcp_congestion_ops" from
 970          * struct bpf_struct_ops_tcp_congestion_ops {
 971          *      [ ... ]
 972          *      struct tcp_congestion_ops data;
 973          * }
 974          */
 975         kern_data_member = btf_members(kern_vtype);
 976         for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
 977                 if (kern_data_member->type == kern_type_id)
 978                         break;
 979         }
 980         if (i == btf_vlen(kern_vtype)) {
 981                 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
 982                         tname, STRUCT_OPS_VALUE_PREFIX, tname);
 983                 return -EINVAL;
 984         }
 985
 986         *type = kern_type;
 987         *type_id = kern_type_id;
 988         *vtype = kern_vtype;
 989         *vtype_id = kern_vtype_id;
 990         *data_member = kern_data_member;
 991
 992         return 0;
 993 }
 994
 995 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
 996 {
 997         return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
 998 }
 999
1000 /* Init the map's fields that depend on kern_btf */
1001 static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
1002                                          const struct btf *btf,
1003                                          const struct btf *kern_btf)
1004 {
1005         const struct btf_member *member, *kern_member, *kern_data_member;
1006         const struct btf_type *type, *kern_type, *kern_vtype;
1007         __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
1008         struct bpf_struct_ops *st_ops;
1009         void *data, *kern_data;
1010         const char *tname;
1011         int err;
1012
1013         st_ops = map->st_ops;
1014         type = st_ops->type;
1015         tname = st_ops->tname;
1016         err = find_struct_ops_kern_types(kern_btf, tname,
1017                                          &kern_type, &kern_type_id,
1018                                          &kern_vtype, &kern_vtype_id,
1019                                          &kern_data_member);
1020         if (err)
1021                 return err;
1022
1023         pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
1024                  map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
1025
1026         map->def.value_size = kern_vtype->size;
1027         map->btf_vmlinux_value_type_id = kern_vtype_id;
1028
1029         st_ops->kern_vdata = calloc(1, kern_vtype->size);
1030         if (!st_ops->kern_vdata)
1031                 return -ENOMEM;
1032
1033         data = st_ops->data;
1034         kern_data_off = kern_data_member->offset / 8;
1035         kern_data = st_ops->kern_vdata + kern_data_off;
1036
1037         member = btf_members(type);
1038         for (i = 0; i < btf_vlen(type); i++, member++) {
1039                 const struct btf_type *mtype, *kern_mtype;
1040                 __u32 mtype_id, kern_mtype_id;
1041                 void *mdata, *kern_mdata;
1042                 __s64 msize, kern_msize;
1043                 __u32 moff, kern_moff;
1044                 __u32 kern_member_idx;
1045                 const char *mname;
1046
1047                 mname = btf__name_by_offset(btf, member->name_off);
1048                 kern_member = find_member_by_name(kern_btf, kern_type, mname);
1049                 if (!kern_member) {
1050                         pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
1051                                 map->name, mname);
1052                         return -ENOTSUP;
1053                 }
1054
1055                 kern_member_idx = kern_member - btf_members(kern_type);
1056                 if (btf_member_bitfield_size(type, i) ||
1057                     btf_member_bitfield_size(kern_type, kern_member_idx)) {
1058                         pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
1059                                 map->name, mname);
1060                         return -ENOTSUP;
1061                 }
1062
1063                 moff = member->offset / 8;
1064                 kern_moff = kern_member->offset / 8;
1065
1066                 mdata = data + moff;
1067                 kern_mdata = kern_data + kern_moff;
1068
1069                 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
1070                 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
1071                                                     &kern_mtype_id);
1072                 if (BTF_INFO_KIND(mtype->info) !=
1073                     BTF_INFO_KIND(kern_mtype->info)) {
1074                         pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
1075                                 map->name, mname, BTF_INFO_KIND(mtype->info),
1076                                 BTF_INFO_KIND(kern_mtype->info));
1077                         return -ENOTSUP;
1078                 }
1079
1080                 if (btf_is_ptr(mtype)) {
1081                         struct bpf_program *prog;
1082
1083                         prog = st_ops->progs[i];
1084                         if (!prog)
1085                                 continue;
1086
1087                         kern_mtype = skip_mods_and_typedefs(kern_btf,
1088                                                             kern_mtype->type,
1089                                                             &kern_mtype_id);
1090
1091                         /* mtype->type must be a func_proto which was
1092                          * guaranteed in bpf_object__collect_st_ops_relos(),
1093                          * so only check kern_mtype for func_proto here.
1094                          */
1095                         if (!btf_is_func_proto(kern_mtype)) {
1096                                 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
1097                                         map->name, mname);
1098                                 return -ENOTSUP;
1099                         }
1100
1101                         prog->attach_btf_id = kern_type_id;
1102                         prog->expected_attach_type = kern_member_idx;
1103
1104                         st_ops->kern_func_off[i] = kern_data_off + kern_moff;
1105
1106                         pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1107                                  map->name, mname, prog->name, moff,
1108                                  kern_moff);
1109
1110                         continue;
1111                 }
1112
1113                 msize = btf__resolve_size(btf, mtype_id);
1114                 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
1115                 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
1116                         pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
1117                                 map->name, mname, (ssize_t)msize,
1118                                 (ssize_t)kern_msize);
1119                         return -ENOTSUP;
1120                 }
1121
1122                 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
1123                          map->name, mname, (unsigned int)msize,
1124                          moff, kern_moff);
1125                 memcpy(kern_mdata, mdata, msize);
1126         }
1127
1128         return 0;
1129 }
1130
1131 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
1132 {
1133         struct bpf_map *map;
1134         size_t i;
1135         int err;
1136
1137         for (i = 0; i < obj->nr_maps; i++) {
1138                 map = &obj->maps[i];
1139
1140                 if (!bpf_map__is_struct_ops(map))
1141                         continue;
1142
1143                 err = bpf_map__init_kern_struct_ops(map, obj->btf,
1144                                                     obj->btf_vmlinux);
1145                 if (err)
1146                         return err;
1147         }
1148
1149         return 0;
1150 }
1151
1152 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
1153 {
1154         const struct btf_type *type, *datasec;
1155         const struct btf_var_secinfo *vsi;
1156         struct bpf_struct_ops *st_ops;
1157         const char *tname, *var_name;
1158         __s32 type_id, datasec_id;
1159         const struct btf *btf;
1160         struct bpf_map *map;
1161         __u32 i;
1162
1163         if (obj->efile.st_ops_shndx == -1)
1164                 return 0;
1165
1166         btf = obj->btf;
1167         datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
1168                                             BTF_KIND_DATASEC);
1169         if (datasec_id < 0) {
1170                 pr_warn("struct_ops init: DATASEC %s not found\n",
1171                         STRUCT_OPS_SEC);
1172                 return -EINVAL;
1173         }
1174
1175         datasec = btf__type_by_id(btf, datasec_id);
1176         vsi = btf_var_secinfos(datasec);
1177         for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
1178                 type = btf__type_by_id(obj->btf, vsi->type);
1179                 var_name = btf__name_by_offset(obj->btf, type->name_off);
1180
1181                 type_id = btf__resolve_type(obj->btf, vsi->type);
1182                 if (type_id < 0) {
1183                         pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
1184                                 vsi->type, STRUCT_OPS_SEC);
1185                         return -EINVAL;
1186                 }
1187
1188                 type = btf__type_by_id(obj->btf, type_id);
1189                 tname = btf__name_by_offset(obj->btf, type->name_off);
1190                 if (!tname[0]) {
1191                         pr_warn("struct_ops init: anonymous type is not supported\n");
1192                         return -ENOTSUP;
1193                 }
1194                 if (!btf_is_struct(type)) {
1195                         pr_warn("struct_ops init: %s is not a struct\n", tname);
1196                         return -EINVAL;
1197                 }
1198
1199                 map = bpf_object__add_map(obj);
1200                 if (IS_ERR(map))
1201                         return PTR_ERR(map);
1202
1203                 map->sec_idx = obj->efile.st_ops_shndx;
1204                 map->sec_offset = vsi->offset;
1205                 map->name = strdup(var_name);
1206                 if (!map->name)
1207                         return -ENOMEM;
1208
1209                 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1210                 map->def.key_size = sizeof(int);
1211                 map->def.value_size = type->size;
1212                 map->def.max_entries = 1;
1213
1214                 map->st_ops = calloc(1, sizeof(*map->st_ops));
1215                 if (!map->st_ops)
1216                         return -ENOMEM;
1217                 st_ops = map->st_ops;
1218                 st_ops->data = malloc(type->size);
1219                 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1220                 st_ops->kern_func_off = malloc(btf_vlen(type) *
1221                                                sizeof(*st_ops->kern_func_off));
1222                 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1223                         return -ENOMEM;
1224
1225                 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
1226                         pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1227                                 var_name, STRUCT_OPS_SEC);
1228                         return -EINVAL;
1229                 }
1230
1231                 memcpy(st_ops->data,
1232                        obj->efile.st_ops_data->d_buf + vsi->offset,
1233                        type->size);
1234                 st_ops->tname = tname;
1235                 st_ops->type = type;
1236                 st_ops->type_id = type_id;
1237
1238                 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1239                          tname, type_id, var_name, vsi->offset);
1240         }
1241
1242         return 0;
1243 }
1244
1245 static struct bpf_object *bpf_object__new(const char *path,
1246                                           const void *obj_buf,
1247                                           size_t obj_buf_sz,
1248                                           const char *obj_name)
1249 {
1250         struct bpf_object *obj;
1251         char *end;
1252
1253         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1254         if (!obj) {
1255                 pr_warn("alloc memory failed for %s\n", path);
1256                 return ERR_PTR(-ENOMEM);
1257         }
1258
1259         strcpy(obj->path, path);
1260         if (obj_name) {
1261                 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1262         } else {
1263                 /* Using basename() GNU version which doesn't modify arg. */
1264                 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1265                 end = strchr(obj->name, '.');
1266                 if (end)
1267                         *end = 0;
1268         }
1269
1270         obj->efile.fd = -1;
1271         /*
1272          * Caller of this function should also call
1273          * bpf_object__elf_finish() after data collection to return
1274          * obj_buf to user. If not, we should duplicate the buffer to
1275          * avoid user freeing them before elf finish.
1276          */
1277         obj->efile.obj_buf = obj_buf;
1278         obj->efile.obj_buf_sz = obj_buf_sz;
1279         obj->efile.btf_maps_shndx = -1;
1280         obj->efile.st_ops_shndx = -1;
1281         obj->kconfig_map_idx = -1;
1282
1283         obj->kern_version = get_kernel_version();
1284         obj->loaded = false;
1285
1286         return obj;
1287 }
1288
1289 static void bpf_object__elf_finish(struct bpf_object *obj)
1290 {
1291         if (!obj->efile.elf)
1292                 return;
1293
1294         elf_end(obj->efile.elf);
1295         obj->efile.elf = NULL;
1296         obj->efile.symbols = NULL;
1297         obj->efile.st_ops_data = NULL;
1298
1299         zfree(&obj->efile.secs);
1300         obj->efile.sec_cnt = 0;
1301         zclose(obj->efile.fd);
1302         obj->efile.obj_buf = NULL;
1303         obj->efile.obj_buf_sz = 0;
1304 }
1305
1306 static int bpf_object__elf_init(struct bpf_object *obj)
1307 {
1308         Elf64_Ehdr *ehdr;
1309         int err = 0;
1310         Elf *elf;
1311
1312         if (obj->efile.elf) {
1313                 pr_warn("elf: init internal error\n");
1314                 return -LIBBPF_ERRNO__LIBELF;
1315         }
1316
1317         if (obj->efile.obj_buf_sz > 0) {
1318                 /* obj_buf should have been validated by bpf_object__open_mem(). */
1319                 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1320         } else {
1321                 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1322                 if (obj->efile.fd < 0) {
1323                         char errmsg[STRERR_BUFSIZE], *cp;
1324
1325                         err = -errno;
1326                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1327                         pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1328                         return err;
1329                 }
1330
1331                 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1332         }
1333
1334         if (!elf) {
1335                 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1336                 err = -LIBBPF_ERRNO__LIBELF;
1337                 goto errout;
1338         }
1339
1340         obj->efile.elf = elf;
1341
1342         if (elf_kind(elf) != ELF_K_ELF) {
1343                 err = -LIBBPF_ERRNO__FORMAT;
1344                 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1345                 goto errout;
1346         }
1347
1348         if (gelf_getclass(elf) != ELFCLASS64) {
1349                 err = -LIBBPF_ERRNO__FORMAT;
1350                 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1351                 goto errout;
1352         }
1353
1354         obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1355         if (!obj->efile.ehdr) {
1356                 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1357                 err = -LIBBPF_ERRNO__FORMAT;
1358                 goto errout;
1359         }
1360
1361         if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1362                 pr_warn("elf: failed to get section names section index for %s: %s\n",
1363                         obj->path, elf_errmsg(-1));
1364                 err = -LIBBPF_ERRNO__FORMAT;
1365                 goto errout;
1366         }
1367
1368         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1369         if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1370                 pr_warn("elf: failed to get section names strings from %s: %s\n",
1371                         obj->path, elf_errmsg(-1));
1372                 err = -LIBBPF_ERRNO__FORMAT;
1373                 goto errout;
1374         }
1375
1376         /* Old LLVM set e_machine to EM_NONE */
1377         if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1378                 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1379                 err = -LIBBPF_ERRNO__FORMAT;
1380                 goto errout;
1381         }
1382
1383         return 0;
1384 errout:
1385         bpf_object__elf_finish(obj);
1386         return err;
1387 }
1388
1389 static int bpf_object__check_endianness(struct bpf_object *obj)
1390 {
1391 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1392         if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1393                 return 0;
1394 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1395         if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1396                 return 0;
1397 #else
1398 # error "Unrecognized __BYTE_ORDER__"
1399 #endif
1400         pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1401         return -LIBBPF_ERRNO__ENDIAN;
1402 }
1403
1404 static int
1405 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1406 {
1407         /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1408          * go over allowed ELF data section buffer
1409          */
1410         libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1411         pr_debug("license of %s is %s\n", obj->path, obj->license);
1412         return 0;
1413 }
1414
1415 static int
1416 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1417 {
1418         __u32 kver;
1419
1420         if (size != sizeof(kver)) {
1421                 pr_warn("invalid kver section in %s\n", obj->path);
1422                 return -LIBBPF_ERRNO__FORMAT;
1423         }
1424         memcpy(&kver, data, sizeof(kver));
1425         obj->kern_version = kver;
1426         pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1427         return 0;
1428 }
1429
1430 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1431 {
1432         if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1433             type == BPF_MAP_TYPE_HASH_OF_MAPS)
1434                 return true;
1435         return false;
1436 }
1437
1438 static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size)
1439 {
1440         Elf_Data *data;
1441         Elf_Scn *scn;
1442
1443         if (!name)
1444                 return -EINVAL;
1445
1446         scn = elf_sec_by_name(obj, name);
1447         data = elf_sec_data(obj, scn);
1448         if (data) {
1449                 *size = data->d_size;
1450                 return 0; /* found it */
1451         }
1452
1453         return -ENOENT;
1454 }
1455
1456 static int find_elf_var_offset(const struct bpf_object *obj, const char *name, __u32 *off)
1457 {
1458         Elf_Data *symbols = obj->efile.symbols;
1459         const char *sname;
1460         size_t si;
1461
1462         if (!name || !off)
1463                 return -EINVAL;
1464
1465         for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1466                 Elf64_Sym *sym = elf_sym_by_idx(obj, si);
1467
1468                 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1469                         continue;
1470
1471                 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1472                     ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1473                         continue;
1474
1475                 sname = elf_sym_str(obj, sym->st_name);
1476                 if (!sname) {
1477                         pr_warn("failed to get sym name string for var %s\n", name);
1478                         return -EIO;
1479                 }
1480                 if (strcmp(name, sname) == 0) {
1481                         *off = sym->st_value;
1482                         return 0;
1483                 }
1484         }
1485
1486         return -ENOENT;
1487 }
1488
1489 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1490 {
1491         struct bpf_map *map;
1492         int err;
1493
1494         err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1495                                 sizeof(*obj->maps), obj->nr_maps + 1);
1496         if (err)
1497                 return ERR_PTR(err);
1498
1499         map = &obj->maps[obj->nr_maps++];
1500         map->obj = obj;
1501         map->fd = -1;
1502         map->inner_map_fd = -1;
1503         map->autocreate = true;
1504
1505         return map;
1506 }
1507
1508 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1509 {
1510         long page_sz = sysconf(_SC_PAGE_SIZE);
1511         size_t map_sz;
1512
1513         map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1514         map_sz = roundup(map_sz, page_sz);
1515         return map_sz;
1516 }
1517
1518 static char *internal_map_name(struct bpf_object *obj, const char *real_name)
1519 {
1520         char map_name[BPF_OBJ_NAME_LEN], *p;
1521         int pfx_len, sfx_len = max((size_t)7, strlen(real_name));
1522
1523         /* This is one of the more confusing parts of libbpf for various
1524          * reasons, some of which are historical. The original idea for naming
1525          * internal names was to include as much of BPF object name prefix as
1526          * possible, so that it can be distinguished from similar internal
1527          * maps of a different BPF object.
1528          * As an example, let's say we have bpf_object named 'my_object_name'
1529          * and internal map corresponding to '.rodata' ELF section. The final
1530          * map name advertised to user and to the kernel will be
1531          * 'my_objec.rodata', taking first 8 characters of object name and
1532          * entire 7 characters of '.rodata'.
1533          * Somewhat confusingly, if internal map ELF section name is shorter
1534          * than 7 characters, e.g., '.bss', we still reserve 7 characters
1535          * for the suffix, even though we only have 4 actual characters, and
1536          * resulting map will be called 'my_objec.bss', not even using all 15
1537          * characters allowed by the kernel. Oh well, at least the truncated
1538          * object name is somewhat consistent in this case. But if the map
1539          * name is '.kconfig', we'll still have entirety of '.kconfig' added
1540          * (8 chars) and thus will be left with only first 7 characters of the
1541          * object name ('my_obje'). Happy guessing, user, that the final map
1542          * name will be "my_obje.kconfig".
1543          * Now, with libbpf starting to support arbitrarily named .rodata.*
1544          * and .data.* data sections, it's possible that ELF section name is
1545          * longer than allowed 15 chars, so we now need to be careful to take
1546          * only up to 15 first characters of ELF name, taking no BPF object
1547          * name characters at all. So '.rodata.abracadabra' will result in
1548          * '.rodata.abracad' kernel and user-visible name.
1549          * We need to keep this convoluted logic intact for .data, .bss and
1550          * .rodata maps, but for new custom .data.custom and .rodata.custom
1551          * maps we use their ELF names as is, not prepending bpf_object name
1552          * in front. We still need to truncate them to 15 characters for the
1553          * kernel. Full name can be recovered for such maps by using DATASEC
1554          * BTF type associated with such map's value type, though.
1555          */
1556         if (sfx_len >= BPF_OBJ_NAME_LEN)
1557                 sfx_len = BPF_OBJ_NAME_LEN - 1;
1558
1559         /* if there are two or more dots in map name, it's a custom dot map */
1560         if (strchr(real_name + 1, '.') != NULL)
1561                 pfx_len = 0;
1562         else
1563                 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1564
1565         snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1566                  sfx_len, real_name);
1567
1568         /* sanitise map name to characters allowed by kernel */
1569         for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1570                 if (!isalnum(*p) && *p != '_' && *p != '.')
1571                         *p = '_';
1572
1573         return strdup(map_name);
1574 }
1575
1576 static int
1577 bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map);
1578
1579 static int
1580 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1581                               const char *real_name, int sec_idx, void *data, size_t data_sz)
1582 {
1583         struct bpf_map_def *def;
1584         struct bpf_map *map;
1585         int err;
1586
1587         map = bpf_object__add_map(obj);
1588         if (IS_ERR(map))
1589                 return PTR_ERR(map);
1590
1591         map->libbpf_type = type;
1592         map->sec_idx = sec_idx;
1593         map->sec_offset = 0;
1594         map->real_name = strdup(real_name);
1595         map->name = internal_map_name(obj, real_name);
1596         if (!map->real_name || !map->name) {
1597                 zfree(&map->real_name);
1598                 zfree(&map->name);
1599                 return -ENOMEM;
1600         }
1601
1602         def = &map->def;
1603         def->type = BPF_MAP_TYPE_ARRAY;
1604         def->key_size = sizeof(int);
1605         def->value_size = data_sz;
1606         def->max_entries = 1;
1607         def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1608                          ? BPF_F_RDONLY_PROG : 0;
1609         def->map_flags |= BPF_F_MMAPABLE;
1610
1611         pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1612                  map->name, map->sec_idx, map->sec_offset, def->map_flags);
1613
1614         map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1615                            MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1616         if (map->mmaped == MAP_FAILED) {
1617                 err = -errno;
1618                 map->mmaped = NULL;
1619                 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1620                         map->name, err);
1621                 zfree(&map->real_name);
1622                 zfree(&map->name);
1623                 return err;
1624         }
1625
1626         /* failures are fine because of maps like .rodata.str1.1 */
1627         (void) bpf_map_find_btf_info(obj, map);
1628
1629         if (data)
1630                 memcpy(map->mmaped, data, data_sz);
1631
1632         pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1633         return 0;
1634 }
1635
1636 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1637 {
1638         struct elf_sec_desc *sec_desc;
1639         const char *sec_name;
1640         int err = 0, sec_idx;
1641
1642         /*
1643          * Populate obj->maps with libbpf internal maps.
1644          */
1645         for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1646                 sec_desc = &obj->efile.secs[sec_idx];
1647
1648                 /* Skip recognized sections with size 0. */
1649                 if (sec_desc->data && sec_desc->data->d_size == 0)
1650                         continue;
1651
1652                 switch (sec_desc->sec_type) {
1653                 case SEC_DATA:
1654                         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1655                         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1656                                                             sec_name, sec_idx,
1657                                                             sec_desc->data->d_buf,
1658                                                             sec_desc->data->d_size);
1659                         break;
1660                 case SEC_RODATA:
1661                         obj->has_rodata = true;
1662                         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1663                         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1664                                                             sec_name, sec_idx,
1665                                                             sec_desc->data->d_buf,
1666                                                             sec_desc->data->d_size);
1667                         break;
1668                 case SEC_BSS:
1669                         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1670                         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1671                                                             sec_name, sec_idx,
1672                                                             NULL,
1673                                                             sec_desc->data->d_size);
1674                         break;
1675                 default:
1676                         /* skip */
1677                         break;
1678                 }
1679                 if (err)
1680                         return err;
1681         }
1682         return 0;
1683 }
1684
1685
1686 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1687                                                const void *name)
1688 {
1689         int i;
1690
1691         for (i = 0; i < obj->nr_extern; i++) {
1692                 if (strcmp(obj->externs[i].name, name) == 0)
1693                         return &obj->externs[i];
1694         }
1695         return NULL;
1696 }
1697
1698 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1699                               char value)
1700 {
1701         switch (ext->kcfg.type) {
1702         case KCFG_BOOL:
1703                 if (value == 'm') {
1704                         pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",
1705                                 ext->name, value);
1706                         return -EINVAL;
1707                 }
1708                 *(bool *)ext_val = value == 'y' ? true : false;
1709                 break;
1710         case KCFG_TRISTATE:
1711                 if (value == 'y')
1712                         *(enum libbpf_tristate *)ext_val = TRI_YES;
1713                 else if (value == 'm')
1714                         *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1715                 else /* value == 'n' */
1716                         *(enum libbpf_tristate *)ext_val = TRI_NO;
1717                 break;
1718         case KCFG_CHAR:
1719                 *(char *)ext_val = value;
1720                 break;
1721         case KCFG_UNKNOWN:
1722         case KCFG_INT:
1723         case KCFG_CHAR_ARR:
1724         default:
1725                 pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",
1726                         ext->name, value);
1727                 return -EINVAL;
1728         }
1729         ext->is_set = true;
1730         return 0;
1731 }
1732
1733 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1734                               const char *value)
1735 {
1736         size_t len;
1737
1738         if (ext->kcfg.type != KCFG_CHAR_ARR) {
1739                 pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",
1740                         ext->name, value);
1741                 return -EINVAL;
1742         }
1743
1744         len = strlen(value);
1745         if (value[len - 1] != '"') {
1746                 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1747                         ext->name, value);
1748                 return -EINVAL;
1749         }
1750
1751         /* strip quotes */
1752         len -= 2;
1753         if (len >= ext->kcfg.sz) {
1754                 pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",
1755                         ext->name, value, len, ext->kcfg.sz - 1);
1756                 len = ext->kcfg.sz - 1;
1757         }
1758         memcpy(ext_val, value + 1, len);
1759         ext_val[len] = '\0';
1760         ext->is_set = true;
1761         return 0;
1762 }
1763
1764 static int parse_u64(const char *value, __u64 *res)
1765 {
1766         char *value_end;
1767         int err;
1768
1769         errno = 0;
1770         *res = strtoull(value, &value_end, 0);
1771         if (errno) {
1772                 err = -errno;
1773                 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1774                 return err;
1775         }
1776         if (*value_end) {
1777                 pr_warn("failed to parse '%s' as integer completely\n", value);
1778                 return -EINVAL;
1779         }
1780         return 0;
1781 }
1782
1783 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1784 {
1785         int bit_sz = ext->kcfg.sz * 8;
1786
1787         if (ext->kcfg.sz == 8)
1788                 return true;
1789
1790         /* Validate that value stored in u64 fits in integer of `ext->sz`
1791          * bytes size without any loss of information. If the target integer
1792          * is signed, we rely on the following limits of integer type of
1793          * Y bits and subsequent transformation:
1794          *
1795          *     -2^(Y-1) <= X           <= 2^(Y-1) - 1
1796          *            0 <= X + 2^(Y-1) <= 2^Y - 1
1797          *            0 <= X + 2^(Y-1) <  2^Y
1798          *
1799          *  For unsigned target integer, check that all the (64 - Y) bits are
1800          *  zero.
1801          */
1802         if (ext->kcfg.is_signed)
1803                 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1804         else
1805                 return (v >> bit_sz) == 0;
1806 }
1807
1808 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1809                               __u64 value)
1810 {
1811         if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
1812             ext->kcfg.type != KCFG_BOOL) {
1813                 pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",
1814                         ext->name, (unsigned long long)value);
1815                 return -EINVAL;
1816         }
1817         if (ext->kcfg.type == KCFG_BOOL && value > 1) {
1818                 pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",
1819                         ext->name, (unsigned long long)value);
1820                 return -EINVAL;
1821
1822         }
1823         if (!is_kcfg_value_in_range(ext, value)) {
1824                 pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",
1825                         ext->name, (unsigned long long)value, ext->kcfg.sz);
1826                 return -ERANGE;
1827         }
1828         switch (ext->kcfg.sz) {
1829                 case 1: *(__u8 *)ext_val = value; break;
1830                 case 2: *(__u16 *)ext_val = value; break;
1831                 case 4: *(__u32 *)ext_val = value; break;
1832                 case 8: *(__u64 *)ext_val = value; break;
1833                 default:
1834                         return -EINVAL;
1835         }
1836         ext->is_set = true;
1837         return 0;
1838 }
1839
1840 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1841                                             char *buf, void *data)
1842 {
1843         struct extern_desc *ext;
1844         char *sep, *value;
1845         int len, err = 0;
1846         void *ext_val;
1847         __u64 num;
1848
1849         if (!str_has_pfx(buf, "CONFIG_"))
1850                 return 0;
1851
1852         sep = strchr(buf, '=');
1853         if (!sep) {
1854                 pr_warn("failed to parse '%s': no separator\n", buf);
1855                 return -EINVAL;
1856         }
1857
1858         /* Trim ending '\n' */
1859         len = strlen(buf);
1860         if (buf[len - 1] == '\n')
1861                 buf[len - 1] = '\0';
1862         /* Split on '=' and ensure that a value is present. */
1863         *sep = '\0';
1864         if (!sep[1]) {
1865                 *sep = '=';
1866                 pr_warn("failed to parse '%s': no value\n", buf);
1867                 return -EINVAL;
1868         }
1869
1870         ext = find_extern_by_name(obj, buf);
1871         if (!ext || ext->is_set)
1872                 return 0;
1873
1874         ext_val = data + ext->kcfg.data_off;
1875         value = sep + 1;
1876
1877         switch (*value) {
1878         case 'y': case 'n': case 'm':
1879                 err = set_kcfg_value_tri(ext, ext_val, *value);
1880                 break;
1881         case '"':
1882                 err = set_kcfg_value_str(ext, ext_val, value);
1883                 break;
1884         default:
1885                 /* assume integer */
1886                 err = parse_u64(value, &num);
1887                 if (err) {
1888                         pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
1889                         return err;
1890                 }
1891                 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1892                         pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
1893                         return -EINVAL;
1894                 }
1895                 err = set_kcfg_value_num(ext, ext_val, num);
1896                 break;
1897         }
1898         if (err)
1899                 return err;
1900         pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
1901         return 0;
1902 }
1903
1904 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1905 {
1906         char buf[PATH_MAX];
1907         struct utsname uts;
1908         int len, err = 0;
1909         gzFile file;
1910
1911         uname(&uts);
1912         len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1913         if (len < 0)
1914                 return -EINVAL;
1915         else if (len >= PATH_MAX)
1916                 return -ENAMETOOLONG;
1917
1918         /* gzopen also accepts uncompressed files. */
1919         file = gzopen(buf, "r");
1920         if (!file)
1921                 file = gzopen("/proc/config.gz", "r");
1922
1923         if (!file) {
1924                 pr_warn("failed to open system Kconfig\n");
1925                 return -ENOENT;
1926         }
1927
1928         while (gzgets(file, buf, sizeof(buf))) {
1929                 err = bpf_object__process_kconfig_line(obj, buf, data);
1930                 if (err) {
1931                         pr_warn("error parsing system Kconfig line '%s': %d\n",
1932                                 buf, err);
1933                         goto out;
1934                 }
1935         }
1936
1937 out:
1938         gzclose(file);
1939         return err;
1940 }
1941
1942 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1943                                         const char *config, void *data)
1944 {
1945         char buf[PATH_MAX];
1946         int err = 0;
1947         FILE *file;
1948
1949         file = fmemopen((void *)config, strlen(config), "r");
1950         if (!file) {
1951                 err = -errno;
1952                 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1953                 return err;
1954         }
1955
1956         while (fgets(buf, sizeof(buf), file)) {
1957                 err = bpf_object__process_kconfig_line(obj, buf, data);
1958                 if (err) {
1959                         pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1960                                 buf, err);
1961                         break;
1962                 }
1963         }
1964
1965         fclose(file);
1966         return err;
1967 }
1968
1969 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1970 {
1971         struct extern_desc *last_ext = NULL, *ext;
1972         size_t map_sz;
1973         int i, err;
1974
1975         for (i = 0; i < obj->nr_extern; i++) {
1976                 ext = &obj->externs[i];
1977                 if (ext->type == EXT_KCFG)
1978                         last_ext = ext;
1979         }
1980
1981         if (!last_ext)
1982                 return 0;
1983
1984         map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
1985         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1986                                             ".kconfig", obj->efile.symbols_shndx,
1987                                             NULL, map_sz);
1988         if (err)
1989                 return err;
1990
1991         obj->kconfig_map_idx = obj->nr_maps - 1;
1992
1993         return 0;
1994 }
1995
1996 const struct btf_type *
1997 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1998 {
1999         const struct btf_type *t = btf__type_by_id(btf, id);
2000
2001         if (res_id)
2002                 *res_id = id;
2003
2004         while (btf_is_mod(t) || btf_is_typedef(t)) {
2005                 if (res_id)
2006                         *res_id = t->type;
2007                 t = btf__type_by_id(btf, t->type);
2008         }
2009
2010         return t;
2011 }
2012
2013 static const struct btf_type *
2014 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
2015 {
2016         const struct btf_type *t;
2017
2018         t = skip_mods_and_typedefs(btf, id, NULL);
2019         if (!btf_is_ptr(t))
2020                 return NULL;
2021
2022         t = skip_mods_and_typedefs(btf, t->type, res_id);
2023
2024         return btf_is_func_proto(t) ? t : NULL;
2025 }
2026
2027 static const char *__btf_kind_str(__u16 kind)
2028 {
2029         switch (kind) {
2030         case BTF_KIND_UNKN: return "void";
2031         case BTF_KIND_INT: return "int";
2032         case BTF_KIND_PTR: return "ptr";
2033         case BTF_KIND_ARRAY: return "array";
2034         case BTF_KIND_STRUCT: return "struct";
2035         case BTF_KIND_UNION: return "union";
2036         case BTF_KIND_ENUM: return "enum";
2037         case BTF_KIND_FWD: return "fwd";
2038         case BTF_KIND_TYPEDEF: return "typedef";
2039         case BTF_KIND_VOLATILE: return "volatile";
2040         case BTF_KIND_CONST: return "const";
2041         case BTF_KIND_RESTRICT: return "restrict";
2042         case BTF_KIND_FUNC: return "func";
2043         case BTF_KIND_FUNC_PROTO: return "func_proto";
2044         case BTF_KIND_VAR: return "var";
2045         case BTF_KIND_DATASEC: return "datasec";
2046         case BTF_KIND_FLOAT: return "float";
2047         case BTF_KIND_DECL_TAG: return "decl_tag";
2048         case BTF_KIND_TYPE_TAG: return "type_tag";
2049         case BTF_KIND_ENUM64: return "enum64";
2050         default: return "unknown";
2051         }
2052 }
2053
2054 const char *btf_kind_str(const struct btf_type *t)
2055 {
2056         return __btf_kind_str(btf_kind(t));
2057 }
2058
2059 /*
2060  * Fetch integer attribute of BTF map definition. Such attributes are
2061  * represented using a pointer to an array, in which dimensionality of array
2062  * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
2063  * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
2064  * type definition, while using only sizeof(void *) space in ELF data section.
2065  */
2066 static bool get_map_field_int(const char *map_name, const struct btf *btf,
2067                               const struct btf_member *m, __u32 *res)
2068 {
2069         const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2070         const char *name = btf__name_by_offset(btf, m->name_off);
2071         const struct btf_array *arr_info;
2072         const struct btf_type *arr_t;
2073
2074         if (!btf_is_ptr(t)) {
2075                 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
2076                         map_name, name, btf_kind_str(t));
2077                 return false;
2078         }
2079
2080         arr_t = btf__type_by_id(btf, t->type);
2081         if (!arr_t) {
2082                 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
2083                         map_name, name, t->type);
2084                 return false;
2085         }
2086         if (!btf_is_array(arr_t)) {
2087                 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
2088                         map_name, name, btf_kind_str(arr_t));
2089                 return false;
2090         }
2091         arr_info = btf_array(arr_t);
2092         *res = arr_info->nelems;
2093         return true;
2094 }
2095
2096 static int build_map_pin_path(struct bpf_map *map, const char *path)
2097 {
2098         char buf[PATH_MAX];
2099         int len;
2100
2101         if (!path)
2102                 path = "/sys/fs/bpf";
2103
2104         len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
2105         if (len < 0)
2106                 return -EINVAL;
2107         else if (len >= PATH_MAX)
2108                 return -ENAMETOOLONG;
2109
2110         return bpf_map__set_pin_path(map, buf);
2111 }
2112
2113 /* should match definition in bpf_helpers.h */
2114 enum libbpf_pin_type {
2115         LIBBPF_PIN_NONE,
2116         /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
2117         LIBBPF_PIN_BY_NAME,
2118 };
2119
2120 int parse_btf_map_def(const char *map_name, struct btf *btf,
2121                       const struct btf_type *def_t, bool strict,
2122                       struct btf_map_def *map_def, struct btf_map_def *inner_def)
2123 {
2124         const struct btf_type *t;
2125         const struct btf_member *m;
2126         bool is_inner = inner_def == NULL;
2127         int vlen, i;
2128
2129         vlen = btf_vlen(def_t);
2130         m = btf_members(def_t);
2131         for (i = 0; i < vlen; i++, m++) {
2132                 const char *name = btf__name_by_offset(btf, m->name_off);
2133
2134                 if (!name) {
2135                         pr_warn("map '%s': invalid field #%d.\n", map_name, i);
2136                         return -EINVAL;
2137                 }
2138                 if (strcmp(name, "type") == 0) {
2139                         if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2140                                 return -EINVAL;
2141                         map_def->parts |= MAP_DEF_MAP_TYPE;
2142                 } else if (strcmp(name, "max_entries") == 0) {
2143                         if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2144                                 return -EINVAL;
2145                         map_def->parts |= MAP_DEF_MAX_ENTRIES;
2146                 } else if (strcmp(name, "map_flags") == 0) {
2147                         if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2148                                 return -EINVAL;
2149                         map_def->parts |= MAP_DEF_MAP_FLAGS;
2150                 } else if (strcmp(name, "numa_node") == 0) {
2151                         if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2152                                 return -EINVAL;
2153                         map_def->parts |= MAP_DEF_NUMA_NODE;
2154                 } else if (strcmp(name, "key_size") == 0) {
2155                         __u32 sz;
2156
2157                         if (!get_map_field_int(map_name, btf, m, &sz))
2158                                 return -EINVAL;
2159                         if (map_def->key_size && map_def->key_size != sz) {
2160                                 pr_warn("map '%s': conflicting key size %u != %u.\n",
2161                                         map_name, map_def->key_size, sz);
2162                                 return -EINVAL;
2163                         }
2164                         map_def->key_size = sz;
2165                         map_def->parts |= MAP_DEF_KEY_SIZE;
2166                 } else if (strcmp(name, "key") == 0) {
2167                         __s64 sz;
2168
2169                         t = btf__type_by_id(btf, m->type);
2170                         if (!t) {
2171                                 pr_warn("map '%s': key type [%d] not found.\n",
2172                                         map_name, m->type);
2173                                 return -EINVAL;
2174                         }
2175                         if (!btf_is_ptr(t)) {
2176                                 pr_warn("map '%s': key spec is not PTR: %s.\n",
2177                                         map_name, btf_kind_str(t));
2178                                 return -EINVAL;
2179                         }
2180                         sz = btf__resolve_size(btf, t->type);
2181                         if (sz < 0) {
2182                                 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2183                                         map_name, t->type, (ssize_t)sz);
2184                                 return sz;
2185                         }
2186                         if (map_def->key_size && map_def->key_size != sz) {
2187                                 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2188                                         map_name, map_def->key_size, (ssize_t)sz);
2189                                 return -EINVAL;
2190                         }
2191                         map_def->key_size = sz;
2192                         map_def->key_type_id = t->type;
2193                         map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2194                 } else if (strcmp(name, "value_size") == 0) {
2195                         __u32 sz;
2196
2197                         if (!get_map_field_int(map_name, btf, m, &sz))
2198                                 return -EINVAL;
2199                         if (map_def->value_size && map_def->value_size != sz) {
2200                                 pr_warn("map '%s': conflicting value size %u != %u.\n",
2201                                         map_name, map_def->value_size, sz);
2202                                 return -EINVAL;
2203                         }
2204                         map_def->value_size = sz;
2205                         map_def->parts |= MAP_DEF_VALUE_SIZE;
2206                 } else if (strcmp(name, "value") == 0) {
2207                         __s64 sz;
2208
2209                         t = btf__type_by_id(btf, m->type);
2210                         if (!t) {
2211                                 pr_warn("map '%s': value type [%d] not found.\n",
2212                                         map_name, m->type);
2213                                 return -EINVAL;
2214                         }
2215                         if (!btf_is_ptr(t)) {
2216                                 pr_warn("map '%s': value spec is not PTR: %s.\n",
2217                                         map_name, btf_kind_str(t));
2218                                 return -EINVAL;
2219                         }
2220                         sz = btf__resolve_size(btf, t->type);
2221                         if (sz < 0) {
2222                                 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2223                                         map_name, t->type, (ssize_t)sz);
2224                                 return sz;
2225                         }
2226                         if (map_def->value_size && map_def->value_size != sz) {
2227                                 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2228                                         map_name, map_def->value_size, (ssize_t)sz);
2229                                 return -EINVAL;
2230                         }
2231                         map_def->value_size = sz;
2232                         map_def->value_type_id = t->type;
2233                         map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2234                 }
2235                 else if (strcmp(name, "values") == 0) {
2236                         bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2237                         bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2238                         const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2239                         char inner_map_name[128];
2240                         int err;
2241
2242                         if (is_inner) {
2243                                 pr_warn("map '%s': multi-level inner maps not supported.\n",
2244                                         map_name);
2245                                 return -ENOTSUP;
2246                         }
2247                         if (i != vlen - 1) {
2248                                 pr_warn("map '%s': '%s' member should be last.\n",
2249                                         map_name, name);
2250                                 return -EINVAL;
2251                         }
2252                         if (!is_map_in_map && !is_prog_array) {
2253                                 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2254                                         map_name);
2255                                 return -ENOTSUP;
2256                         }
2257                         if (map_def->value_size && map_def->value_size != 4) {
2258                                 pr_warn("map '%s': conflicting value size %u != 4.\n",
2259                                         map_name, map_def->value_size);
2260                                 return -EINVAL;
2261                         }
2262                         map_def->value_size = 4;
2263                         t = btf__type_by_id(btf, m->type);
2264                         if (!t) {
2265                                 pr_warn("map '%s': %s type [%d] not found.\n",
2266                                         map_name, desc, m->type);
2267                                 return -EINVAL;
2268                         }
2269                         if (!btf_is_array(t) || btf_array(t)->nelems) {
2270                                 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2271                                         map_name, desc);
2272                                 return -EINVAL;
2273                         }
2274                         t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2275                         if (!btf_is_ptr(t)) {
2276                                 pr_warn("map '%s': %s def is of unexpected kind %s.\n",
2277                                         map_name, desc, btf_kind_str(t));
2278                                 return -EINVAL;
2279                         }
2280                         t = skip_mods_and_typedefs(btf, t->type, NULL);
2281                         if (is_prog_array) {
2282                                 if (!btf_is_func_proto(t)) {
2283                                         pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2284                                                 map_name, btf_kind_str(t));
2285                                         return -EINVAL;
2286                                 }
2287                                 continue;
2288                         }
2289                         if (!btf_is_struct(t)) {
2290                                 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2291                                         map_name, btf_kind_str(t));
2292                                 return -EINVAL;
2293                         }
2294
2295                         snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
2296                         err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
2297                         if (err)
2298                                 return err;
2299
2300                         map_def->parts |= MAP_DEF_INNER_MAP;
2301                 } else if (strcmp(name, "pinning") == 0) {
2302                         __u32 val;
2303
2304                         if (is_inner) {
2305                                 pr_warn("map '%s': inner def can't be pinned.\n", map_name);
2306                                 return -EINVAL;
2307                         }
2308                         if (!get_map_field_int(map_name, btf, m, &val))
2309                                 return -EINVAL;
2310                         if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
2311                                 pr_warn("map '%s': invalid pinning value %u.\n",
2312                                         map_name, val);
2313                                 return -EINVAL;
2314                         }
2315                         map_def->pinning = val;
2316                         map_def->parts |= MAP_DEF_PINNING;
2317                 } else if (strcmp(name, "map_extra") == 0) {
2318                         __u32 map_extra;
2319
2320                         if (!get_map_field_int(map_name, btf, m, &map_extra))
2321                                 return -EINVAL;
2322                         map_def->map_extra = map_extra;
2323                         map_def->parts |= MAP_DEF_MAP_EXTRA;
2324                 } else {
2325                         if (strict) {
2326                                 pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2327                                 return -ENOTSUP;
2328                         }
2329                         pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2330                 }
2331         }
2332
2333         if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2334                 pr_warn("map '%s': map type isn't specified.\n", map_name);
2335                 return -EINVAL;
2336         }
2337
2338         return 0;
2339 }
2340
2341 static size_t adjust_ringbuf_sz(size_t sz)
2342 {
2343         __u32 page_sz = sysconf(_SC_PAGE_SIZE);
2344         __u32 mul;
2345
2346         /* if user forgot to set any size, make sure they see error */
2347         if (sz == 0)
2348                 return 0;
2349         /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
2350          * a power-of-2 multiple of kernel's page size. If user diligently
2351          * satisified these conditions, pass the size through.
2352          */
2353         if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
2354                 return sz;
2355
2356         /* Otherwise find closest (page_sz * power_of_2) product bigger than
2357          * user-set size to satisfy both user size request and kernel
2358          * requirements and substitute correct max_entries for map creation.
2359          */
2360         for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
2361                 if (mul * page_sz > sz)
2362                         return mul * page_sz;
2363         }
2364
2365         /* if it's impossible to satisfy the conditions (i.e., user size is
2366          * very close to UINT_MAX but is not a power-of-2 multiple of
2367          * page_size) then just return original size and let kernel reject it
2368          */
2369         return sz;
2370 }
2371
2372 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
2373 {
2374         map->def.type = def->map_type;
2375         map->def.key_size = def->key_size;
2376         map->def.value_size = def->value_size;
2377         map->def.max_entries = def->max_entries;
2378         map->def.map_flags = def->map_flags;
2379         map->map_extra = def->map_extra;
2380
2381         map->numa_node = def->numa_node;
2382         map->btf_key_type_id = def->key_type_id;
2383         map->btf_value_type_id = def->value_type_id;
2384
2385         /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2386         if (map->def.type == BPF_MAP_TYPE_RINGBUF)
2387                 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2388
2389         if (def->parts & MAP_DEF_MAP_TYPE)
2390                 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2391
2392         if (def->parts & MAP_DEF_KEY_TYPE)
2393                 pr_debug("map '%s': found key [%u], sz = %u.\n",
2394                          map->name, def->key_type_id, def->key_size);
2395         else if (def->parts & MAP_DEF_KEY_SIZE)
2396                 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2397
2398         if (def->parts & MAP_DEF_VALUE_TYPE)
2399                 pr_debug("map '%s': found value [%u], sz = %u.\n",
2400                          map->name, def->value_type_id, def->value_size);
2401         else if (def->parts & MAP_DEF_VALUE_SIZE)
2402                 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2403
2404         if (def->parts & MAP_DEF_MAX_ENTRIES)
2405                 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2406         if (def->parts & MAP_DEF_MAP_FLAGS)
2407                 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2408         if (def->parts & MAP_DEF_MAP_EXTRA)
2409                 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2410                          (unsigned long long)def->map_extra);
2411         if (def->parts & MAP_DEF_PINNING)
2412                 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2413         if (def->parts & MAP_DEF_NUMA_NODE)
2414                 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2415
2416         if (def->parts & MAP_DEF_INNER_MAP)
2417                 pr_debug("map '%s': found inner map definition.\n", map->name);
2418 }
2419
2420 static const char *btf_var_linkage_str(__u32 linkage)
2421 {
2422         switch (linkage) {
2423         case BTF_VAR_STATIC: return "static";
2424         case BTF_VAR_GLOBAL_ALLOCATED: return "global";
2425         case BTF_VAR_GLOBAL_EXTERN: return "extern";
2426         default: return "unknown";
2427         }
2428 }
2429
2430 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2431                                          const struct btf_type *sec,
2432                                          int var_idx, int sec_idx,
2433                                          const Elf_Data *data, bool strict,
2434                                          const char *pin_root_path)
2435 {
2436         struct btf_map_def map_def = {}, inner_def = {};
2437         const struct btf_type *var, *def;
2438         const struct btf_var_secinfo *vi;
2439         const struct btf_var *var_extra;
2440         const char *map_name;
2441         struct bpf_map *map;
2442         int err;
2443
2444         vi = btf_var_secinfos(sec) + var_idx;
2445         var = btf__type_by_id(obj->btf, vi->type);
2446         var_extra = btf_var(var);
2447         map_name = btf__name_by_offset(obj->btf, var->name_off);
2448
2449         if (map_name == NULL || map_name[0] == '\0') {
2450                 pr_warn("map #%d: empty name.\n", var_idx);
2451                 return -EINVAL;
2452         }
2453         if ((__u64)vi->offset + vi->size > data->d_size) {
2454                 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2455                 return -EINVAL;
2456         }
2457         if (!btf_is_var(var)) {
2458                 pr_warn("map '%s': unexpected var kind %s.\n",
2459                         map_name, btf_kind_str(var));
2460                 return -EINVAL;
2461         }
2462         if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2463                 pr_warn("map '%s': unsupported map linkage %s.\n",
2464                         map_name, btf_var_linkage_str(var_extra->linkage));
2465                 return -EOPNOTSUPP;
2466         }
2467
2468         def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2469         if (!btf_is_struct(def)) {
2470                 pr_warn("map '%s': unexpected def kind %s.\n",
2471                         map_name, btf_kind_str(var));
2472                 return -EINVAL;
2473         }
2474         if (def->size > vi->size) {
2475                 pr_warn("map '%s': invalid def size.\n", map_name);
2476                 return -EINVAL;
2477         }
2478
2479         map = bpf_object__add_map(obj);
2480         if (IS_ERR(map))
2481                 return PTR_ERR(map);
2482         map->name = strdup(map_name);
2483         if (!map->name) {
2484                 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2485                 return -ENOMEM;
2486         }
2487         map->libbpf_type = LIBBPF_MAP_UNSPEC;
2488         map->def.type = BPF_MAP_TYPE_UNSPEC;
2489         map->sec_idx = sec_idx;
2490         map->sec_offset = vi->offset;
2491         map->btf_var_idx = var_idx;
2492         pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2493                  map_name, map->sec_idx, map->sec_offset);
2494
2495         err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2496         if (err)
2497                 return err;
2498
2499         fill_map_from_def(map, &map_def);
2500
2501         if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
2502                 err = build_map_pin_path(map, pin_root_path);
2503                 if (err) {
2504                         pr_warn("map '%s': couldn't build pin path.\n", map->name);
2505                         return err;
2506                 }
2507         }
2508
2509         if (map_def.parts & MAP_DEF_INNER_MAP) {
2510                 map->inner_map = calloc(1, sizeof(*map->inner_map));
2511                 if (!map->inner_map)
2512                         return -ENOMEM;
2513                 map->inner_map->fd = -1;
2514                 map->inner_map->sec_idx = sec_idx;
2515                 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2516                 if (!map->inner_map->name)
2517                         return -ENOMEM;
2518                 sprintf(map->inner_map->name, "%s.inner", map_name);
2519
2520                 fill_map_from_def(map->inner_map, &inner_def);
2521         }
2522
2523         err = bpf_map_find_btf_info(obj, map);
2524         if (err)
2525                 return err;
2526
2527         return 0;
2528 }
2529
2530 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2531                                           const char *pin_root_path)
2532 {
2533         const struct btf_type *sec = NULL;
2534         int nr_types, i, vlen, err;
2535         const struct btf_type *t;
2536         const char *name;
2537         Elf_Data *data;
2538         Elf_Scn *scn;
2539
2540         if (obj->efile.btf_maps_shndx < 0)
2541                 return 0;
2542
2543         scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2544         data = elf_sec_data(obj, scn);
2545         if (!scn || !data) {
2546                 pr_warn("elf: failed to get %s map definitions for %s\n",
2547                         MAPS_ELF_SEC, obj->path);
2548                 return -EINVAL;
2549         }
2550
2551         nr_types = btf__type_cnt(obj->btf);
2552         for (i = 1; i < nr_types; i++) {
2553                 t = btf__type_by_id(obj->btf, i);
2554                 if (!btf_is_datasec(t))
2555                         continue;
2556                 name = btf__name_by_offset(obj->btf, t->name_off);
2557                 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2558                         sec = t;
2559                         obj->efile.btf_maps_sec_btf_id = i;
2560                         break;
2561                 }
2562         }
2563
2564         if (!sec) {
2565                 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2566                 return -ENOENT;
2567         }
2568
2569         vlen = btf_vlen(sec);
2570         for (i = 0; i < vlen; i++) {
2571                 err = bpf_object__init_user_btf_map(obj, sec, i,
2572                                                     obj->efile.btf_maps_shndx,
2573                                                     data, strict,
2574                                                     pin_root_path);
2575                 if (err)
2576                         return err;
2577         }
2578
2579         return 0;
2580 }
2581
2582 static int bpf_object__init_maps(struct bpf_object *obj,
2583                                  const struct bpf_object_open_opts *opts)
2584 {
2585         const char *pin_root_path;
2586         bool strict;
2587         int err = 0;
2588
2589         strict = !OPTS_GET(opts, relaxed_maps, false);
2590         pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2591
2592         err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2593         err = err ?: bpf_object__init_global_data_maps(obj);
2594         err = err ?: bpf_object__init_kconfig_map(obj);
2595         err = err ?: bpf_object__init_struct_ops_maps(obj);
2596
2597         return err;
2598 }
2599
2600 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2601 {
2602         Elf64_Shdr *sh;
2603
2604         sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));
2605         if (!sh)
2606                 return false;
2607
2608         return sh->sh_flags & SHF_EXECINSTR;
2609 }
2610
2611 static bool btf_needs_sanitization(struct bpf_object *obj)
2612 {
2613         bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2614         bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2615         bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2616         bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2617         bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2618         bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2619         bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2620
2621         return !has_func || !has_datasec || !has_func_global || !has_float ||
2622                !has_decl_tag || !has_type_tag || !has_enum64;
2623 }
2624
2625 static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2626 {
2627         bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2628         bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2629         bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2630         bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2631         bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2632         bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2633         bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2634         int enum64_placeholder_id = 0;
2635         struct btf_type *t;
2636         int i, j, vlen;
2637
2638         for (i = 1; i < btf__type_cnt(btf); i++) {
2639                 t = (struct btf_type *)btf__type_by_id(btf, i);
2640
2641                 if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) {
2642                         /* replace VAR/DECL_TAG with INT */
2643                         t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2644                         /*
2645                          * using size = 1 is the safest choice, 4 will be too
2646                          * big and cause kernel BTF validation failure if
2647                          * original variable took less than 4 bytes
2648                          */
2649                         t->size = 1;
2650                         *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2651                 } else if (!has_datasec && btf_is_datasec(t)) {
2652                         /* replace DATASEC with STRUCT */
2653                         const struct btf_var_secinfo *v = btf_var_secinfos(t);
2654                         struct btf_member *m = btf_members(t);
2655                         struct btf_type *vt;
2656                         char *name;
2657
2658                         name = (char *)btf__name_by_offset(btf, t->name_off);
2659                         while (*name) {
2660                                 if (*name == '.')
2661                                         *name = '_';
2662                                 name++;
2663                         }
2664
2665                         vlen = btf_vlen(t);
2666                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2667                         for (j = 0; j < vlen; j++, v++, m++) {
2668                                 /* order of field assignments is important */
2669                                 m->offset = v->offset * 8;
2670                                 m->type = v->type;
2671                                 /* preserve variable name as member name */
2672                                 vt = (void *)btf__type_by_id(btf, v->type);
2673                                 m->name_off = vt->name_off;
2674                         }
2675                 } else if (!has_func && btf_is_func_proto(t)) {
2676                         /* replace FUNC_PROTO with ENUM */
2677                         vlen = btf_vlen(t);
2678                         t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2679                         t->size = sizeof(__u32); /* kernel enforced */
2680                 } else if (!has_func && btf_is_func(t)) {
2681                         /* replace FUNC with TYPEDEF */
2682                         t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2683                 } else if (!has_func_global && btf_is_func(t)) {
2684                         /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2685                         t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2686                 } else if (!has_float && btf_is_float(t)) {
2687                         /* replace FLOAT with an equally-sized empty STRUCT;
2688                          * since C compilers do not accept e.g. "float" as a
2689                          * valid struct name, make it anonymous
2690                          */
2691                         t->name_off = 0;
2692                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2693                 } else if (!has_type_tag && btf_is_type_tag(t)) {
2694                         /* replace TYPE_TAG with a CONST */
2695                         t->name_off = 0;
2696                         t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
2697                 } else if (!has_enum64 && btf_is_enum(t)) {
2698                         /* clear the kflag */
2699                         t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
2700                 } else if (!has_enum64 && btf_is_enum64(t)) {
2701                         /* replace ENUM64 with a union */
2702                         struct btf_member *m;
2703
2704                         if (enum64_placeholder_id == 0) {
2705                                 enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
2706                                 if (enum64_placeholder_id < 0)
2707                                         return enum64_placeholder_id;
2708
2709                                 t = (struct btf_type *)btf__type_by_id(btf, i);
2710                         }
2711
2712                         m = btf_members(t);
2713                         vlen = btf_vlen(t);
2714                         t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
2715                         for (j = 0; j < vlen; j++, m++) {
2716                                 m->type = enum64_placeholder_id;
2717                                 m->offset = 0;
2718                         }
2719                 }
2720         }
2721
2722         return 0;
2723 }
2724
2725 static bool libbpf_needs_btf(const struct bpf_object *obj)
2726 {
2727         return obj->efile.btf_maps_shndx >= 0 ||
2728                obj->efile.st_ops_shndx >= 0 ||
2729                obj->nr_extern > 0;
2730 }
2731
2732 static bool kernel_needs_btf(const struct bpf_object *obj)
2733 {
2734         return obj->efile.st_ops_shndx >= 0;
2735 }
2736
2737 static int bpf_object__init_btf(struct bpf_object *obj,
2738                                 Elf_Data *btf_data,
2739                                 Elf_Data *btf_ext_data)
2740 {
2741         int err = -ENOENT;
2742
2743         if (btf_data) {
2744                 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2745                 err = libbpf_get_error(obj->btf);
2746                 if (err) {
2747                         obj->btf = NULL;
2748                         pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);
2749                         goto out;
2750                 }
2751                 /* enforce 8-byte pointers for BPF-targeted BTFs */
2752                 btf__set_pointer_size(obj->btf, 8);
2753         }
2754         if (btf_ext_data) {
2755                 struct btf_ext_info *ext_segs[3];
2756                 int seg_num, sec_num;
2757
2758                 if (!obj->btf) {
2759                         pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2760                                  BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2761                         goto out;
2762                 }
2763                 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2764                 err = libbpf_get_error(obj->btf_ext);
2765                 if (err) {
2766                         pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",
2767                                 BTF_EXT_ELF_SEC, err);
2768                         obj->btf_ext = NULL;
2769                         goto out;
2770                 }
2771
2772                 /* setup .BTF.ext to ELF section mapping */
2773                 ext_segs[0] = &obj->btf_ext->func_info;
2774                 ext_segs[1] = &obj->btf_ext->line_info;
2775                 ext_segs[2] = &obj->btf_ext->core_relo_info;
2776                 for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) {
2777                         struct btf_ext_info *seg = ext_segs[seg_num];
2778                         const struct btf_ext_info_sec *sec;
2779                         const char *sec_name;
2780                         Elf_Scn *scn;
2781
2782                         if (seg->sec_cnt == 0)
2783                                 continue;
2784
2785                         seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
2786                         if (!seg->sec_idxs) {
2787                                 err = -ENOMEM;
2788                                 goto out;
2789                         }
2790
2791                         sec_num = 0;
2792                         for_each_btf_ext_sec(seg, sec) {
2793                                 /* preventively increment index to avoid doing
2794                                  * this before every continue below
2795                                  */
2796                                 sec_num++;
2797
2798                                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
2799                                 if (str_is_empty(sec_name))
2800                                         continue;
2801                                 scn = elf_sec_by_name(obj, sec_name);
2802                                 if (!scn)
2803                                         continue;
2804
2805                                 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
2806                         }
2807                 }
2808         }
2809 out:
2810         if (err && libbpf_needs_btf(obj)) {
2811                 pr_warn("BTF is required, but is missing or corrupted.\n");
2812                 return err;
2813         }
2814         return 0;
2815 }
2816
2817 static int compare_vsi_off(const void *_a, const void *_b)
2818 {
2819         const struct btf_var_secinfo *a = _a;
2820         const struct btf_var_secinfo *b = _b;
2821
2822         return a->offset - b->offset;
2823 }
2824
2825 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
2826                              struct btf_type *t)
2827 {
2828         __u32 size = 0, off = 0, i, vars = btf_vlen(t);
2829         const char *name = btf__name_by_offset(btf, t->name_off);
2830         const struct btf_type *t_var;
2831         struct btf_var_secinfo *vsi;
2832         const struct btf_var *var;
2833         int ret;
2834
2835         if (!name) {
2836                 pr_debug("No name found in string section for DATASEC kind.\n");
2837                 return -ENOENT;
2838         }
2839
2840         /* .extern datasec size and var offsets were set correctly during
2841          * extern collection step, so just skip straight to sorting variables
2842          */
2843         if (t->size)
2844                 goto sort_vars;
2845
2846         ret = find_elf_sec_sz(obj, name, &size);
2847         if (ret || !size) {
2848                 pr_debug("Invalid size for section %s: %u bytes\n", name, size);
2849                 return -ENOENT;
2850         }
2851
2852         t->size = size;
2853
2854         for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
2855                 t_var = btf__type_by_id(btf, vsi->type);
2856                 if (!t_var || !btf_is_var(t_var)) {
2857                         pr_debug("Non-VAR type seen in section %s\n", name);
2858                         return -EINVAL;
2859                 }
2860
2861                 var = btf_var(t_var);
2862                 if (var->linkage == BTF_VAR_STATIC)
2863                         continue;
2864
2865                 name = btf__name_by_offset(btf, t_var->name_off);
2866                 if (!name) {
2867                         pr_debug("No name found in string section for VAR kind\n");
2868                         return -ENOENT;
2869                 }
2870
2871                 ret = find_elf_var_offset(obj, name, &off);
2872                 if (ret) {
2873                         pr_debug("No offset found in symbol table for VAR %s\n",
2874                                  name);
2875                         return -ENOENT;
2876                 }
2877
2878                 vsi->offset = off;
2879         }
2880
2881 sort_vars:
2882         qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
2883         return 0;
2884 }
2885
2886 static int btf_finalize_data(struct bpf_object *obj, struct btf *btf)
2887 {
2888         int err = 0;
2889         __u32 i, n = btf__type_cnt(btf);
2890
2891         for (i = 1; i < n; i++) {
2892                 struct btf_type *t = btf_type_by_id(btf, i);
2893
2894                 /* Loader needs to fix up some of the things compiler
2895                  * couldn't get its hands on while emitting BTF. This
2896                  * is section size and global variable offset. We use
2897                  * the info from the ELF itself for this purpose.
2898                  */
2899                 if (btf_is_datasec(t)) {
2900                         err = btf_fixup_datasec(obj, btf, t);
2901                         if (err)
2902                                 break;
2903                 }
2904         }
2905
2906         return libbpf_err(err);
2907 }
2908
2909 static int bpf_object__finalize_btf(struct bpf_object *obj)
2910 {
2911         int err;
2912
2913         if (!obj->btf)
2914                 return 0;
2915
2916         err = btf_finalize_data(obj, obj->btf);
2917         if (err) {
2918                 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2919                 return err;
2920         }
2921
2922         return 0;
2923 }
2924
2925 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
2926 {
2927         if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2928             prog->type == BPF_PROG_TYPE_LSM)
2929                 return true;
2930
2931         /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
2932          * also need vmlinux BTF
2933          */
2934         if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2935                 return true;
2936
2937         return false;
2938 }
2939
2940 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
2941 {
2942         struct bpf_program *prog;
2943         int i;
2944
2945         /* CO-RE relocations need kernel BTF, only when btf_custom_path
2946          * is not specified
2947          */
2948         if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
2949                 return true;
2950
2951         /* Support for typed ksyms needs kernel BTF */
2952         for (i = 0; i < obj->nr_extern; i++) {
2953                 const struct extern_desc *ext;
2954
2955                 ext = &obj->externs[i];
2956                 if (ext->type == EXT_KSYM && ext->ksym.type_id)
2957                         return true;
2958         }
2959
2960         bpf_object__for_each_program(prog, obj) {
2961                 if (!prog->autoload)
2962                         continue;
2963                 if (prog_needs_vmlinux_btf(prog))
2964                         return true;
2965         }
2966
2967         return false;
2968 }
2969
2970 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
2971 {
2972         int err;
2973
2974         /* btf_vmlinux could be loaded earlier */
2975         if (obj->btf_vmlinux || obj->gen_loader)
2976                 return 0;
2977
2978         if (!force && !obj_needs_vmlinux_btf(obj))
2979                 return 0;
2980
2981         obj->btf_vmlinux = btf__load_vmlinux_btf();
2982         err = libbpf_get_error(obj->btf_vmlinux);
2983         if (err) {
2984                 pr_warn("Error loading vmlinux BTF: %d\n", err);
2985                 obj->btf_vmlinux = NULL;
2986                 return err;
2987         }
2988         return 0;
2989 }
2990
2991 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
2992 {
2993         struct btf *kern_btf = obj->btf;
2994         bool btf_mandatory, sanitize;
2995         int i, err = 0;
2996
2997         if (!obj->btf)
2998                 return 0;
2999
3000         if (!kernel_supports(obj, FEAT_BTF)) {
3001                 if (kernel_needs_btf(obj)) {
3002                         err = -EOPNOTSUPP;
3003                         goto report;
3004                 }
3005                 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
3006                 return 0;
3007         }
3008
3009         /* Even though some subprogs are global/weak, user might prefer more
3010          * permissive BPF verification process that BPF verifier performs for
3011          * static functions, taking into account more context from the caller
3012          * functions. In such case, they need to mark such subprogs with
3013          * __attribute__((visibility("hidden"))) and libbpf will adjust
3014          * corresponding FUNC BTF type to be marked as static and trigger more
3015          * involved BPF verification process.
3016          */
3017         for (i = 0; i < obj->nr_programs; i++) {
3018                 struct bpf_program *prog = &obj->programs[i];
3019                 struct btf_type *t;
3020                 const char *name;
3021                 int j, n;
3022
3023                 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3024                         continue;
3025
3026                 n = btf__type_cnt(obj->btf);
3027                 for (j = 1; j < n; j++) {
3028                         t = btf_type_by_id(obj->btf, j);
3029                         if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)
3030                                 continue;
3031
3032                         name = btf__str_by_offset(obj->btf, t->name_off);
3033                         if (strcmp(name, prog->name) != 0)
3034                                 continue;
3035
3036                         t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3037                         break;
3038                 }
3039         }
3040
3041         sanitize = btf_needs_sanitization(obj);
3042         if (sanitize) {
3043                 const void *raw_data;
3044                 __u32 sz;
3045
3046                 /* clone BTF to sanitize a copy and leave the original intact */
3047                 raw_data = btf__raw_data(obj->btf, &sz);
3048                 kern_btf = btf__new(raw_data, sz);
3049                 err = libbpf_get_error(kern_btf);
3050                 if (err)
3051                         return err;
3052
3053                 /* enforce 8-byte pointers for BPF-targeted BTFs */
3054                 btf__set_pointer_size(obj->btf, 8);
3055                 err = bpf_object__sanitize_btf(obj, kern_btf);
3056                 if (err)
3057                         return err;
3058         }
3059
3060         if (obj->gen_loader) {
3061                 __u32 raw_size = 0;
3062                 const void *raw_data = btf__raw_data(kern_btf, &raw_size);
3063
3064                 if (!raw_data)
3065                         return -ENOMEM;
3066                 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3067                 /* Pretend to have valid FD to pass various fd >= 0 checks.
3068                  * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3069                  */
3070                 btf__set_fd(kern_btf, 0);
3071         } else {
3072                 /* currently BPF_BTF_LOAD only supports log_level 1 */
3073                 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3074                                            obj->log_level ? 1 : 0);
3075         }
3076         if (sanitize) {
3077                 if (!err) {
3078                         /* move fd to libbpf's BTF */
3079                         btf__set_fd(obj->btf, btf__fd(kern_btf));
3080                         btf__set_fd(kern_btf, -1);
3081                 }
3082                 btf__free(kern_btf);
3083         }
3084 report:
3085         if (err) {
3086                 btf_mandatory = kernel_needs_btf(obj);
3087                 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
3088                         btf_mandatory ? "BTF is mandatory, can't proceed."
3089                                       : "BTF is optional, ignoring.");
3090                 if (!btf_mandatory)
3091                         err = 0;
3092         }
3093         return err;
3094 }
3095
3096 static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
3097 {
3098         const char *name;
3099
3100         name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3101         if (!name) {
3102                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3103                         off, obj->path, elf_errmsg(-1));
3104                 return NULL;
3105         }
3106
3107         return name;
3108 }
3109
3110 static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
3111 {
3112         const char *name;
3113
3114         name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3115         if (!name) {
3116                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3117                         off, obj->path, elf_errmsg(-1));
3118                 return NULL;
3119         }
3120
3121         return name;
3122 }
3123
3124 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
3125 {
3126         Elf_Scn *scn;
3127
3128         scn = elf_getscn(obj->efile.elf, idx);
3129         if (!scn) {
3130                 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
3131                         idx, obj->path, elf_errmsg(-1));
3132                 return NULL;
3133         }
3134         return scn;
3135 }
3136
3137 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
3138 {
3139         Elf_Scn *scn = NULL;
3140         Elf *elf = obj->efile.elf;
3141         const char *sec_name;
3142
3143         while ((scn = elf_nextscn(elf, scn)) != NULL) {
3144                 sec_name = elf_sec_name(obj, scn);
3145                 if (!sec_name)
3146                         return NULL;
3147
3148                 if (strcmp(sec_name, name) != 0)
3149                         continue;
3150
3151                 return scn;
3152         }
3153         return NULL;
3154 }
3155
3156 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn)
3157 {
3158         Elf64_Shdr *shdr;
3159
3160         if (!scn)
3161                 return NULL;
3162
3163         shdr = elf64_getshdr(scn);
3164         if (!shdr) {
3165                 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
3166                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3167                 return NULL;
3168         }
3169
3170         return shdr;
3171 }
3172
3173 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
3174 {
3175         const char *name;
3176         Elf64_Shdr *sh;
3177
3178         if (!scn)
3179                 return NULL;
3180
3181         sh = elf_sec_hdr(obj, scn);
3182         if (!sh)
3183                 return NULL;
3184
3185         name = elf_sec_str(obj, sh->sh_name);
3186         if (!name) {
3187                 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
3188                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3189                 return NULL;
3190         }
3191
3192         return name;
3193 }
3194
3195 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
3196 {
3197         Elf_Data *data;
3198
3199         if (!scn)
3200                 return NULL;
3201
3202         data = elf_getdata(scn, 0);
3203         if (!data) {
3204                 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
3205                         elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
3206                         obj->path, elf_errmsg(-1));
3207                 return NULL;
3208         }
3209
3210         return data;
3211 }
3212
3213 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx)
3214 {
3215         if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3216                 return NULL;
3217
3218         return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3219 }
3220
3221 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx)
3222 {
3223         if (idx >= data->d_size / sizeof(Elf64_Rel))
3224                 return NULL;
3225
3226         return (Elf64_Rel *)data->d_buf + idx;
3227 }
3228
3229 static bool is_sec_name_dwarf(const char *name)
3230 {
3231         /* approximation, but the actual list is too long */
3232         return str_has_pfx(name, ".debug_");
3233 }
3234
3235 static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name)
3236 {
3237         /* no special handling of .strtab */
3238         if (hdr->sh_type == SHT_STRTAB)
3239                 return true;
3240
3241         /* ignore .llvm_addrsig section as well */
3242         if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3243                 return true;
3244
3245         /* no subprograms will lead to an empty .text section, ignore it */
3246         if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3247             strcmp(name, ".text") == 0)
3248                 return true;
3249
3250         /* DWARF sections */
3251         if (is_sec_name_dwarf(name))
3252                 return true;
3253
3254         if (str_has_pfx(name, ".rel")) {
3255                 name += sizeof(".rel") - 1;
3256                 /* DWARF section relocations */
3257                 if (is_sec_name_dwarf(name))
3258                         return true;
3259
3260                 /* .BTF and .BTF.ext don't need relocations */
3261                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
3262                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
3263                         return true;
3264         }
3265
3266         return false;
3267 }
3268
3269 static int cmp_progs(const void *_a, const void *_b)
3270 {
3271         const struct bpf_program *a = _a;
3272         const struct bpf_program *b = _b;
3273
3274         if (a->sec_idx != b->sec_idx)
3275                 return a->sec_idx < b->sec_idx ? -1 : 1;
3276
3277         /* sec_insn_off can't be the same within the section */
3278         return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3279 }
3280
3281 static int bpf_object__elf_collect(struct bpf_object *obj)
3282 {
3283         struct elf_sec_desc *sec_desc;
3284         Elf *elf = obj->efile.elf;
3285         Elf_Data *btf_ext_data = NULL;
3286         Elf_Data *btf_data = NULL;
3287         int idx = 0, err = 0;
3288         const char *name;
3289         Elf_Data *data;
3290         Elf_Scn *scn;
3291         Elf64_Shdr *sh;
3292
3293         /* ELF section indices are 0-based, but sec #0 is special "invalid"
3294          * section. e_shnum does include sec #0, so e_shnum is the necessary
3295          * size of an array to keep all the sections.
3296          */
3297         obj->efile.sec_cnt = obj->efile.ehdr->e_shnum;
3298         obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3299         if (!obj->efile.secs)
3300                 return -ENOMEM;
3301
3302         /* a bunch of ELF parsing functionality depends on processing symbols,
3303          * so do the first pass and find the symbol table
3304          */
3305         scn = NULL;
3306         while ((scn = elf_nextscn(elf, scn)) != NULL) {
3307                 sh = elf_sec_hdr(obj, scn);
3308                 if (!sh)
3309                         return -LIBBPF_ERRNO__FORMAT;
3310
3311                 if (sh->sh_type == SHT_SYMTAB) {
3312                         if (obj->efile.symbols) {
3313                                 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3314                                 return -LIBBPF_ERRNO__FORMAT;
3315                         }
3316
3317                         data = elf_sec_data(obj, scn);
3318                         if (!data)
3319                                 return -LIBBPF_ERRNO__FORMAT;
3320
3321                         idx = elf_ndxscn(scn);
3322
3323                         obj->efile.symbols = data;
3324                         obj->efile.symbols_shndx = idx;
3325                         obj->efile.strtabidx = sh->sh_link;
3326                 }
3327         }
3328
3329         if (!obj->efile.symbols) {
3330                 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
3331                         obj->path);
3332                 return -ENOENT;
3333         }
3334
3335         scn = NULL;
3336         while ((scn = elf_nextscn(elf, scn)) != NULL) {
3337                 idx = elf_ndxscn(scn);
3338                 sec_desc = &obj->efile.secs[idx];
3339
3340                 sh = elf_sec_hdr(obj, scn);
3341                 if (!sh)
3342                         return -LIBBPF_ERRNO__FORMAT;
3343
3344                 name = elf_sec_str(obj, sh->sh_name);
3345                 if (!name)
3346                         return -LIBBPF_ERRNO__FORMAT;
3347
3348                 if (ignore_elf_section(sh, name))
3349                         continue;
3350
3351                 data = elf_sec_data(obj, scn);
3352                 if (!data)
3353                         return -LIBBPF_ERRNO__FORMAT;
3354
3355                 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
3356                          idx, name, (unsigned long)data->d_size,
3357                          (int)sh->sh_link, (unsigned long)sh->sh_flags,
3358                          (int)sh->sh_type);
3359
3360                 if (strcmp(name, "license") == 0) {
3361                         err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3362                         if (err)
3363                                 return err;
3364                 } else if (strcmp(name, "version") == 0) {
3365                         err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3366                         if (err)
3367                                 return err;
3368                 } else if (strcmp(name, "maps") == 0) {
3369                         pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");
3370                         return -ENOTSUP;
3371                 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
3372                         obj->efile.btf_maps_shndx = idx;
3373                 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
3374                         if (sh->sh_type != SHT_PROGBITS)
3375                                 return -LIBBPF_ERRNO__FORMAT;
3376                         btf_data = data;
3377                 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
3378                         if (sh->sh_type != SHT_PROGBITS)
3379                                 return -LIBBPF_ERRNO__FORMAT;
3380                         btf_ext_data = data;
3381                 } else if (sh->sh_type == SHT_SYMTAB) {
3382                         /* already processed during the first pass above */
3383                 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3384                         if (sh->sh_flags & SHF_EXECINSTR) {
3385                                 if (strcmp(name, ".text") == 0)
3386                                         obj->efile.text_shndx = idx;
3387                                 err = bpf_object__add_programs(obj, data, name, idx);
3388                                 if (err)
3389                                         return err;
3390                         } else if (strcmp(name, DATA_SEC) == 0 ||
3391                                    str_has_pfx(name, DATA_SEC ".")) {
3392                                 sec_desc->sec_type = SEC_DATA;
3393                                 sec_desc->shdr = sh;
3394                                 sec_desc->data = data;
3395                         } else if (strcmp(name, RODATA_SEC) == 0 ||
3396                                    str_has_pfx(name, RODATA_SEC ".")) {
3397                                 sec_desc->sec_type = SEC_RODATA;
3398                                 sec_desc->shdr = sh;
3399                                 sec_desc->data = data;
3400                         } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
3401                                 obj->efile.st_ops_data = data;
3402                                 obj->efile.st_ops_shndx = idx;
3403                         } else {
3404                                 pr_info("elf: skipping unrecognized data section(%d) %s\n",
3405                                         idx, name);
3406                         }
3407                 } else if (sh->sh_type == SHT_REL) {
3408                         int targ_sec_idx = sh->sh_info; /* points to other section */
3409
3410                         if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3411                             targ_sec_idx >= obj->efile.sec_cnt)
3412                                 return -LIBBPF_ERRNO__FORMAT;
3413
3414                         /* Only do relo for section with exec instructions */
3415                         if (!section_have_execinstr(obj, targ_sec_idx) &&
3416                             strcmp(name, ".rel" STRUCT_OPS_SEC) &&
3417                             strcmp(name, ".rel" MAPS_ELF_SEC)) {
3418                                 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
3419                                         idx, name, targ_sec_idx,
3420                                         elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");
3421                                 continue;
3422                         }
3423
3424                         sec_desc->sec_type = SEC_RELO;
3425                         sec_desc->shdr = sh;
3426                         sec_desc->data = data;
3427                 } else if (sh->sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) {
3428                         sec_desc->sec_type = SEC_BSS;
3429                         sec_desc->shdr = sh;
3430                         sec_desc->data = data;
3431                 } else {
3432                         pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
3433                                 (size_t)sh->sh_size);
3434                 }
3435         }
3436
3437         if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3438                 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3439                 return -LIBBPF_ERRNO__FORMAT;
3440         }
3441
3442         /* sort BPF programs by section name and in-section instruction offset
3443          * for faster search */
3444         if (obj->nr_programs)
3445                 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3446
3447         return bpf_object__init_btf(obj, btf_data, btf_ext_data);
3448 }
3449
3450 static bool sym_is_extern(const Elf64_Sym *sym)
3451 {
3452         int bind = ELF64_ST_BIND(sym->st_info);
3453         /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
3454         return sym->st_shndx == SHN_UNDEF &&
3455                (bind == STB_GLOBAL || bind == STB_WEAK) &&
3456                ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3457 }
3458
3459 static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx)
3460 {
3461         int bind = ELF64_ST_BIND(sym->st_info);
3462         int type = ELF64_ST_TYPE(sym->st_info);
3463
3464         /* in .text section */
3465         if (sym->st_shndx != text_shndx)
3466                 return false;
3467
3468         /* local function */
3469         if (bind == STB_LOCAL && type == STT_SECTION)
3470                 return true;
3471
3472         /* global function */
3473         return bind == STB_GLOBAL && type == STT_FUNC;
3474 }
3475
3476 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
3477 {
3478         const struct btf_type *t;
3479         const char *tname;
3480         int i, n;
3481
3482         if (!btf)
3483                 return -ESRCH;
3484
3485         n = btf__type_cnt(btf);
3486         for (i = 1; i < n; i++) {
3487                 t = btf__type_by_id(btf, i);
3488
3489                 if (!btf_is_var(t) && !btf_is_func(t))
3490                         continue;
3491
3492                 tname = btf__name_by_offset(btf, t->name_off);
3493                 if (strcmp(tname, ext_name))
3494                         continue;
3495
3496                 if (btf_is_var(t) &&
3497                     btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3498                         return -EINVAL;
3499
3500                 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
3501                         return -EINVAL;
3502
3503                 return i;
3504         }
3505
3506         return -ENOENT;
3507 }
3508
3509 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3510         const struct btf_var_secinfo *vs;
3511         const struct btf_type *t;
3512         int i, j, n;
3513
3514         if (!btf)
3515                 return -ESRCH;
3516
3517         n = btf__type_cnt(btf);
3518         for (i = 1; i < n; i++) {
3519                 t = btf__type_by_id(btf, i);
3520
3521                 if (!btf_is_datasec(t))
3522                         continue;
3523
3524                 vs = btf_var_secinfos(t);
3525                 for (j = 0; j < btf_vlen(t); j++, vs++) {
3526                         if (vs->type == ext_btf_id)
3527                                 return i;
3528                 }
3529         }
3530
3531         return -ENOENT;
3532 }
3533
3534 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3535                                      bool *is_signed)
3536 {
3537         const struct btf_type *t;
3538         const char *name;
3539
3540         t = skip_mods_and_typedefs(btf, id, NULL);
3541         name = btf__name_by_offset(btf, t->name_off);
3542
3543         if (is_signed)
3544                 *is_signed = false;
3545         switch (btf_kind(t)) {
3546         case BTF_KIND_INT: {
3547                 int enc = btf_int_encoding(t);
3548
3549                 if (enc & BTF_INT_BOOL)
3550                         return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3551                 if (is_signed)
3552                         *is_signed = enc & BTF_INT_SIGNED;
3553                 if (t->size == 1)
3554                         return KCFG_CHAR;
3555                 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3556                         return KCFG_UNKNOWN;
3557                 return KCFG_INT;
3558         }
3559         case BTF_KIND_ENUM:
3560                 if (t->size != 4)
3561                         return KCFG_UNKNOWN;
3562                 if (strcmp(name, "libbpf_tristate"))
3563                         return KCFG_UNKNOWN;
3564                 return KCFG_TRISTATE;
3565         case BTF_KIND_ENUM64:
3566                 if (strcmp(name, "libbpf_tristate"))
3567                         return KCFG_UNKNOWN;
3568                 return KCFG_TRISTATE;
3569         case BTF_KIND_ARRAY:
3570                 if (btf_array(t)->nelems == 0)
3571                         return KCFG_UNKNOWN;
3572                 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3573                         return KCFG_UNKNOWN;
3574                 return KCFG_CHAR_ARR;
3575         default:
3576                 return KCFG_UNKNOWN;
3577         }
3578 }
3579
3580 static int cmp_externs(const void *_a, const void *_b)
3581 {
3582         const struct extern_desc *a = _a;
3583         const struct extern_desc *b = _b;
3584
3585         if (a->type != b->type)
3586                 return a->type < b->type ? -1 : 1;
3587
3588         if (a->type == EXT_KCFG) {
3589                 /* descending order by alignment requirements */
3590                 if (a->kcfg.align != b->kcfg.align)
3591                         return a->kcfg.align > b->kcfg.align ? -1 : 1;
3592                 /* ascending order by size, within same alignment class */
3593                 if (a->kcfg.sz != b->kcfg.sz)
3594                         return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3595         }
3596
3597         /* resolve ties by name */
3598         return strcmp(a->name, b->name);
3599 }
3600
3601 static int find_int_btf_id(const struct btf *btf)
3602 {
3603         const struct btf_type *t;
3604         int i, n;
3605
3606         n = btf__type_cnt(btf);
3607         for (i = 1; i < n; i++) {
3608                 t = btf__type_by_id(btf, i);
3609
3610                 if (btf_is_int(t) && btf_int_bits(t) == 32)
3611                         return i;
3612         }
3613
3614         return 0;
3615 }
3616
3617 static int add_dummy_ksym_var(struct btf *btf)
3618 {
3619         int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
3620         const struct btf_var_secinfo *vs;
3621         const struct btf_type *sec;
3622
3623         if (!btf)
3624                 return 0;
3625
3626         sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
3627                                             BTF_KIND_DATASEC);
3628         if (sec_btf_id < 0)
3629                 return 0;
3630
3631         sec = btf__type_by_id(btf, sec_btf_id);
3632         vs = btf_var_secinfos(sec);
3633         for (i = 0; i < btf_vlen(sec); i++, vs++) {
3634                 const struct btf_type *vt;
3635
3636                 vt = btf__type_by_id(btf, vs->type);
3637                 if (btf_is_func(vt))
3638                         break;
3639         }
3640
3641         /* No func in ksyms sec.  No need to add dummy var. */
3642         if (i == btf_vlen(sec))
3643                 return 0;
3644
3645         int_btf_id = find_int_btf_id(btf);
3646         dummy_var_btf_id = btf__add_var(btf,
3647                                         "dummy_ksym",
3648                                         BTF_VAR_GLOBAL_ALLOCATED,
3649                                         int_btf_id);
3650         if (dummy_var_btf_id < 0)
3651                 pr_warn("cannot create a dummy_ksym var\n");
3652
3653         return dummy_var_btf_id;
3654 }
3655
3656 static int bpf_object__collect_externs(struct bpf_object *obj)
3657 {
3658         struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3659         const struct btf_type *t;
3660         struct extern_desc *ext;
3661         int i, n, off, dummy_var_btf_id;
3662         const char *ext_name, *sec_name;
3663         Elf_Scn *scn;
3664         Elf64_Shdr *sh;
3665
3666         if (!obj->efile.symbols)
3667                 return 0;
3668
3669         scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3670         sh = elf_sec_hdr(obj, scn);
3671         if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
3672                 return -LIBBPF_ERRNO__FORMAT;
3673
3674         dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
3675         if (dummy_var_btf_id < 0)
3676                 return dummy_var_btf_id;
3677
3678         n = sh->sh_size / sh->sh_entsize;
3679         pr_debug("looking for externs among %d symbols...\n", n);
3680
3681         for (i = 0; i < n; i++) {
3682                 Elf64_Sym *sym = elf_sym_by_idx(obj, i);
3683
3684                 if (!sym)
3685                         return -LIBBPF_ERRNO__FORMAT;
3686                 if (!sym_is_extern(sym))
3687                         continue;
3688                 ext_name = elf_sym_str(obj, sym->st_name);
3689                 if (!ext_name || !ext_name[0])
3690                         continue;
3691
3692                 ext = obj->externs;
3693                 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3694                 if (!ext)
3695                         return -ENOMEM;
3696                 obj->externs = ext;
3697                 ext = &ext[obj->nr_extern];
3698                 memset(ext, 0, sizeof(*ext));
3699                 obj->nr_extern++;
3700
3701                 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3702                 if (ext->btf_id <= 0) {
3703                         pr_warn("failed to find BTF for extern '%s': %d\n",
3704                                 ext_name, ext->btf_id);
3705                         return ext->btf_id;
3706                 }
3707                 t = btf__type_by_id(obj->btf, ext->btf_id);
3708                 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3709                 ext->sym_idx = i;
3710                 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
3711
3712                 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3713                 if (ext->sec_btf_id <= 0) {
3714                         pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3715                                 ext_name, ext->btf_id, ext->sec_btf_id);
3716                         return ext->sec_btf_id;
3717                 }
3718                 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3719                 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3720
3721                 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3722                         if (btf_is_func(t)) {
3723                                 pr_warn("extern function %s is unsupported under %s section\n",
3724                                         ext->name, KCONFIG_SEC);
3725                                 return -ENOTSUP;
3726                         }
3727                         kcfg_sec = sec;
3728                         ext->type = EXT_KCFG;
3729                         ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3730                         if (ext->kcfg.sz <= 0) {
3731                                 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3732                                         ext_name, ext->kcfg.sz);
3733                                 return ext->kcfg.sz;
3734                         }
3735                         ext->kcfg.align = btf__align_of(obj->btf, t->type);
3736                         if (ext->kcfg.align <= 0) {
3737                                 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3738                                         ext_name, ext->kcfg.align);
3739                                 return -EINVAL;
3740                         }
3741                         ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3742                                                         &ext->kcfg.is_signed);
3743                         if (ext->kcfg.type == KCFG_UNKNOWN) {
3744                                 pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);
3745                                 return -ENOTSUP;
3746                         }
3747                 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3748                         ksym_sec = sec;
3749                         ext->type = EXT_KSYM;
3750                         skip_mods_and_typedefs(obj->btf, t->type,
3751                                                &ext->ksym.type_id);
3752                 } else {
3753                         pr_warn("unrecognized extern section '%s'\n", sec_name);
3754                         return -ENOTSUP;
3755                 }
3756         }
3757         pr_debug("collected %d externs total\n", obj->nr_extern);
3758
3759         if (!obj->nr_extern)
3760                 return 0;
3761
3762         /* sort externs by type, for kcfg ones also by (align, size, name) */
3763         qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3764
3765         /* for .ksyms section, we need to turn all externs into allocated
3766          * variables in BTF to pass kernel verification; we do this by
3767          * pretending that each extern is a 8-byte variable
3768          */
3769         if (ksym_sec) {
3770                 /* find existing 4-byte integer type in BTF to use for fake
3771                  * extern variables in DATASEC
3772                  */
3773                 int int_btf_id = find_int_btf_id(obj->btf);
3774                 /* For extern function, a dummy_var added earlier
3775                  * will be used to replace the vs->type and
3776                  * its name string will be used to refill
3777                  * the missing param's name.
3778                  */
3779                 const struct btf_type *dummy_var;
3780
3781                 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
3782                 for (i = 0; i < obj->nr_extern; i++) {
3783                         ext = &obj->externs[i];
3784                         if (ext->type != EXT_KSYM)
3785                                 continue;
3786                         pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
3787                                  i, ext->sym_idx, ext->name);
3788                 }
3789
3790                 sec = ksym_sec;
3791                 n = btf_vlen(sec);
3792                 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
3793                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3794                         struct btf_type *vt;
3795
3796                         vt = (void *)btf__type_by_id(obj->btf, vs->type);
3797                         ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3798                         ext = find_extern_by_name(obj, ext_name);
3799                         if (!ext) {
3800                                 pr_warn("failed to find extern definition for BTF %s '%s'\n",
3801                                         btf_kind_str(vt), ext_name);
3802                                 return -ESRCH;
3803                         }
3804                         if (btf_is_func(vt)) {
3805                                 const struct btf_type *func_proto;
3806                                 struct btf_param *param;
3807                                 int j;
3808
3809                                 func_proto = btf__type_by_id(obj->btf,
3810                                                              vt->type);
3811                                 param = btf_params(func_proto);
3812                                 /* Reuse the dummy_var string if the
3813                                  * func proto does not have param name.
3814                                  */
3815                                 for (j = 0; j < btf_vlen(func_proto); j++)
3816                                         if (param[j].type && !param[j].name_off)
3817                                                 param[j].name_off =
3818                                                         dummy_var->name_off;
3819                                 vs->type = dummy_var_btf_id;
3820                                 vt->info &= ~0xffff;
3821                                 vt->info |= BTF_FUNC_GLOBAL;
3822                         } else {
3823                                 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3824                                 vt->type = int_btf_id;
3825                         }
3826                         vs->offset = off;
3827                         vs->size = sizeof(int);
3828                 }
3829                 sec->size = off;
3830         }
3831
3832         if (kcfg_sec) {
3833                 sec = kcfg_sec;
3834                 /* for kcfg externs calculate their offsets within a .kconfig map */
3835                 off = 0;
3836                 for (i = 0; i < obj->nr_extern; i++) {
3837                         ext = &obj->externs[i];
3838                         if (ext->type != EXT_KCFG)
3839                                 continue;
3840
3841                         ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3842                         off = ext->kcfg.data_off + ext->kcfg.sz;
3843                         pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3844                                  i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3845                 }
3846                 sec->size = off;
3847                 n = btf_vlen(sec);
3848                 for (i = 0; i < n; i++) {
3849                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3850
3851                         t = btf__type_by_id(obj->btf, vs->type);
3852                         ext_name = btf__name_by_offset(obj->btf, t->name_off);
3853                         ext = find_extern_by_name(obj, ext_name);
3854                         if (!ext) {
3855                                 pr_warn("failed to find extern definition for BTF var '%s'\n",
3856                                         ext_name);
3857                                 return -ESRCH;
3858                         }
3859                         btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3860                         vs->offset = ext->kcfg.data_off;
3861                 }
3862         }
3863         return 0;
3864 }
3865
3866 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
3867 {
3868         return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3869 }
3870
3871 struct bpf_program *
3872 bpf_object__find_program_by_name(const struct bpf_object *obj,
3873                                  const char *name)
3874 {
3875         struct bpf_program *prog;
3876
3877         bpf_object__for_each_program(prog, obj) {
3878                 if (prog_is_subprog(obj, prog))
3879                         continue;
3880                 if (!strcmp(prog->name, name))
3881                         return prog;
3882         }
3883         return errno = ENOENT, NULL;
3884 }
3885
3886 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3887                                       int shndx)
3888 {
3889         switch (obj->efile.secs[shndx].sec_type) {
3890         case SEC_BSS:
3891         case SEC_DATA:
3892         case SEC_RODATA:
3893                 return true;
3894         default:
3895                 return false;
3896         }
3897 }
3898
3899 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3900                                       int shndx)
3901 {
3902         return shndx == obj->efile.btf_maps_shndx;
3903 }
3904
3905 static enum libbpf_map_type
3906 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3907 {
3908         if (shndx == obj->efile.symbols_shndx)
3909                 return LIBBPF_MAP_KCONFIG;
3910
3911         switch (obj->efile.secs[shndx].sec_type) {
3912         case SEC_BSS:
3913                 return LIBBPF_MAP_BSS;
3914         case SEC_DATA:
3915                 return LIBBPF_MAP_DATA;
3916         case SEC_RODATA:
3917                 return LIBBPF_MAP_RODATA;
3918         default:
3919                 return LIBBPF_MAP_UNSPEC;
3920         }
3921 }
3922
3923 static int bpf_program__record_reloc(struct bpf_program *prog,
3924                                      struct reloc_desc *reloc_desc,
3925                                      __u32 insn_idx, const char *sym_name,
3926                                      const Elf64_Sym *sym, const Elf64_Rel *rel)
3927 {
3928         struct bpf_insn *insn = &prog->insns[insn_idx];
3929         size_t map_idx, nr_maps = prog->obj->nr_maps;
3930         struct bpf_object *obj = prog->obj;
3931         __u32 shdr_idx = sym->st_shndx;
3932         enum libbpf_map_type type;
3933         const char *sym_sec_name;
3934         struct bpf_map *map;
3935
3936         if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
3937                 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
3938                         prog->name, sym_name, insn_idx, insn->code);
3939                 return -LIBBPF_ERRNO__RELOC;
3940         }
3941
3942         if (sym_is_extern(sym)) {
3943                 int sym_idx = ELF64_R_SYM(rel->r_info);
3944                 int i, n = obj->nr_extern;
3945                 struct extern_desc *ext;
3946
3947                 for (i = 0; i < n; i++) {
3948                         ext = &obj->externs[i];
3949                         if (ext->sym_idx == sym_idx)
3950                                 break;
3951                 }
3952                 if (i >= n) {
3953                         pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
3954                                 prog->name, sym_name, sym_idx);
3955                         return -LIBBPF_ERRNO__RELOC;
3956                 }
3957                 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
3958                          prog->name, i, ext->name, ext->sym_idx, insn_idx);
3959                 if (insn->code == (BPF_JMP | BPF_CALL))
3960                         reloc_desc->type = RELO_EXTERN_FUNC;
3961                 else
3962                         reloc_desc->type = RELO_EXTERN_VAR;
3963                 reloc_desc->insn_idx = insn_idx;
3964                 reloc_desc->sym_off = i; /* sym_off stores extern index */
3965                 return 0;
3966         }
3967
3968         /* sub-program call relocation */
3969         if (is_call_insn(insn)) {
3970                 if (insn->src_reg != BPF_PSEUDO_CALL) {
3971                         pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3972                         return -LIBBPF_ERRNO__RELOC;
3973                 }
3974                 /* text_shndx can be 0, if no default "main" program exists */
3975                 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3976                         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3977                         pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3978                                 prog->name, sym_name, sym_sec_name);
3979                         return -LIBBPF_ERRNO__RELOC;
3980                 }
3981                 if (sym->st_value % BPF_INSN_SZ) {
3982                         pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
3983                                 prog->name, sym_name, (size_t)sym->st_value);
3984                         return -LIBBPF_ERRNO__RELOC;
3985                 }
3986                 reloc_desc->type = RELO_CALL;
3987                 reloc_desc->insn_idx = insn_idx;
3988                 reloc_desc->sym_off = sym->st_value;
3989                 return 0;
3990         }
3991
3992         if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
3993                 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
3994                         prog->name, sym_name, shdr_idx);
3995                 return -LIBBPF_ERRNO__RELOC;
3996         }
3997
3998         /* loading subprog addresses */
3999         if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4000                 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4001                  * local_func: sym->st_value = 0, insn->imm = offset in the section.
4002                  */
4003                 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4004                         pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4005                                 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4006                         return -LIBBPF_ERRNO__RELOC;
4007                 }
4008
4009                 reloc_desc->type = RELO_SUBPROG_ADDR;
4010                 reloc_desc->insn_idx = insn_idx;
4011                 reloc_desc->sym_off = sym->st_value;
4012                 return 0;
4013         }
4014
4015         type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
4016         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4017
4018         /* generic map reference relocation */
4019         if (type == LIBBPF_MAP_UNSPEC) {
4020                 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
4021                         pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4022                                 prog->name, sym_name, sym_sec_name);
4023                         return -LIBBPF_ERRNO__RELOC;
4024                 }
4025                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4026                         map = &obj->maps[map_idx];
4027                         if (map->libbpf_type != type ||
4028                             map->sec_idx != sym->st_shndx ||
4029                             map->sec_offset != sym->st_value)
4030                                 continue;
4031                         pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
4032                                  prog->name, map_idx, map->name, map->sec_idx,
4033                                  map->sec_offset, insn_idx);
4034                         break;
4035                 }
4036                 if (map_idx >= nr_maps) {
4037                         pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4038                                 prog->name, sym_sec_name, (size_t)sym->st_value);
4039                         return -LIBBPF_ERRNO__RELOC;
4040                 }
4041                 reloc_desc->type = RELO_LD64;
4042                 reloc_desc->insn_idx = insn_idx;
4043                 reloc_desc->map_idx = map_idx;
4044                 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4045                 return 0;
4046         }
4047
4048         /* global data map relocation */
4049         if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
4050                 pr_warn("prog '%s': bad data relo against section '%s'\n",
4051                         prog->name, sym_sec_name);
4052                 return -LIBBPF_ERRNO__RELOC;
4053         }
4054         for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4055                 map = &obj->maps[map_idx];
4056                 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4057                         continue;
4058                 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
4059                          prog->name, map_idx, map->name, map->sec_idx,
4060                          map->sec_offset, insn_idx);
4061                 break;
4062         }
4063         if (map_idx >= nr_maps) {
4064                 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4065                         prog->name, sym_sec_name);
4066                 return -LIBBPF_ERRNO__RELOC;
4067         }
4068
4069         reloc_desc->type = RELO_DATA;
4070         reloc_desc->insn_idx = insn_idx;
4071         reloc_desc->map_idx = map_idx;
4072         reloc_desc->sym_off = sym->st_value;
4073         return 0;
4074 }
4075
4076 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4077 {
4078         return insn_idx >= prog->sec_insn_off &&
4079                insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4080 }
4081
4082 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
4083                                                  size_t sec_idx, size_t insn_idx)
4084 {
4085         int l = 0, r = obj->nr_programs - 1, m;
4086         struct bpf_program *prog;
4087
4088         while (l < r) {
4089                 m = l + (r - l + 1) / 2;
4090                 prog = &obj->programs[m];
4091
4092                 if (prog->sec_idx < sec_idx ||
4093                     (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4094                         l = m;
4095                 else
4096                         r = m - 1;
4097         }
4098         /* matching program could be at index l, but it still might be the
4099          * wrong one, so we need to double check conditions for the last time
4100          */
4101         prog = &obj->programs[l];
4102         if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4103                 return prog;
4104         return NULL;
4105 }
4106
4107 static int
4108 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)
4109 {
4110         const char *relo_sec_name, *sec_name;
4111         size_t sec_idx = shdr->sh_info, sym_idx;
4112         struct bpf_program *prog;
4113         struct reloc_desc *relos;
4114         int err, i, nrels;
4115         const char *sym_name;
4116         __u32 insn_idx;
4117         Elf_Scn *scn;
4118         Elf_Data *scn_data;
4119         Elf64_Sym *sym;
4120         Elf64_Rel *rel;
4121
4122         if (sec_idx >= obj->efile.sec_cnt)
4123                 return -EINVAL;
4124
4125         scn = elf_sec_by_idx(obj, sec_idx);
4126         scn_data = elf_sec_data(obj, scn);
4127
4128         relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4129         sec_name = elf_sec_name(obj, scn);
4130         if (!relo_sec_name || !sec_name)
4131                 return -EINVAL;
4132
4133         pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
4134                  relo_sec_name, sec_idx, sec_name);
4135         nrels = shdr->sh_size / shdr->sh_entsize;
4136
4137         for (i = 0; i < nrels; i++) {
4138                 rel = elf_rel_by_idx(data, i);
4139                 if (!rel) {
4140                         pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
4141                         return -LIBBPF_ERRNO__FORMAT;
4142                 }
4143
4144                 sym_idx = ELF64_R_SYM(rel->r_info);
4145                 sym = elf_sym_by_idx(obj, sym_idx);
4146                 if (!sym) {
4147                         pr_warn("sec '%s': symbol #%zu not found for relo #%d\n",
4148                                 relo_sec_name, sym_idx, i);
4149                         return -LIBBPF_ERRNO__FORMAT;
4150                 }
4151
4152                 if (sym->st_shndx >= obj->efile.sec_cnt) {
4153                         pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n",
4154                                 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4155                         return -LIBBPF_ERRNO__FORMAT;
4156                 }
4157
4158                 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4159                         pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
4160                                 relo_sec_name, (size_t)rel->r_offset, i);
4161                         return -LIBBPF_ERRNO__FORMAT;
4162                 }
4163
4164                 insn_idx = rel->r_offset / BPF_INSN_SZ;
4165                 /* relocations against static functions are recorded as
4166                  * relocations against the section that contains a function;
4167                  * in such case, symbol will be STT_SECTION and sym.st_name
4168                  * will point to empty string (0), so fetch section name
4169                  * instead
4170                  */
4171                 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4172                         sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4173                 else
4174                         sym_name = elf_sym_str(obj, sym->st_name);
4175                 sym_name = sym_name ?: "<?";
4176
4177                 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
4178                          relo_sec_name, i, insn_idx, sym_name);
4179
4180                 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4181                 if (!prog) {
4182                         pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
4183                                 relo_sec_name, i, sec_name, insn_idx);
4184                         continue;
4185                 }
4186
4187                 relos = libbpf_reallocarray(prog->reloc_desc,
4188                                             prog->nr_reloc + 1, sizeof(*relos));
4189                 if (!relos)
4190                         return -ENOMEM;
4191                 prog->reloc_desc = relos;
4192
4193                 /* adjust insn_idx to local BPF program frame of reference */
4194                 insn_idx -= prog->sec_insn_off;
4195                 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4196                                                 insn_idx, sym_name, sym, rel);
4197                 if (err)
4198                         return err;
4199
4200                 prog->nr_reloc++;
4201         }
4202         return 0;
4203 }
4204
4205 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
4206 {
4207         int id;
4208
4209         if (!obj->btf)
4210                 return -ENOENT;
4211
4212         /* if it's BTF-defined map, we don't need to search for type IDs.
4213          * For struct_ops map, it does not need btf_key_type_id and
4214          * btf_value_type_id.
4215          */
4216         if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4217                 return 0;
4218
4219         /*
4220          * LLVM annotates global data differently in BTF, that is,
4221          * only as '.data', '.bss' or '.rodata'.
4222          */
4223         if (!bpf_map__is_internal(map))
4224                 return -ENOENT;
4225
4226         id = btf__find_by_name(obj->btf, map->real_name);
4227         if (id < 0)
4228                 return id;
4229
4230         map->btf_key_type_id = 0;
4231         map->btf_value_type_id = id;
4232         return 0;
4233 }
4234
4235 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
4236 {
4237         char file[PATH_MAX], buff[4096];
4238         FILE *fp;
4239         __u32 val;
4240         int err;
4241
4242         snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
4243         memset(info, 0, sizeof(*info));
4244
4245         fp = fopen(file, "r");
4246         if (!fp) {
4247                 err = -errno;
4248                 pr_warn("failed to open %s: %d. No procfs support?\n", file,
4249                         err);
4250                 return err;
4251         }
4252
4253         while (fgets(buff, sizeof(buff), fp)) {
4254                 if (sscanf(buff, "map_type:\t%u", &val) == 1)
4255                         info->type = val;
4256                 else if (sscanf(buff, "key_size:\t%u", &val) == 1)
4257                         info->key_size = val;
4258                 else if (sscanf(buff, "value_size:\t%u", &val) == 1)
4259                         info->value_size = val;
4260                 else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
4261                         info->max_entries = val;
4262                 else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
4263                         info->map_flags = val;
4264         }
4265
4266         fclose(fp);
4267
4268         return 0;
4269 }
4270
4271 bool bpf_map__autocreate(const struct bpf_map *map)
4272 {
4273         return map->autocreate;
4274 }
4275
4276 int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
4277 {
4278         if (map->obj->loaded)
4279                 return libbpf_err(-EBUSY);
4280
4281         map->autocreate = autocreate;
4282         return 0;
4283 }
4284
4285 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
4286 {
4287         struct bpf_map_info info = {};
4288         __u32 len = sizeof(info), name_len;
4289         int new_fd, err;
4290         char *new_name;
4291
4292         err = bpf_obj_get_info_by_fd(fd, &info, &len);
4293         if (err && errno == EINVAL)
4294                 err = bpf_get_map_info_from_fdinfo(fd, &info);
4295         if (err)
4296                 return libbpf_err(err);
4297
4298         name_len = strlen(info.name);
4299         if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4300                 new_name = strdup(map->name);
4301         else
4302                 new_name = strdup(info.name);
4303
4304         if (!new_name)
4305                 return libbpf_err(-errno);
4306
4307         new_fd = open("/", O_RDONLY | O_CLOEXEC);
4308         if (new_fd < 0) {
4309                 err = -errno;
4310                 goto err_free_new_name;
4311         }
4312
4313         new_fd = dup3(fd, new_fd, O_CLOEXEC);
4314         if (new_fd < 0) {
4315                 err = -errno;
4316                 goto err_close_new_fd;
4317         }
4318
4319         err = zclose(map->fd);
4320         if (err) {
4321                 err = -errno;
4322                 goto err_close_new_fd;
4323         }
4324         free(map->name);
4325
4326         map->fd = new_fd;
4327         map->name = new_name;
4328         map->def.type = info.type;
4329         map->def.key_size = info.key_size;
4330         map->def.value_size = info.value_size;
4331         map->def.max_entries = info.max_entries;
4332         map->def.map_flags = info.map_flags;
4333         map->btf_key_type_id = info.btf_key_type_id;
4334         map->btf_value_type_id = info.btf_value_type_id;
4335         map->reused = true;
4336         map->map_extra = info.map_extra;
4337
4338         return 0;
4339
4340 err_close_new_fd:
4341         close(new_fd);
4342 err_free_new_name:
4343         free(new_name);
4344         return libbpf_err(err);
4345 }
4346
4347 __u32 bpf_map__max_entries(const struct bpf_map *map)
4348 {
4349         return map->def.max_entries;
4350 }
4351
4352 struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
4353 {
4354         if (!bpf_map_type__is_map_in_map(map->def.type))
4355                 return errno = EINVAL, NULL;
4356
4357         return map->inner_map;
4358 }
4359
4360 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
4361 {
4362         if (map->obj->loaded)
4363                 return libbpf_err(-EBUSY);
4364
4365         map->def.max_entries = max_entries;
4366
4367         /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4368         if (map->def.type == BPF_MAP_TYPE_RINGBUF)
4369                 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4370
4371         return 0;
4372 }
4373
4374 static int
4375 bpf_object__probe_loading(struct bpf_object *obj)
4376 {
4377         char *cp, errmsg[STRERR_BUFSIZE];
4378         struct bpf_insn insns[] = {
4379                 BPF_MOV64_IMM(BPF_REG_0, 0),
4380                 BPF_EXIT_INSN(),
4381         };
4382         int ret, insn_cnt = ARRAY_SIZE(insns);
4383
4384         if (obj->gen_loader)
4385                 return 0;
4386
4387         ret = bump_rlimit_memlock();
4388         if (ret)
4389                 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);
4390
4391         /* make sure basic loading works */
4392         ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4393         if (ret < 0)
4394                 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
4395         if (ret < 0) {
4396                 ret = errno;
4397                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4398                 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
4399                         "program. Make sure your kernel supports BPF "
4400                         "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
4401                         "set to big enough value.\n", __func__, cp, ret);
4402                 return -ret;
4403         }
4404         close(ret);
4405
4406         return 0;
4407 }
4408
4409 static int probe_fd(int fd)
4410 {
4411         if (fd >= 0)
4412                 close(fd);
4413         return fd >= 0;
4414 }
4415
4416 static int probe_kern_prog_name(void)
4417 {
4418         const size_t attr_sz = offsetofend(union bpf_attr, prog_name);
4419         struct bpf_insn insns[] = {
4420                 BPF_MOV64_IMM(BPF_REG_0, 0),
4421                 BPF_EXIT_INSN(),
4422         };
4423         union bpf_attr attr;
4424         int ret;
4425
4426         memset(&attr, 0, attr_sz);
4427         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4428         attr.license = ptr_to_u64("GPL");
4429         attr.insns = ptr_to_u64(insns);
4430         attr.insn_cnt = (__u32)ARRAY_SIZE(insns);
4431         libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name));
4432
4433         /* make sure loading with name works */
4434         ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS);
4435         return probe_fd(ret);
4436 }
4437
4438 static int probe_kern_global_data(void)
4439 {
4440         char *cp, errmsg[STRERR_BUFSIZE];
4441         struct bpf_insn insns[] = {
4442                 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
4443                 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
4444                 BPF_MOV64_IMM(BPF_REG_0, 0),
4445                 BPF_EXIT_INSN(),
4446         };
4447         int ret, map, insn_cnt = ARRAY_SIZE(insns);
4448
4449         map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL);
4450         if (map < 0) {
4451                 ret = -errno;
4452                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4453                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4454                         __func__, cp, -ret);
4455                 return ret;
4456         }
4457
4458         insns[0].imm = map;
4459
4460         ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4461         close(map);
4462         return probe_fd(ret);
4463 }
4464
4465 static int probe_kern_btf(void)
4466 {
4467         static const char strs[] = "\0int";
4468         __u32 types[] = {
4469                 /* int */
4470                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4471         };
4472
4473         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4474                                              strs, sizeof(strs)));
4475 }
4476
4477 static int probe_kern_btf_func(void)
4478 {
4479         static const char strs[] = "\0int\0x\0a";
4480         /* void x(int a) {} */
4481         __u32 types[] = {
4482                 /* int */
4483                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4484                 /* FUNC_PROTO */                                /* [2] */
4485                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4486                 BTF_PARAM_ENC(7, 1),
4487                 /* FUNC x */                                    /* [3] */
4488                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
4489         };
4490
4491         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4492                                              strs, sizeof(strs)));
4493 }
4494
4495 static int probe_kern_btf_func_global(void)
4496 {
4497         static const char strs[] = "\0int\0x\0a";
4498         /* static void x(int a) {} */
4499         __u32 types[] = {
4500                 /* int */
4501                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4502                 /* FUNC_PROTO */                                /* [2] */
4503                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4504                 BTF_PARAM_ENC(7, 1),
4505                 /* FUNC x BTF_FUNC_GLOBAL */                    /* [3] */
4506                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
4507         };
4508
4509         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4510                                              strs, sizeof(strs)));
4511 }
4512
4513 static int probe_kern_btf_datasec(void)
4514 {
4515         static const char strs[] = "\0x\0.data";
4516         /* static int a; */
4517         __u32 types[] = {
4518                 /* int */
4519                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4520                 /* VAR x */                                     /* [2] */
4521                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4522                 BTF_VAR_STATIC,
4523                 /* DATASEC val */                               /* [3] */
4524                 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
4525                 BTF_VAR_SECINFO_ENC(2, 0, 4),
4526         };
4527
4528         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4529                                              strs, sizeof(strs)));
4530 }
4531
4532 static int probe_kern_btf_float(void)
4533 {
4534         static const char strs[] = "\0float";
4535         __u32 types[] = {
4536                 /* float */
4537                 BTF_TYPE_FLOAT_ENC(1, 4),
4538         };
4539
4540         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4541                                              strs, sizeof(strs)));
4542 }
4543
4544 static int probe_kern_btf_decl_tag(void)
4545 {
4546         static const char strs[] = "\0tag";
4547         __u32 types[] = {
4548                 /* int */
4549                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4550                 /* VAR x */                                     /* [2] */
4551                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4552                 BTF_VAR_STATIC,
4553                 /* attr */
4554                 BTF_TYPE_DECL_TAG_ENC(1, 2, -1),
4555         };
4556
4557         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4558                                              strs, sizeof(strs)));
4559 }
4560
4561 static int probe_kern_btf_type_tag(void)
4562 {
4563         static const char strs[] = "\0tag";
4564         __u32 types[] = {
4565                 /* int */
4566                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),          /* [1] */
4567                 /* attr */
4568                 BTF_TYPE_TYPE_TAG_ENC(1, 1),                            /* [2] */
4569                 /* ptr */
4570                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2),   /* [3] */
4571         };
4572
4573         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4574                                              strs, sizeof(strs)));
4575 }
4576
4577 static int probe_kern_array_mmap(void)
4578 {
4579         LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE);
4580         int fd;
4581
4582         fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts);
4583         return probe_fd(fd);
4584 }
4585
4586 static int probe_kern_exp_attach_type(void)
4587 {
4588         LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE);
4589         struct bpf_insn insns[] = {
4590                 BPF_MOV64_IMM(BPF_REG_0, 0),
4591                 BPF_EXIT_INSN(),
4592         };
4593         int fd, insn_cnt = ARRAY_SIZE(insns);
4594
4595         /* use any valid combination of program type and (optional)
4596          * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
4597          * to see if kernel supports expected_attach_type field for
4598          * BPF_PROG_LOAD command
4599          */
4600         fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts);
4601         return probe_fd(fd);
4602 }
4603
4604 static int probe_kern_probe_read_kernel(void)
4605 {
4606         struct bpf_insn insns[] = {
4607                 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),   /* r1 = r10 (fp) */
4608                 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),  /* r1 += -8 */
4609                 BPF_MOV64_IMM(BPF_REG_2, 8),            /* r2 = 8 */
4610                 BPF_MOV64_IMM(BPF_REG_3, 0),            /* r3 = 0 */
4611                 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
4612                 BPF_EXIT_INSN(),
4613         };
4614         int fd, insn_cnt = ARRAY_SIZE(insns);
4615
4616         fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
4617         return probe_fd(fd);
4618 }
4619
4620 static int probe_prog_bind_map(void)
4621 {
4622         char *cp, errmsg[STRERR_BUFSIZE];
4623         struct bpf_insn insns[] = {
4624                 BPF_MOV64_IMM(BPF_REG_0, 0),
4625                 BPF_EXIT_INSN(),
4626         };
4627         int ret, map, prog, insn_cnt = ARRAY_SIZE(insns);
4628
4629         map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL);
4630         if (map < 0) {
4631                 ret = -errno;
4632                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4633                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4634                         __func__, cp, -ret);
4635                 return ret;
4636         }
4637
4638         prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4639         if (prog < 0) {
4640                 close(map);
4641                 return 0;
4642         }
4643
4644         ret = bpf_prog_bind_map(prog, map, NULL);
4645
4646         close(map);
4647         close(prog);
4648
4649         return ret >= 0;
4650 }
4651
4652 static int probe_module_btf(void)
4653 {
4654         static const char strs[] = "\0int";
4655         __u32 types[] = {
4656                 /* int */
4657                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4658         };
4659         struct bpf_btf_info info;
4660         __u32 len = sizeof(info);
4661         char name[16];
4662         int fd, err;
4663
4664         fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
4665         if (fd < 0)
4666                 return 0; /* BTF not supported at all */
4667
4668         memset(&info, 0, sizeof(info));
4669         info.name = ptr_to_u64(name);
4670         info.name_len = sizeof(name);
4671
4672         /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
4673          * kernel's module BTF support coincides with support for
4674          * name/name_len fields in struct bpf_btf_info.
4675          */
4676         err = bpf_obj_get_info_by_fd(fd, &info, &len);
4677         close(fd);
4678         return !err;
4679 }
4680
4681 static int probe_perf_link(void)
4682 {
4683         struct bpf_insn insns[] = {
4684                 BPF_MOV64_IMM(BPF_REG_0, 0),
4685                 BPF_EXIT_INSN(),
4686         };
4687         int prog_fd, link_fd, err;
4688
4689         prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL",
4690                                 insns, ARRAY_SIZE(insns), NULL);
4691         if (prog_fd < 0)
4692                 return -errno;
4693
4694         /* use invalid perf_event FD to get EBADF, if link is supported;
4695          * otherwise EINVAL should be returned
4696          */
4697         link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
4698         err = -errno; /* close() can clobber errno */
4699
4700         if (link_fd >= 0)
4701                 close(link_fd);
4702         close(prog_fd);
4703
4704         return link_fd < 0 && err == -EBADF;
4705 }
4706
4707 static int probe_kern_bpf_cookie(void)
4708 {
4709         struct bpf_insn insns[] = {
4710                 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie),
4711                 BPF_EXIT_INSN(),
4712         };
4713         int ret, insn_cnt = ARRAY_SIZE(insns);
4714
4715         ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL);
4716         return probe_fd(ret);
4717 }
4718
4719 static int probe_kern_btf_enum64(void)
4720 {
4721         static const char strs[] = "\0enum64";
4722         __u32 types[] = {
4723                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8),
4724         };
4725
4726         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4727                                              strs, sizeof(strs)));
4728 }
4729
4730 static int probe_kern_syscall_wrapper(void);
4731
4732 enum kern_feature_result {
4733         FEAT_UNKNOWN = 0,
4734         FEAT_SUPPORTED = 1,
4735         FEAT_MISSING = 2,
4736 };
4737
4738 typedef int (*feature_probe_fn)(void);
4739
4740 static struct kern_feature_desc {
4741         const char *desc;
4742         feature_probe_fn probe;
4743         enum kern_feature_result res;
4744 } feature_probes[__FEAT_CNT] = {
4745         [FEAT_PROG_NAME] = {
4746                 "BPF program name", probe_kern_prog_name,
4747         },
4748         [FEAT_GLOBAL_DATA] = {
4749                 "global variables", probe_kern_global_data,
4750         },
4751         [FEAT_BTF] = {
4752                 "minimal BTF", probe_kern_btf,
4753         },
4754         [FEAT_BTF_FUNC] = {
4755                 "BTF functions", probe_kern_btf_func,
4756         },
4757         [FEAT_BTF_GLOBAL_FUNC] = {
4758                 "BTF global function", probe_kern_btf_func_global,
4759         },
4760         [FEAT_BTF_DATASEC] = {
4761                 "BTF data section and variable", probe_kern_btf_datasec,
4762         },
4763         [FEAT_ARRAY_MMAP] = {
4764                 "ARRAY map mmap()", probe_kern_array_mmap,
4765         },
4766         [FEAT_EXP_ATTACH_TYPE] = {
4767                 "BPF_PROG_LOAD expected_attach_type attribute",
4768                 probe_kern_exp_attach_type,
4769         },
4770         [FEAT_PROBE_READ_KERN] = {
4771                 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
4772         },
4773         [FEAT_PROG_BIND_MAP] = {
4774                 "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
4775         },
4776         [FEAT_MODULE_BTF] = {
4777                 "module BTF support", probe_module_btf,
4778         },
4779         [FEAT_BTF_FLOAT] = {
4780                 "BTF_KIND_FLOAT support", probe_kern_btf_float,
4781         },
4782         [FEAT_PERF_LINK] = {
4783                 "BPF perf link support", probe_perf_link,
4784         },
4785         [FEAT_BTF_DECL_TAG] = {
4786                 "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag,
4787         },
4788         [FEAT_BTF_TYPE_TAG] = {
4789                 "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag,
4790         },
4791         [FEAT_MEMCG_ACCOUNT] = {
4792                 "memcg-based memory accounting", probe_memcg_account,
4793         },
4794         [FEAT_BPF_COOKIE] = {
4795                 "BPF cookie support", probe_kern_bpf_cookie,
4796         },
4797         [FEAT_BTF_ENUM64] = {
4798                 "BTF_KIND_ENUM64 support", probe_kern_btf_enum64,
4799         },
4800         [FEAT_SYSCALL_WRAPPER] = {
4801                 "Kernel using syscall wrapper", probe_kern_syscall_wrapper,
4802         },
4803 };
4804
4805 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
4806 {
4807         struct kern_feature_desc *feat = &feature_probes[feat_id];
4808         int ret;
4809
4810         if (obj && obj->gen_loader)
4811                 /* To generate loader program assume the latest kernel
4812                  * to avoid doing extra prog_load, map_create syscalls.
4813                  */
4814                 return true;
4815
4816         if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
4817                 ret = feat->probe();
4818                 if (ret > 0) {
4819                         WRITE_ONCE(feat->res, FEAT_SUPPORTED);
4820                 } else if (ret == 0) {
4821                         WRITE_ONCE(feat->res, FEAT_MISSING);
4822                 } else {
4823                         pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
4824                         WRITE_ONCE(feat->res, FEAT_MISSING);
4825                 }
4826         }
4827
4828         return READ_ONCE(feat->res) == FEAT_SUPPORTED;
4829 }
4830
4831 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4832 {
4833         struct bpf_map_info map_info = {};
4834         char msg[STRERR_BUFSIZE];
4835         __u32 map_info_len;
4836         int err;
4837
4838         map_info_len = sizeof(map_info);
4839
4840         err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len);
4841         if (err && errno == EINVAL)
4842                 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
4843         if (err) {
4844                 pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
4845                         libbpf_strerror_r(errno, msg, sizeof(msg)));
4846                 return false;
4847         }
4848
4849         return (map_info.type == map->def.type &&
4850                 map_info.key_size == map->def.key_size &&
4851                 map_info.value_size == map->def.value_size &&
4852                 map_info.max_entries == map->def.max_entries &&
4853                 map_info.map_flags == map->def.map_flags &&
4854                 map_info.map_extra == map->map_extra);
4855 }
4856
4857 static int
4858 bpf_object__reuse_map(struct bpf_map *map)
4859 {
4860         char *cp, errmsg[STRERR_BUFSIZE];
4861         int err, pin_fd;
4862
4863         pin_fd = bpf_obj_get(map->pin_path);
4864         if (pin_fd < 0) {
4865                 err = -errno;
4866                 if (err == -ENOENT) {
4867                         pr_debug("found no pinned map to reuse at '%s'\n",
4868                                  map->pin_path);
4869                         return 0;
4870                 }
4871
4872                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4873                 pr_warn("couldn't retrieve pinned map '%s': %s\n",
4874                         map->pin_path, cp);
4875                 return err;
4876         }
4877
4878         if (!map_is_reuse_compat(map, pin_fd)) {
4879                 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
4880                         map->pin_path);
4881                 close(pin_fd);
4882                 return -EINVAL;
4883         }
4884
4885         err = bpf_map__reuse_fd(map, pin_fd);
4886         close(pin_fd);
4887         if (err) {
4888                 return err;
4889         }
4890         map->pinned = true;
4891         pr_debug("reused pinned map at '%s'\n", map->pin_path);
4892
4893         return 0;
4894 }
4895
4896 static int
4897 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
4898 {
4899         enum libbpf_map_type map_type = map->libbpf_type;
4900         char *cp, errmsg[STRERR_BUFSIZE];
4901         int err, zero = 0;
4902
4903         if (obj->gen_loader) {
4904                 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
4905                                          map->mmaped, map->def.value_size);
4906                 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG)
4907                         bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
4908                 return 0;
4909         }
4910         err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
4911         if (err) {
4912                 err = -errno;
4913                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4914                 pr_warn("Error setting initial map(%s) contents: %s\n",
4915                         map->name, cp);
4916                 return err;
4917         }
4918
4919         /* Freeze .rodata and .kconfig map as read-only from syscall side. */
4920         if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
4921                 err = bpf_map_freeze(map->fd);
4922                 if (err) {
4923                         err = -errno;
4924                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4925                         pr_warn("Error freezing map(%s) as read-only: %s\n",
4926                                 map->name, cp);
4927                         return err;
4928                 }
4929         }
4930         return 0;
4931 }
4932
4933 static void bpf_map__destroy(struct bpf_map *map);
4934
4935 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
4936 {
4937         LIBBPF_OPTS(bpf_map_create_opts, create_attr);
4938         struct bpf_map_def *def = &map->def;
4939         const char *map_name = NULL;
4940         int err = 0;
4941
4942         if (kernel_supports(obj, FEAT_PROG_NAME))
4943                 map_name = map->name;
4944         create_attr.map_ifindex = map->map_ifindex;
4945         create_attr.map_flags = def->map_flags;
4946         create_attr.numa_node = map->numa_node;
4947         create_attr.map_extra = map->map_extra;
4948
4949         if (bpf_map__is_struct_ops(map))
4950                 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
4951
4952         if (obj->btf && btf__fd(obj->btf) >= 0) {
4953                 create_attr.btf_fd = btf__fd(obj->btf);
4954                 create_attr.btf_key_type_id = map->btf_key_type_id;
4955                 create_attr.btf_value_type_id = map->btf_value_type_id;
4956         }
4957
4958         if (bpf_map_type__is_map_in_map(def->type)) {
4959                 if (map->inner_map) {
4960                         err = bpf_object__create_map(obj, map->inner_map, true);
4961                         if (err) {
4962                                 pr_warn("map '%s': failed to create inner map: %d\n",
4963                                         map->name, err);
4964                                 return err;
4965                         }
4966                         map->inner_map_fd = bpf_map__fd(map->inner_map);
4967                 }
4968                 if (map->inner_map_fd >= 0)
4969                         create_attr.inner_map_fd = map->inner_map_fd;
4970         }
4971
4972         switch (def->type) {
4973         case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
4974         case BPF_MAP_TYPE_CGROUP_ARRAY:
4975         case BPF_MAP_TYPE_STACK_TRACE:
4976         case BPF_MAP_TYPE_ARRAY_OF_MAPS:
4977         case BPF_MAP_TYPE_HASH_OF_MAPS:
4978         case BPF_MAP_TYPE_DEVMAP:
4979         case BPF_MAP_TYPE_DEVMAP_HASH:
4980         case BPF_MAP_TYPE_CPUMAP:
4981         case BPF_MAP_TYPE_XSKMAP:
4982         case BPF_MAP_TYPE_SOCKMAP:
4983         case BPF_MAP_TYPE_SOCKHASH:
4984         case BPF_MAP_TYPE_QUEUE:
4985         case BPF_MAP_TYPE_STACK:
4986                 create_attr.btf_fd = 0;
4987                 create_attr.btf_key_type_id = 0;
4988                 create_attr.btf_value_type_id = 0;
4989                 map->btf_key_type_id = 0;
4990                 map->btf_value_type_id = 0;
4991         default:
4992                 break;
4993         }
4994
4995         if (obj->gen_loader) {
4996                 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
4997                                     def->key_size, def->value_size, def->max_entries,
4998                                     &create_attr, is_inner ? -1 : map - obj->maps);
4999                 /* Pretend to have valid FD to pass various fd >= 0 checks.
5000                  * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
5001                  */
5002                 map->fd = 0;
5003         } else {
5004                 map->fd = bpf_map_create(def->type, map_name,
5005                                          def->key_size, def->value_size,
5006                                          def->max_entries, &create_attr);
5007         }
5008         if (map->fd < 0 && (create_attr.btf_key_type_id ||
5009                             create_attr.btf_value_type_id)) {
5010                 char *cp, errmsg[STRERR_BUFSIZE];
5011
5012                 err = -errno;
5013                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5014                 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
5015                         map->name, cp, err);
5016                 create_attr.btf_fd = 0;
5017                 create_attr.btf_key_type_id = 0;
5018                 create_attr.btf_value_type_id = 0;
5019                 map->btf_key_type_id = 0;
5020                 map->btf_value_type_id = 0;
5021                 map->fd = bpf_map_create(def->type, map_name,
5022                                          def->key_size, def->value_size,
5023                                          def->max_entries, &create_attr);
5024         }
5025
5026         err = map->fd < 0 ? -errno : 0;
5027
5028         if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5029                 if (obj->gen_loader)
5030                         map->inner_map->fd = -1;
5031                 bpf_map__destroy(map->inner_map);
5032                 zfree(&map->inner_map);
5033         }
5034
5035         return err;
5036 }
5037
5038 static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map)
5039 {
5040         const struct bpf_map *targ_map;
5041         unsigned int i;
5042         int fd, err = 0;
5043
5044         for (i = 0; i < map->init_slots_sz; i++) {
5045                 if (!map->init_slots[i])
5046                         continue;
5047
5048                 targ_map = map->init_slots[i];
5049                 fd = bpf_map__fd(targ_map);
5050
5051                 if (obj->gen_loader) {
5052                         bpf_gen__populate_outer_map(obj->gen_loader,
5053                                                     map - obj->maps, i,
5054                                                     targ_map - obj->maps);
5055                 } else {
5056                         err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5057                 }
5058                 if (err) {
5059                         err = -errno;
5060                         pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
5061                                 map->name, i, targ_map->name, fd, err);
5062                         return err;
5063                 }
5064                 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
5065                          map->name, i, targ_map->name, fd);
5066         }
5067
5068         zfree(&map->init_slots);
5069         map->init_slots_sz = 0;
5070
5071         return 0;
5072 }
5073
5074 static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map)
5075 {
5076         const struct bpf_program *targ_prog;
5077         unsigned int i;
5078         int fd, err;
5079
5080         if (obj->gen_loader)
5081                 return -ENOTSUP;
5082
5083         for (i = 0; i < map->init_slots_sz; i++) {
5084                 if (!map->init_slots[i])
5085                         continue;
5086
5087                 targ_prog = map->init_slots[i];
5088                 fd = bpf_program__fd(targ_prog);
5089
5090                 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5091                 if (err) {
5092                         err = -errno;
5093                         pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",
5094                                 map->name, i, targ_prog->name, fd, err);
5095                         return err;
5096                 }
5097                 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
5098                          map->name, i, targ_prog->name, fd);
5099         }
5100
5101         zfree(&map->init_slots);
5102         map->init_slots_sz = 0;
5103
5104         return 0;
5105 }
5106
5107 static int bpf_object_init_prog_arrays(struct bpf_object *obj)
5108 {
5109         struct bpf_map *map;
5110         int i, err;
5111
5112         for (i = 0; i < obj->nr_maps; i++) {
5113                 map = &obj->maps[i];
5114
5115                 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5116                         continue;
5117
5118                 err = init_prog_array_slots(obj, map);
5119                 if (err < 0) {
5120                         zclose(map->fd);
5121                         return err;
5122                 }
5123         }
5124         return 0;
5125 }
5126
5127 static int map_set_def_max_entries(struct bpf_map *map)
5128 {
5129         if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5130                 int nr_cpus;
5131
5132                 nr_cpus = libbpf_num_possible_cpus();
5133                 if (nr_cpus < 0) {
5134                         pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
5135                                 map->name, nr_cpus);
5136                         return nr_cpus;
5137                 }
5138                 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5139                 map->def.max_entries = nr_cpus;
5140         }
5141
5142         return 0;
5143 }
5144
5145 static int
5146 bpf_object__create_maps(struct bpf_object *obj)
5147 {
5148         struct bpf_map *map;
5149         char *cp, errmsg[STRERR_BUFSIZE];
5150         unsigned int i, j;
5151         int err;
5152         bool retried;
5153
5154         for (i = 0; i < obj->nr_maps; i++) {
5155                 map = &obj->maps[i];
5156
5157                 /* To support old kernels, we skip creating global data maps
5158                  * (.rodata, .data, .kconfig, etc); later on, during program
5159                  * loading, if we detect that at least one of the to-be-loaded
5160                  * programs is referencing any global data map, we'll error
5161                  * out with program name and relocation index logged.
5162                  * This approach allows to accommodate Clang emitting
5163                  * unnecessary .rodata.str1.1 sections for string literals,
5164                  * but also it allows to have CO-RE applications that use
5165                  * global variables in some of BPF programs, but not others.
5166                  * If those global variable-using programs are not loaded at
5167                  * runtime due to bpf_program__set_autoload(prog, false),
5168                  * bpf_object loading will succeed just fine even on old
5169                  * kernels.
5170                  */
5171                 if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA))
5172                         map->autocreate = false;
5173
5174                 if (!map->autocreate) {
5175                         pr_debug("map '%s': skipped auto-creating...\n", map->name);
5176                         continue;
5177                 }
5178
5179                 err = map_set_def_max_entries(map);
5180                 if (err)
5181                         goto err_out;
5182
5183                 retried = false;
5184 retry:
5185                 if (map->pin_path) {
5186                         err = bpf_object__reuse_map(map);
5187                         if (err) {
5188                                 pr_warn("map '%s': error reusing pinned map\n",
5189                                         map->name);
5190                                 goto err_out;
5191                         }
5192                         if (retried && map->fd < 0) {
5193                                 pr_warn("map '%s': cannot find pinned map\n",
5194                                         map->name);
5195                                 err = -ENOENT;
5196                                 goto err_out;
5197                         }
5198                 }
5199
5200                 if (map->fd >= 0) {
5201                         pr_debug("map '%s': skipping creation (preset fd=%d)\n",
5202                                  map->name, map->fd);
5203                 } else {
5204                         err = bpf_object__create_map(obj, map, false);
5205                         if (err)
5206                                 goto err_out;
5207
5208                         pr_debug("map '%s': created successfully, fd=%d\n",
5209                                  map->name, map->fd);
5210
5211                         if (bpf_map__is_internal(map)) {
5212                                 err = bpf_object__populate_internal_map(obj, map);
5213                                 if (err < 0) {
5214                                         zclose(map->fd);
5215                                         goto err_out;
5216                                 }
5217                         }
5218
5219                         if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5220                                 err = init_map_in_map_slots(obj, map);
5221                                 if (err < 0) {
5222                                         zclose(map->fd);
5223                                         goto err_out;
5224                                 }
5225                         }
5226                 }
5227
5228                 if (map->pin_path && !map->pinned) {
5229                         err = bpf_map__pin(map, NULL);
5230                         if (err) {
5231                                 zclose(map->fd);
5232                                 if (!retried && err == -EEXIST) {
5233                                         retried = true;
5234                                         goto retry;
5235                                 }
5236                                 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
5237                                         map->name, map->pin_path, err);
5238                                 goto err_out;
5239                         }
5240                 }
5241         }
5242
5243         return 0;
5244
5245 err_out:
5246         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5247         pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5248         pr_perm_msg(err);
5249         for (j = 0; j < i; j++)
5250                 zclose(obj->maps[j].fd);
5251         return err;
5252 }
5253
5254 static bool bpf_core_is_flavor_sep(const char *s)
5255 {
5256         /* check X___Y name pattern, where X and Y are not underscores */
5257         return s[0] != '_' &&                                 /* X */
5258                s[1] == '_' && s[2] == '_' && s[3] == '_' &&   /* ___ */
5259                s[4] != '_';                                   /* Y */
5260 }
5261
5262 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
5263  * before last triple underscore. Struct name part after last triple
5264  * underscore is ignored by BPF CO-RE relocation during relocation matching.
5265  */
5266 size_t bpf_core_essential_name_len(const char *name)
5267 {
5268         size_t n = strlen(name);
5269         int i;
5270
5271         for (i = n - 5; i >= 0; i--) {
5272                 if (bpf_core_is_flavor_sep(name + i))
5273                         return i + 1;
5274         }
5275         return n;
5276 }
5277
5278 void bpf_core_free_cands(struct bpf_core_cand_list *cands)
5279 {
5280         if (!cands)
5281                 return;
5282
5283         free(cands->cands);
5284         free(cands);
5285 }
5286
5287 int bpf_core_add_cands(struct bpf_core_cand *local_cand,
5288                        size_t local_essent_len,
5289                        const struct btf *targ_btf,
5290                        const char *targ_btf_name,
5291                        int targ_start_id,
5292                        struct bpf_core_cand_list *cands)
5293 {
5294         struct bpf_core_cand *new_cands, *cand;
5295         const struct btf_type *t, *local_t;
5296         const char *targ_name, *local_name;
5297         size_t targ_essent_len;
5298         int n, i;
5299
5300         local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5301         local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5302
5303         n = btf__type_cnt(targ_btf);
5304         for (i = targ_start_id; i < n; i++) {
5305                 t = btf__type_by_id(targ_btf, i);
5306                 if (!btf_kind_core_compat(t, local_t))
5307                         continue;
5308
5309                 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5310                 if (str_is_empty(targ_name))
5311                         continue;
5312
5313                 targ_essent_len = bpf_core_essential_name_len(targ_name);
5314                 if (targ_essent_len != local_essent_len)
5315                         continue;
5316
5317                 if (strncmp(local_name, targ_name, local_essent_len) != 0)
5318                         continue;
5319
5320                 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
5321                          local_cand->id, btf_kind_str(local_t),
5322                          local_name, i, btf_kind_str(t), targ_name,
5323                          targ_btf_name);
5324                 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5325                                               sizeof(*cands->cands));
5326                 if (!new_cands)
5327                         return -ENOMEM;
5328
5329                 cand = &new_cands[cands->len];
5330                 cand->btf = targ_btf;
5331                 cand->id = i;
5332
5333                 cands->cands = new_cands;
5334                 cands->len++;
5335         }
5336         return 0;
5337 }
5338
5339 static int load_module_btfs(struct bpf_object *obj)
5340 {
5341         struct bpf_btf_info info;
5342         struct module_btf *mod_btf;
5343         struct btf *btf;
5344         char name[64];
5345         __u32 id = 0, len;
5346         int err, fd;
5347
5348         if (obj->btf_modules_loaded)
5349                 return 0;
5350
5351         if (obj->gen_loader)
5352                 return 0;
5353
5354         /* don't do this again, even if we find no module BTFs */
5355         obj->btf_modules_loaded = true;
5356
5357         /* kernel too old to support module BTFs */
5358         if (!kernel_supports(obj, FEAT_MODULE_BTF))
5359                 return 0;
5360
5361         while (true) {
5362                 err = bpf_btf_get_next_id(id, &id);
5363                 if (err && errno == ENOENT)
5364                         return 0;
5365                 if (err) {
5366                         err = -errno;
5367                         pr_warn("failed to iterate BTF objects: %d\n", err);
5368                         return err;
5369                 }
5370
5371                 fd = bpf_btf_get_fd_by_id(id);
5372                 if (fd < 0) {
5373                         if (errno == ENOENT)
5374                                 continue; /* expected race: BTF was unloaded */
5375                         err = -errno;
5376                         pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
5377                         return err;
5378                 }
5379
5380                 len = sizeof(info);
5381                 memset(&info, 0, sizeof(info));
5382                 info.name = ptr_to_u64(name);
5383                 info.name_len = sizeof(name);
5384
5385                 err = bpf_obj_get_info_by_fd(fd, &info, &len);
5386                 if (err) {
5387                         err = -errno;
5388                         pr_warn("failed to get BTF object #%d info: %d\n", id, err);
5389                         goto err_out;
5390                 }
5391
5392                 /* ignore non-module BTFs */
5393                 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
5394                         close(fd);
5395                         continue;
5396                 }
5397
5398                 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5399                 err = libbpf_get_error(btf);
5400                 if (err) {
5401                         pr_warn("failed to load module [%s]'s BTF object #%d: %d\n",
5402                                 name, id, err);
5403                         goto err_out;
5404                 }
5405
5406                 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5407                                         sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5408                 if (err)
5409                         goto err_out;
5410
5411                 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
5412
5413                 mod_btf->btf = btf;
5414                 mod_btf->id = id;
5415                 mod_btf->fd = fd;
5416                 mod_btf->name = strdup(name);
5417                 if (!mod_btf->name) {
5418                         err = -ENOMEM;
5419                         goto err_out;
5420                 }
5421                 continue;
5422
5423 err_out:
5424                 close(fd);
5425                 return err;
5426         }
5427
5428         return 0;
5429 }
5430
5431 static struct bpf_core_cand_list *
5432 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
5433 {
5434         struct bpf_core_cand local_cand = {};
5435         struct bpf_core_cand_list *cands;
5436         const struct btf *main_btf;
5437         const struct btf_type *local_t;
5438         const char *local_name;
5439         size_t local_essent_len;
5440         int err, i;
5441
5442         local_cand.btf = local_btf;
5443         local_cand.id = local_type_id;
5444         local_t = btf__type_by_id(local_btf, local_type_id);
5445         if (!local_t)
5446                 return ERR_PTR(-EINVAL);
5447
5448         local_name = btf__name_by_offset(local_btf, local_t->name_off);
5449         if (str_is_empty(local_name))
5450                 return ERR_PTR(-EINVAL);
5451         local_essent_len = bpf_core_essential_name_len(local_name);
5452
5453         cands = calloc(1, sizeof(*cands));
5454         if (!cands)
5455                 return ERR_PTR(-ENOMEM);
5456
5457         /* Attempt to find target candidates in vmlinux BTF first */
5458         main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5459         err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
5460         if (err)
5461                 goto err_out;
5462
5463         /* if vmlinux BTF has any candidate, don't got for module BTFs */
5464         if (cands->len)
5465                 return cands;
5466
5467         /* if vmlinux BTF was overridden, don't attempt to load module BTFs */
5468         if (obj->btf_vmlinux_override)
5469                 return cands;
5470
5471         /* now look through module BTFs, trying to still find candidates */
5472         err = load_module_btfs(obj);
5473         if (err)
5474                 goto err_out;
5475
5476         for (i = 0; i < obj->btf_module_cnt; i++) {
5477                 err = bpf_core_add_cands(&local_cand, local_essent_len,
5478                                          obj->btf_modules[i].btf,
5479                                          obj->btf_modules[i].name,
5480                                          btf__type_cnt(obj->btf_vmlinux),
5481                                          cands);
5482                 if (err)
5483                         goto err_out;
5484         }
5485
5486         return cands;
5487 err_out:
5488         bpf_core_free_cands(cands);
5489         return ERR_PTR(err);
5490 }
5491
5492 /* Check local and target types for compatibility. This check is used for
5493  * type-based CO-RE relocations and follow slightly different rules than
5494  * field-based relocations. This function assumes that root types were already
5495  * checked for name match. Beyond that initial root-level name check, names
5496  * are completely ignored. Compatibility rules are as follows:
5497  *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5498  *     kind should match for local and target types (i.e., STRUCT is not
5499  *     compatible with UNION);
5500  *   - for ENUMs, the size is ignored;
5501  *   - for INT, size and signedness are ignored;
5502  *   - for ARRAY, dimensionality is ignored, element types are checked for
5503  *     compatibility recursively;
5504  *   - CONST/VOLATILE/RESTRICT modifiers are ignored;
5505  *   - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5506  *   - FUNC_PROTOs are compatible if they have compatible signature: same
5507  *     number of input args and compatible return and argument types.
5508  * These rules are not set in stone and probably will be adjusted as we get
5509  * more experience with using BPF CO-RE relocations.
5510  */
5511 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5512                               const struct btf *targ_btf, __u32 targ_id)
5513 {
5514         return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32);
5515 }
5516
5517 int bpf_core_types_match(const struct btf *local_btf, __u32 local_id,
5518                          const struct btf *targ_btf, __u32 targ_id)
5519 {
5520         return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32);
5521 }
5522
5523 static size_t bpf_core_hash_fn(const void *key, void *ctx)
5524 {
5525         return (size_t)key;
5526 }
5527
5528 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
5529 {
5530         return k1 == k2;
5531 }
5532
5533 static void *u32_as_hash_key(__u32 x)
5534 {
5535         return (void *)(uintptr_t)x;
5536 }
5537
5538 static int record_relo_core(struct bpf_program *prog,
5539                             const struct bpf_core_relo *core_relo, int insn_idx)
5540 {
5541         struct reloc_desc *relos, *relo;
5542
5543         relos = libbpf_reallocarray(prog->reloc_desc,
5544                                     prog->nr_reloc + 1, sizeof(*relos));
5545         if (!relos)
5546                 return -ENOMEM;
5547         relo = &relos[prog->nr_reloc];
5548         relo->type = RELO_CORE;
5549         relo->insn_idx = insn_idx;
5550         relo->core_relo = core_relo;
5551         prog->reloc_desc = relos;
5552         prog->nr_reloc++;
5553         return 0;
5554 }
5555
5556 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx)
5557 {
5558         struct reloc_desc *relo;
5559         int i;
5560
5561         for (i = 0; i < prog->nr_reloc; i++) {
5562                 relo = &prog->reloc_desc[i];
5563                 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
5564                         continue;
5565
5566                 return relo->core_relo;
5567         }
5568
5569         return NULL;
5570 }
5571
5572 static int bpf_core_resolve_relo(struct bpf_program *prog,
5573                                  const struct bpf_core_relo *relo,
5574                                  int relo_idx,
5575                                  const struct btf *local_btf,
5576                                  struct hashmap *cand_cache,
5577                                  struct bpf_core_relo_res *targ_res)
5578 {
5579         struct bpf_core_spec specs_scratch[3] = {};
5580         const void *type_key = u32_as_hash_key(relo->type_id);
5581         struct bpf_core_cand_list *cands = NULL;
5582         const char *prog_name = prog->name;
5583         const struct btf_type *local_type;
5584         const char *local_name;
5585         __u32 local_id = relo->type_id;
5586         int err;
5587
5588         local_type = btf__type_by_id(local_btf, local_id);
5589         if (!local_type)
5590                 return -EINVAL;
5591
5592         local_name = btf__name_by_offset(local_btf, local_type->name_off);
5593         if (!local_name)
5594                 return -EINVAL;
5595
5596         if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
5597             !hashmap__find(cand_cache, type_key, (void **)&cands)) {
5598                 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5599                 if (IS_ERR(cands)) {
5600                         pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5601                                 prog_name, relo_idx, local_id, btf_kind_str(local_type),
5602                                 local_name, PTR_ERR(cands));
5603                         return PTR_ERR(cands);
5604                 }
5605                 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
5606                 if (err) {
5607                         bpf_core_free_cands(cands);
5608                         return err;
5609                 }
5610         }
5611
5612         return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch,
5613                                        targ_res);
5614 }
5615
5616 static int
5617 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
5618 {
5619         const struct btf_ext_info_sec *sec;
5620         struct bpf_core_relo_res targ_res;
5621         const struct bpf_core_relo *rec;
5622         const struct btf_ext_info *seg;
5623         struct hashmap_entry *entry;
5624         struct hashmap *cand_cache = NULL;
5625         struct bpf_program *prog;
5626         struct bpf_insn *insn;
5627         const char *sec_name;
5628         int i, err = 0, insn_idx, sec_idx, sec_num;
5629
5630         if (obj->btf_ext->core_relo_info.len == 0)
5631                 return 0;
5632
5633         if (targ_btf_path) {
5634                 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
5635                 err = libbpf_get_error(obj->btf_vmlinux_override);
5636                 if (err) {
5637                         pr_warn("failed to parse target BTF: %d\n", err);
5638                         return err;
5639                 }
5640         }
5641
5642         cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
5643         if (IS_ERR(cand_cache)) {
5644                 err = PTR_ERR(cand_cache);
5645                 goto out;
5646         }
5647
5648         seg = &obj->btf_ext->core_relo_info;
5649         sec_num = 0;
5650         for_each_btf_ext_sec(seg, sec) {
5651                 sec_idx = seg->sec_idxs[sec_num];
5652                 sec_num++;
5653
5654                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
5655                 if (str_is_empty(sec_name)) {
5656                         err = -EINVAL;
5657                         goto out;
5658                 }
5659
5660                 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
5661
5662                 for_each_btf_ext_rec(seg, sec, i, rec) {
5663                         if (rec->insn_off % BPF_INSN_SZ)
5664                                 return -EINVAL;
5665                         insn_idx = rec->insn_off / BPF_INSN_SZ;
5666                         prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
5667                         if (!prog) {
5668                                 /* When __weak subprog is "overridden" by another instance
5669                                  * of the subprog from a different object file, linker still
5670                                  * appends all the .BTF.ext info that used to belong to that
5671                                  * eliminated subprogram.
5672                                  * This is similar to what x86-64 linker does for relocations.
5673                                  * So just ignore such relocations just like we ignore
5674                                  * subprog instructions when discovering subprograms.
5675                                  */
5676                                 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n",
5677                                          sec_name, i, insn_idx);
5678                                 continue;
5679                         }
5680                         /* no need to apply CO-RE relocation if the program is
5681                          * not going to be loaded
5682                          */
5683                         if (!prog->autoload)
5684                                 continue;
5685
5686                         /* adjust insn_idx from section frame of reference to the local
5687                          * program's frame of reference; (sub-)program code is not yet
5688                          * relocated, so it's enough to just subtract in-section offset
5689                          */
5690                         insn_idx = insn_idx - prog->sec_insn_off;
5691                         if (insn_idx >= prog->insns_cnt)
5692                                 return -EINVAL;
5693                         insn = &prog->insns[insn_idx];
5694
5695                         err = record_relo_core(prog, rec, insn_idx);
5696                         if (err) {
5697                                 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",
5698                                         prog->name, i, err);
5699                                 goto out;
5700                         }
5701
5702                         if (prog->obj->gen_loader)
5703                                 continue;
5704
5705                         err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
5706                         if (err) {
5707                                 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
5708                                         prog->name, i, err);
5709                                 goto out;
5710                         }
5711
5712                         err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
5713                         if (err) {
5714                                 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",
5715                                         prog->name, i, insn_idx, err);
5716                                 goto out;
5717                         }
5718                 }
5719         }
5720
5721 out:
5722         /* obj->btf_vmlinux and module BTFs are freed after object load */
5723         btf__free(obj->btf_vmlinux_override);
5724         obj->btf_vmlinux_override = NULL;
5725
5726         if (!IS_ERR_OR_NULL(cand_cache)) {
5727                 hashmap__for_each_entry(cand_cache, entry, i) {
5728                         bpf_core_free_cands(entry->value);
5729                 }
5730                 hashmap__free(cand_cache);
5731         }
5732         return err;
5733 }
5734
5735 /* base map load ldimm64 special constant, used also for log fixup logic */
5736 #define MAP_LDIMM64_POISON_BASE 2001000000
5737 #define MAP_LDIMM64_POISON_PFX "200100"
5738
5739 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,
5740                                int insn_idx, struct bpf_insn *insn,
5741                                int map_idx, const struct bpf_map *map)
5742 {
5743         int i;
5744
5745         pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",
5746                  prog->name, relo_idx, insn_idx, map_idx, map->name);
5747
5748         /* we turn single ldimm64 into two identical invalid calls */
5749         for (i = 0; i < 2; i++) {
5750                 insn->code = BPF_JMP | BPF_CALL;
5751                 insn->dst_reg = 0;
5752                 insn->src_reg = 0;
5753                 insn->off = 0;
5754                 /* if this instruction is reachable (not a dead code),
5755                  * verifier will complain with something like:
5756                  * invalid func unknown#2001000123
5757                  * where lower 123 is map index into obj->maps[] array
5758                  */
5759                 insn->imm = MAP_LDIMM64_POISON_BASE + map_idx;
5760
5761                 insn++;
5762         }
5763 }
5764
5765 /* Relocate data references within program code:
5766  *  - map references;
5767  *  - global variable references;
5768  *  - extern references.
5769  */
5770 static int
5771 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
5772 {
5773         int i;
5774
5775         for (i = 0; i < prog->nr_reloc; i++) {
5776                 struct reloc_desc *relo = &prog->reloc_desc[i];
5777                 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
5778                 const struct bpf_map *map;
5779                 struct extern_desc *ext;
5780
5781                 switch (relo->type) {
5782                 case RELO_LD64:
5783                         map = &obj->maps[relo->map_idx];
5784                         if (obj->gen_loader) {
5785                                 insn[0].src_reg = BPF_PSEUDO_MAP_IDX;
5786                                 insn[0].imm = relo->map_idx;
5787                         } else if (map->autocreate) {
5788                                 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
5789                                 insn[0].imm = map->fd;
5790                         } else {
5791                                 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
5792                                                    relo->map_idx, map);
5793                         }
5794                         break;
5795                 case RELO_DATA:
5796                         map = &obj->maps[relo->map_idx];
5797                         insn[1].imm = insn[0].imm + relo->sym_off;
5798                         if (obj->gen_loader) {
5799                                 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
5800                                 insn[0].imm = relo->map_idx;
5801                         } else if (map->autocreate) {
5802                                 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5803                                 insn[0].imm = map->fd;
5804                         } else {
5805                                 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
5806                                                    relo->map_idx, map);
5807                         }
5808                         break;
5809                 case RELO_EXTERN_VAR:
5810                         ext = &obj->externs[relo->sym_off];
5811                         if (ext->type == EXT_KCFG) {
5812                                 if (obj->gen_loader) {
5813                                         insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
5814                                         insn[0].imm = obj->kconfig_map_idx;
5815                                 } else {
5816                                         insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5817                                         insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
5818                                 }
5819                                 insn[1].imm = ext->kcfg.data_off;
5820                         } else /* EXT_KSYM */ {
5821                                 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
5822                                         insn[0].src_reg = BPF_PSEUDO_BTF_ID;
5823                                         insn[0].imm = ext->ksym.kernel_btf_id;
5824                                         insn[1].imm = ext->ksym.kernel_btf_obj_fd;
5825                                 } else { /* typeless ksyms or unresolved typed ksyms */
5826                                         insn[0].imm = (__u32)ext->ksym.addr;
5827                                         insn[1].imm = ext->ksym.addr >> 32;
5828                                 }
5829                         }
5830                         break;
5831                 case RELO_EXTERN_FUNC:
5832                         ext = &obj->externs[relo->sym_off];
5833                         insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
5834                         if (ext->is_set) {
5835                                 insn[0].imm = ext->ksym.kernel_btf_id;
5836                                 insn[0].off = ext->ksym.btf_fd_idx;
5837                         } else { /* unresolved weak kfunc */
5838                                 insn[0].imm = 0;
5839                                 insn[0].off = 0;
5840                         }
5841                         break;
5842                 case RELO_SUBPROG_ADDR:
5843                         if (insn[0].src_reg != BPF_PSEUDO_FUNC) {
5844                                 pr_warn("prog '%s': relo #%d: bad insn\n",
5845                                         prog->name, i);
5846                                 return -EINVAL;
5847                         }
5848                         /* handled already */
5849                         break;
5850                 case RELO_CALL:
5851                         /* handled already */
5852                         break;
5853                 case RELO_CORE:
5854                         /* will be handled by bpf_program_record_relos() */
5855                         break;
5856                 default:
5857                         pr_warn("prog '%s': relo #%d: bad relo type %d\n",
5858                                 prog->name, i, relo->type);
5859                         return -EINVAL;
5860                 }
5861         }
5862
5863         return 0;
5864 }
5865
5866 static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
5867                                     const struct bpf_program *prog,
5868                                     const struct btf_ext_info *ext_info,
5869                                     void **prog_info, __u32 *prog_rec_cnt,
5870                                     __u32 *prog_rec_sz)
5871 {
5872         void *copy_start = NULL, *copy_end = NULL;
5873         void *rec, *rec_end, *new_prog_info;
5874         const struct btf_ext_info_sec *sec;
5875         size_t old_sz, new_sz;
5876         int i, sec_num, sec_idx, off_adj;
5877
5878         sec_num = 0;
5879         for_each_btf_ext_sec(ext_info, sec) {
5880                 sec_idx = ext_info->sec_idxs[sec_num];
5881                 sec_num++;
5882                 if (prog->sec_idx != sec_idx)
5883                         continue;
5884
5885                 for_each_btf_ext_rec(ext_info, sec, i, rec) {
5886                         __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
5887
5888                         if (insn_off < prog->sec_insn_off)
5889                                 continue;
5890                         if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
5891                                 break;
5892
5893                         if (!copy_start)
5894                                 copy_start = rec;
5895                         copy_end = rec + ext_info->rec_size;
5896                 }
5897
5898                 if (!copy_start)
5899                         return -ENOENT;
5900
5901                 /* append func/line info of a given (sub-)program to the main
5902                  * program func/line info
5903                  */
5904                 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
5905                 new_sz = old_sz + (copy_end - copy_start);
5906                 new_prog_info = realloc(*prog_info, new_sz);
5907                 if (!new_prog_info)
5908                         return -ENOMEM;
5909                 *prog_info = new_prog_info;
5910                 *prog_rec_cnt = new_sz / ext_info->rec_size;
5911                 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
5912
5913                 /* Kernel instruction offsets are in units of 8-byte
5914                  * instructions, while .BTF.ext instruction offsets generated
5915                  * by Clang are in units of bytes. So convert Clang offsets
5916                  * into kernel offsets and adjust offset according to program
5917                  * relocated position.
5918                  */
5919                 off_adj = prog->sub_insn_off - prog->sec_insn_off;
5920                 rec = new_prog_info + old_sz;
5921                 rec_end = new_prog_info + new_sz;
5922                 for (; rec < rec_end; rec += ext_info->rec_size) {
5923                         __u32 *insn_off = rec;
5924
5925                         *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
5926                 }
5927                 *prog_rec_sz = ext_info->rec_size;
5928                 return 0;
5929         }
5930
5931         return -ENOENT;
5932 }
5933
5934 static int
5935 reloc_prog_func_and_line_info(const struct bpf_object *obj,
5936                               struct bpf_program *main_prog,
5937                               const struct bpf_program *prog)
5938 {
5939         int err;
5940
5941         /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
5942          * supprot func/line info
5943          */
5944         if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
5945                 return 0;
5946
5947         /* only attempt func info relocation if main program's func_info
5948          * relocation was successful
5949          */
5950         if (main_prog != prog && !main_prog->func_info)
5951                 goto line_info;
5952
5953         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
5954                                        &main_prog->func_info,
5955                                        &main_prog->func_info_cnt,
5956                                        &main_prog->func_info_rec_size);
5957         if (err) {
5958                 if (err != -ENOENT) {
5959                         pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
5960                                 prog->name, err);
5961                         return err;
5962                 }
5963                 if (main_prog->func_info) {
5964                         /*
5965                          * Some info has already been found but has problem
5966                          * in the last btf_ext reloc. Must have to error out.
5967                          */
5968                         pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
5969                         return err;
5970                 }
5971                 /* Have problem loading the very first info. Ignore the rest. */
5972                 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
5973                         prog->name);
5974         }
5975
5976 line_info:
5977         /* don't relocate line info if main program's relocation failed */
5978         if (main_prog != prog && !main_prog->line_info)
5979                 return 0;
5980
5981         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
5982                                        &main_prog->line_info,
5983                                        &main_prog->line_info_cnt,
5984                                        &main_prog->line_info_rec_size);
5985         if (err) {
5986                 if (err != -ENOENT) {
5987                         pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
5988                                 prog->name, err);
5989                         return err;
5990                 }
5991                 if (main_prog->line_info) {
5992                         /*
5993                          * Some info has already been found but has problem
5994                          * in the last btf_ext reloc. Must have to error out.
5995                          */
5996                         pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
5997                         return err;
5998                 }
5999                 /* Have problem loading the very first info. Ignore the rest. */
6000                 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6001                         prog->name);
6002         }
6003         return 0;
6004 }
6005
6006 static int cmp_relo_by_insn_idx(const void *key, const void *elem)
6007 {
6008         size_t insn_idx = *(const size_t *)key;
6009         const struct reloc_desc *relo = elem;
6010
6011         if (insn_idx == relo->insn_idx)
6012                 return 0;
6013         return insn_idx < relo->insn_idx ? -1 : 1;
6014 }
6015
6016 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6017 {
6018         if (!prog->nr_reloc)
6019                 return NULL;
6020         return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6021                        sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6022 }
6023
6024 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog)
6025 {
6026         int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6027         struct reloc_desc *relos;
6028         int i;
6029
6030         if (main_prog == subprog)
6031                 return 0;
6032         relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6033         if (!relos)
6034                 return -ENOMEM;
6035         if (subprog->nr_reloc)
6036                 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6037                        sizeof(*relos) * subprog->nr_reloc);
6038
6039         for (i = main_prog->nr_reloc; i < new_cnt; i++)
6040                 relos[i].insn_idx += subprog->sub_insn_off;
6041         /* After insn_idx adjustment the 'relos' array is still sorted
6042          * by insn_idx and doesn't break bsearch.
6043          */
6044         main_prog->reloc_desc = relos;
6045         main_prog->nr_reloc = new_cnt;
6046         return 0;
6047 }
6048
6049 static int
6050 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6051                        struct bpf_program *prog)
6052 {
6053         size_t sub_insn_idx, insn_idx, new_cnt;
6054         struct bpf_program *subprog;
6055         struct bpf_insn *insns, *insn;
6056         struct reloc_desc *relo;
6057         int err;
6058
6059         err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6060         if (err)
6061                 return err;
6062
6063         for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6064                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6065                 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))
6066                         continue;
6067
6068                 relo = find_prog_insn_relo(prog, insn_idx);
6069                 if (relo && relo->type == RELO_EXTERN_FUNC)
6070                         /* kfunc relocations will be handled later
6071                          * in bpf_object__relocate_data()
6072                          */
6073                         continue;
6074                 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6075                         pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6076                                 prog->name, insn_idx, relo->type);
6077                         return -LIBBPF_ERRNO__RELOC;
6078                 }
6079                 if (relo) {
6080                         /* sub-program instruction index is a combination of
6081                          * an offset of a symbol pointed to by relocation and
6082                          * call instruction's imm field; for global functions,
6083                          * call always has imm = -1, but for static functions
6084                          * relocation is against STT_SECTION and insn->imm
6085                          * points to a start of a static function
6086                          *
6087                          * for subprog addr relocation, the relo->sym_off + insn->imm is
6088                          * the byte offset in the corresponding section.
6089                          */
6090                         if (relo->type == RELO_CALL)
6091                                 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6092                         else
6093                                 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6094                 } else if (insn_is_pseudo_func(insn)) {
6095                         /*
6096                          * RELO_SUBPROG_ADDR relo is always emitted even if both
6097                          * functions are in the same section, so it shouldn't reach here.
6098                          */
6099                         pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
6100                                 prog->name, insn_idx);
6101                         return -LIBBPF_ERRNO__RELOC;
6102                 } else {
6103                         /* if subprogram call is to a static function within
6104                          * the same ELF section, there won't be any relocation
6105                          * emitted, but it also means there is no additional
6106                          * offset necessary, insns->imm is relative to
6107                          * instruction's original position within the section
6108                          */
6109                         sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6110                 }
6111
6112                 /* we enforce that sub-programs should be in .text section */
6113                 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6114                 if (!subprog) {
6115                         pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6116                                 prog->name);
6117                         return -LIBBPF_ERRNO__RELOC;
6118                 }
6119
6120                 /* if it's the first call instruction calling into this
6121                  * subprogram (meaning this subprog hasn't been processed
6122                  * yet) within the context of current main program:
6123                  *   - append it at the end of main program's instructions blog;
6124                  *   - process is recursively, while current program is put on hold;
6125                  *   - if that subprogram calls some other not yet processes
6126                  *   subprogram, same thing will happen recursively until
6127                  *   there are no more unprocesses subprograms left to append
6128                  *   and relocate.
6129                  */
6130                 if (subprog->sub_insn_off == 0) {
6131                         subprog->sub_insn_off = main_prog->insns_cnt;
6132
6133                         new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6134                         insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6135                         if (!insns) {
6136                                 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6137                                 return -ENOMEM;
6138                         }
6139                         main_prog->insns = insns;
6140                         main_prog->insns_cnt = new_cnt;
6141
6142                         memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6143                                subprog->insns_cnt * sizeof(*insns));
6144
6145                         pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6146                                  main_prog->name, subprog->insns_cnt, subprog->name);
6147
6148                         /* The subprog insns are now appended. Append its relos too. */
6149                         err = append_subprog_relos(main_prog, subprog);
6150                         if (err)
6151                                 return err;
6152                         err = bpf_object__reloc_code(obj, main_prog, subprog);
6153                         if (err)
6154                                 return err;
6155                 }
6156
6157                 /* main_prog->insns memory could have been re-allocated, so
6158                  * calculate pointer again
6159                  */
6160                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6161                 /* calculate correct instruction position within current main
6162                  * prog; each main prog can have a different set of
6163                  * subprograms appended (potentially in different order as
6164                  * well), so position of any subprog can be different for
6165                  * different main programs */
6166                 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6167
6168                 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6169                          prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6170         }
6171
6172         return 0;
6173 }
6174
6175 /*
6176  * Relocate sub-program calls.
6177  *
6178  * Algorithm operates as follows. Each entry-point BPF program (referred to as
6179  * main prog) is processed separately. For each subprog (non-entry functions,
6180  * that can be called from either entry progs or other subprogs) gets their
6181  * sub_insn_off reset to zero. This serves as indicator that this subprogram
6182  * hasn't been yet appended and relocated within current main prog. Once its
6183  * relocated, sub_insn_off will point at the position within current main prog
6184  * where given subprog was appended. This will further be used to relocate all
6185  * the call instructions jumping into this subprog.
6186  *
6187  * We start with main program and process all call instructions. If the call
6188  * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6189  * is zero), subprog instructions are appended at the end of main program's
6190  * instruction array. Then main program is "put on hold" while we recursively
6191  * process newly appended subprogram. If that subprogram calls into another
6192  * subprogram that hasn't been appended, new subprogram is appended again to
6193  * the *main* prog's instructions (subprog's instructions are always left
6194  * untouched, as they need to be in unmodified state for subsequent main progs
6195  * and subprog instructions are always sent only as part of a main prog) and
6196  * the process continues recursively. Once all the subprogs called from a main
6197  * prog or any of its subprogs are appended (and relocated), all their
6198  * positions within finalized instructions array are known, so it's easy to
6199  * rewrite call instructions with correct relative offsets, corresponding to
6200  * desired target subprog.
6201  *
6202  * Its important to realize that some subprogs might not be called from some
6203  * main prog and any of its called/used subprogs. Those will keep their
6204  * subprog->sub_insn_off as zero at all times and won't be appended to current
6205  * main prog and won't be relocated within the context of current main prog.
6206  * They might still be used from other main progs later.
6207  *
6208  * Visually this process can be shown as below. Suppose we have two main
6209  * programs mainA and mainB and BPF object contains three subprogs: subA,
6210  * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
6211  * subC both call subB:
6212  *
6213  *        +--------+ +-------+
6214  *        |        v v       |
6215  *     +--+---+ +--+-+-+ +---+--+
6216  *     | subA | | subB | | subC |
6217  *     +--+---+ +------+ +---+--+
6218  *        ^                  ^
6219  *        |                  |
6220  *    +---+-------+   +------+----+
6221  *    |   mainA   |   |   mainB   |
6222  *    +-----------+   +-----------+
6223  *
6224  * We'll start relocating mainA, will find subA, append it and start
6225  * processing sub A recursively:
6226  *
6227  *    +-----------+------+
6228  *    |   mainA   | subA |
6229  *    +-----------+------+
6230  *
6231  * At this point we notice that subB is used from subA, so we append it and
6232  * relocate (there are no further subcalls from subB):
6233  *
6234  *    +-----------+------+------+
6235  *    |   mainA   | subA | subB |
6236  *    +-----------+------+------+
6237  *
6238  * At this point, we relocate subA calls, then go one level up and finish with
6239  * relocatin mainA calls. mainA is done.
6240  *
6241  * For mainB process is similar but results in different order. We start with
6242  * mainB and skip subA and subB, as mainB never calls them (at least
6243  * directly), but we see subC is needed, so we append and start processing it:
6244  *
6245  *    +-----------+------+
6246  *    |   mainB   | subC |
6247  *    +-----------+------+
6248  * Now we see subC needs subB, so we go back to it, append and relocate it:
6249  *
6250  *    +-----------+------+------+
6251  *    |   mainB   | subC | subB |
6252  *    +-----------+------+------+
6253  *
6254  * At this point we unwind recursion, relocate calls in subC, then in mainB.
6255  */
6256 static int
6257 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6258 {
6259         struct bpf_program *subprog;
6260         int i, err;
6261
6262         /* mark all subprogs as not relocated (yet) within the context of
6263          * current main program
6264          */
6265         for (i = 0; i < obj->nr_programs; i++) {
6266                 subprog = &obj->programs[i];
6267                 if (!prog_is_subprog(obj, subprog))
6268                         continue;
6269
6270                 subprog->sub_insn_off = 0;
6271         }
6272
6273         err = bpf_object__reloc_code(obj, prog, prog);
6274         if (err)
6275                 return err;
6276
6277         return 0;
6278 }
6279
6280 static void
6281 bpf_object__free_relocs(struct bpf_object *obj)
6282 {
6283         struct bpf_program *prog;
6284         int i;
6285
6286         /* free up relocation descriptors */
6287         for (i = 0; i < obj->nr_programs; i++) {
6288                 prog = &obj->programs[i];
6289                 zfree(&prog->reloc_desc);
6290                 prog->nr_reloc = 0;
6291         }
6292 }
6293
6294 static int cmp_relocs(const void *_a, const void *_b)
6295 {
6296         const struct reloc_desc *a = _a;
6297         const struct reloc_desc *b = _b;
6298
6299         if (a->insn_idx != b->insn_idx)
6300                 return a->insn_idx < b->insn_idx ? -1 : 1;
6301
6302         /* no two relocations should have the same insn_idx, but ... */
6303         if (a->type != b->type)
6304                 return a->type < b->type ? -1 : 1;
6305
6306         return 0;
6307 }
6308
6309 static void bpf_object__sort_relos(struct bpf_object *obj)
6310 {
6311         int i;
6312
6313         for (i = 0; i < obj->nr_programs; i++) {
6314                 struct bpf_program *p = &obj->programs[i];
6315
6316                 if (!p->nr_reloc)
6317                         continue;
6318
6319                 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6320         }
6321 }
6322
6323 static int
6324 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
6325 {
6326         struct bpf_program *prog;
6327         size_t i, j;
6328         int err;
6329
6330         if (obj->btf_ext) {
6331                 err = bpf_object__relocate_core(obj, targ_btf_path);
6332                 if (err) {
6333                         pr_warn("failed to perform CO-RE relocations: %d\n",
6334                                 err);
6335                         return err;
6336                 }
6337                 bpf_object__sort_relos(obj);
6338         }
6339
6340         /* Before relocating calls pre-process relocations and mark
6341          * few ld_imm64 instructions that points to subprogs.
6342          * Otherwise bpf_object__reloc_code() later would have to consider
6343          * all ld_imm64 insns as relocation candidates. That would
6344          * reduce relocation speed, since amount of find_prog_insn_relo()
6345          * would increase and most of them will fail to find a relo.
6346          */
6347         for (i = 0; i < obj->nr_programs; i++) {
6348                 prog = &obj->programs[i];
6349                 for (j = 0; j < prog->nr_reloc; j++) {
6350                         struct reloc_desc *relo = &prog->reloc_desc[j];
6351                         struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6352
6353                         /* mark the insn, so it's recognized by insn_is_pseudo_func() */
6354                         if (relo->type == RELO_SUBPROG_ADDR)
6355                                 insn[0].src_reg = BPF_PSEUDO_FUNC;
6356                 }
6357         }
6358
6359         /* relocate subprogram calls and append used subprograms to main
6360          * programs; each copy of subprogram code needs to be relocated
6361          * differently for each main program, because its code location might
6362          * have changed.
6363          * Append subprog relos to main programs to allow data relos to be
6364          * processed after text is completely relocated.
6365          */
6366         for (i = 0; i < obj->nr_programs; i++) {
6367                 prog = &obj->programs[i];
6368                 /* sub-program's sub-calls are relocated within the context of
6369                  * its main program only
6370                  */
6371                 if (prog_is_subprog(obj, prog))
6372                         continue;
6373                 if (!prog->autoload)
6374                         continue;
6375
6376                 err = bpf_object__relocate_calls(obj, prog);
6377                 if (err) {
6378                         pr_warn("prog '%s': failed to relocate calls: %d\n",
6379                                 prog->name, err);
6380                         return err;
6381                 }
6382         }
6383         /* Process data relos for main programs */
6384         for (i = 0; i < obj->nr_programs; i++) {
6385                 prog = &obj->programs[i];
6386                 if (prog_is_subprog(obj, prog))
6387                         continue;
6388                 if (!prog->autoload)
6389                         continue;
6390                 err = bpf_object__relocate_data(obj, prog);
6391                 if (err) {
6392                         pr_warn("prog '%s': failed to relocate data references: %d\n",
6393                                 prog->name, err);
6394                         return err;
6395                 }
6396         }
6397
6398         return 0;
6399 }
6400
6401 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
6402                                             Elf64_Shdr *shdr, Elf_Data *data);
6403
6404 static int bpf_object__collect_map_relos(struct bpf_object *obj,
6405                                          Elf64_Shdr *shdr, Elf_Data *data)
6406 {
6407         const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
6408         int i, j, nrels, new_sz;
6409         const struct btf_var_secinfo *vi = NULL;
6410         const struct btf_type *sec, *var, *def;
6411         struct bpf_map *map = NULL, *targ_map = NULL;
6412         struct bpf_program *targ_prog = NULL;
6413         bool is_prog_array, is_map_in_map;
6414         const struct btf_member *member;
6415         const char *name, *mname, *type;
6416         unsigned int moff;
6417         Elf64_Sym *sym;
6418         Elf64_Rel *rel;
6419         void *tmp;
6420
6421         if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6422                 return -EINVAL;
6423         sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6424         if (!sec)
6425                 return -EINVAL;
6426
6427         nrels = shdr->sh_size / shdr->sh_entsize;
6428         for (i = 0; i < nrels; i++) {
6429                 rel = elf_rel_by_idx(data, i);
6430                 if (!rel) {
6431                         pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
6432                         return -LIBBPF_ERRNO__FORMAT;
6433                 }
6434
6435                 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
6436                 if (!sym) {
6437                         pr_warn(".maps relo #%d: symbol %zx not found\n",
6438                                 i, (size_t)ELF64_R_SYM(rel->r_info));
6439                         return -LIBBPF_ERRNO__FORMAT;
6440                 }
6441                 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
6442
6443                 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
6444                          i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
6445                          (size_t)rel->r_offset, sym->st_name, name);
6446
6447                 for (j = 0; j < obj->nr_maps; j++) {
6448                         map = &obj->maps[j];
6449                         if (map->sec_idx != obj->efile.btf_maps_shndx)
6450                                 continue;
6451
6452                         vi = btf_var_secinfos(sec) + map->btf_var_idx;
6453                         if (vi->offset <= rel->r_offset &&
6454                             rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
6455                                 break;
6456                 }
6457                 if (j == obj->nr_maps) {
6458                         pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
6459                                 i, name, (size_t)rel->r_offset);
6460                         return -EINVAL;
6461                 }
6462
6463                 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
6464                 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
6465                 type = is_map_in_map ? "map" : "prog";
6466                 if (is_map_in_map) {
6467                         if (sym->st_shndx != obj->efile.btf_maps_shndx) {
6468                                 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
6469                                         i, name);
6470                                 return -LIBBPF_ERRNO__RELOC;
6471                         }
6472                         if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
6473                             map->def.key_size != sizeof(int)) {
6474                                 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
6475                                         i, map->name, sizeof(int));
6476                                 return -EINVAL;
6477                         }
6478                         targ_map = bpf_object__find_map_by_name(obj, name);
6479                         if (!targ_map) {
6480                                 pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n",
6481                                         i, name);
6482                                 return -ESRCH;
6483                         }
6484                 } else if (is_prog_array) {
6485                         targ_prog = bpf_object__find_program_by_name(obj, name);
6486                         if (!targ_prog) {
6487                                 pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n",
6488                                         i, name);
6489                                 return -ESRCH;
6490                         }
6491                         if (targ_prog->sec_idx != sym->st_shndx ||
6492                             targ_prog->sec_insn_off * 8 != sym->st_value ||
6493                             prog_is_subprog(obj, targ_prog)) {
6494                                 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
6495                                         i, name);
6496                                 return -LIBBPF_ERRNO__RELOC;
6497                         }
6498                 } else {
6499                         return -EINVAL;
6500                 }
6501
6502                 var = btf__type_by_id(obj->btf, vi->type);
6503                 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6504                 if (btf_vlen(def) == 0)
6505                         return -EINVAL;
6506                 member = btf_members(def) + btf_vlen(def) - 1;
6507                 mname = btf__name_by_offset(obj->btf, member->name_off);
6508                 if (strcmp(mname, "values"))
6509                         return -EINVAL;
6510
6511                 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
6512                 if (rel->r_offset - vi->offset < moff)
6513                         return -EINVAL;
6514
6515                 moff = rel->r_offset - vi->offset - moff;
6516                 /* here we use BPF pointer size, which is always 64 bit, as we
6517                  * are parsing ELF that was built for BPF target
6518                  */
6519                 if (moff % bpf_ptr_sz)
6520                         return -EINVAL;
6521                 moff /= bpf_ptr_sz;
6522                 if (moff >= map->init_slots_sz) {
6523                         new_sz = moff + 1;
6524                         tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
6525                         if (!tmp)
6526                                 return -ENOMEM;
6527                         map->init_slots = tmp;
6528                         memset(map->init_slots + map->init_slots_sz, 0,
6529                                (new_sz - map->init_slots_sz) * host_ptr_sz);
6530                         map->init_slots_sz = new_sz;
6531                 }
6532                 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
6533
6534                 pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n",
6535                          i, map->name, moff, type, name);
6536         }
6537
6538         return 0;
6539 }
6540
6541 static int bpf_object__collect_relos(struct bpf_object *obj)
6542 {
6543         int i, err;
6544
6545         for (i = 0; i < obj->efile.sec_cnt; i++) {
6546                 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
6547                 Elf64_Shdr *shdr;
6548                 Elf_Data *data;
6549                 int idx;
6550
6551                 if (sec_desc->sec_type != SEC_RELO)
6552                         continue;
6553
6554                 shdr = sec_desc->shdr;
6555                 data = sec_desc->data;
6556                 idx = shdr->sh_info;
6557
6558                 if (shdr->sh_type != SHT_REL) {
6559                         pr_warn("internal error at %d\n", __LINE__);
6560                         return -LIBBPF_ERRNO__INTERNAL;
6561                 }
6562
6563                 if (idx == obj->efile.st_ops_shndx)
6564                         err = bpf_object__collect_st_ops_relos(obj, shdr, data);
6565                 else if (idx == obj->efile.btf_maps_shndx)
6566                         err = bpf_object__collect_map_relos(obj, shdr, data);
6567                 else
6568                         err = bpf_object__collect_prog_relos(obj, shdr, data);
6569                 if (err)
6570                         return err;
6571         }
6572
6573         bpf_object__sort_relos(obj);
6574         return 0;
6575 }
6576
6577 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
6578 {
6579         if (BPF_CLASS(insn->code) == BPF_JMP &&
6580             BPF_OP(insn->code) == BPF_CALL &&
6581             BPF_SRC(insn->code) == BPF_K &&
6582             insn->src_reg == 0 &&
6583             insn->dst_reg == 0) {
6584                     *func_id = insn->imm;
6585                     return true;
6586         }
6587         return false;
6588 }
6589
6590 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
6591 {
6592         struct bpf_insn *insn = prog->insns;
6593         enum bpf_func_id func_id;
6594         int i;
6595
6596         if (obj->gen_loader)
6597                 return 0;
6598
6599         for (i = 0; i < prog->insns_cnt; i++, insn++) {
6600                 if (!insn_is_helper_call(insn, &func_id))
6601                         continue;
6602
6603                 /* on kernels that don't yet support
6604                  * bpf_probe_read_{kernel,user}[_str] helpers, fall back
6605                  * to bpf_probe_read() which works well for old kernels
6606                  */
6607                 switch (func_id) {
6608                 case BPF_FUNC_probe_read_kernel:
6609                 case BPF_FUNC_probe_read_user:
6610                         if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6611                                 insn->imm = BPF_FUNC_probe_read;
6612                         break;
6613                 case BPF_FUNC_probe_read_kernel_str:
6614                 case BPF_FUNC_probe_read_user_str:
6615                         if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6616                                 insn->imm = BPF_FUNC_probe_read_str;
6617                         break;
6618                 default:
6619                         break;
6620                 }
6621         }
6622         return 0;
6623 }
6624
6625 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
6626                                      int *btf_obj_fd, int *btf_type_id);
6627
6628 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
6629 static int libbpf_prepare_prog_load(struct bpf_program *prog,
6630                                     struct bpf_prog_load_opts *opts, long cookie)
6631 {
6632         enum sec_def_flags def = cookie;
6633
6634         /* old kernels might not support specifying expected_attach_type */
6635         if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
6636                 opts->expected_attach_type = 0;
6637
6638         if (def & SEC_SLEEPABLE)
6639                 opts->prog_flags |= BPF_F_SLEEPABLE;
6640
6641         if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
6642                 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
6643
6644         if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
6645                 int btf_obj_fd = 0, btf_type_id = 0, err;
6646                 const char *attach_name;
6647
6648                 attach_name = strchr(prog->sec_name, '/');
6649                 if (!attach_name) {
6650                         /* if BPF program is annotated with just SEC("fentry")
6651                          * (or similar) without declaratively specifying
6652                          * target, then it is expected that target will be
6653                          * specified with bpf_program__set_attach_target() at
6654                          * runtime before BPF object load step. If not, then
6655                          * there is nothing to load into the kernel as BPF
6656                          * verifier won't be able to validate BPF program
6657                          * correctness anyways.
6658                          */
6659                         pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
6660                                 prog->name);
6661                         return -EINVAL;
6662                 }
6663                 attach_name++; /* skip over / */
6664
6665                 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);
6666                 if (err)
6667                         return err;
6668
6669                 /* cache resolved BTF FD and BTF type ID in the prog */
6670                 prog->attach_btf_obj_fd = btf_obj_fd;
6671                 prog->attach_btf_id = btf_type_id;
6672
6673                 /* but by now libbpf common logic is not utilizing
6674                  * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
6675                  * this callback is called after opts were populated by
6676                  * libbpf, so this callback has to update opts explicitly here
6677                  */
6678                 opts->attach_btf_obj_fd = btf_obj_fd;
6679                 opts->attach_btf_id = btf_type_id;
6680         }
6681         return 0;
6682 }
6683
6684 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
6685
6686 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
6687                                 struct bpf_insn *insns, int insns_cnt,
6688                                 const char *license, __u32 kern_version, int *prog_fd)
6689 {
6690         LIBBPF_OPTS(bpf_prog_load_opts, load_attr);
6691         const char *prog_name = NULL;
6692         char *cp, errmsg[STRERR_BUFSIZE];
6693         size_t log_buf_size = 0;
6694         char *log_buf = NULL, *tmp;
6695         int btf_fd, ret, err;
6696         bool own_log_buf = true;
6697         __u32 log_level = prog->log_level;
6698
6699         if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6700                 /*
6701                  * The program type must be set.  Most likely we couldn't find a proper
6702                  * section definition at load time, and thus we didn't infer the type.
6703                  */
6704                 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
6705                         prog->name, prog->sec_name);
6706                 return -EINVAL;
6707         }
6708
6709         if (!insns || !insns_cnt)
6710                 return -EINVAL;
6711
6712         load_attr.expected_attach_type = prog->expected_attach_type;
6713         if (kernel_supports(obj, FEAT_PROG_NAME))
6714                 prog_name = prog->name;
6715         load_attr.attach_prog_fd = prog->attach_prog_fd;
6716         load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
6717         load_attr.attach_btf_id = prog->attach_btf_id;
6718         load_attr.kern_version = kern_version;
6719         load_attr.prog_ifindex = prog->prog_ifindex;
6720
6721         /* specify func_info/line_info only if kernel supports them */
6722         btf_fd = bpf_object__btf_fd(obj);
6723         if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
6724                 load_attr.prog_btf_fd = btf_fd;
6725                 load_attr.func_info = prog->func_info;
6726                 load_attr.func_info_rec_size = prog->func_info_rec_size;
6727                 load_attr.func_info_cnt = prog->func_info_cnt;
6728                 load_attr.line_info = prog->line_info;
6729                 load_attr.line_info_rec_size = prog->line_info_rec_size;
6730                 load_attr.line_info_cnt = prog->line_info_cnt;
6731         }
6732         load_attr.log_level = log_level;
6733         load_attr.prog_flags = prog->prog_flags;
6734         load_attr.fd_array = obj->fd_array;
6735
6736         /* adjust load_attr if sec_def provides custom preload callback */
6737         if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
6738                 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
6739                 if (err < 0) {
6740                         pr_warn("prog '%s': failed to prepare load attributes: %d\n",
6741                                 prog->name, err);
6742                         return err;
6743                 }
6744                 insns = prog->insns;
6745                 insns_cnt = prog->insns_cnt;
6746         }
6747
6748         if (obj->gen_loader) {
6749                 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
6750                                    license, insns, insns_cnt, &load_attr,
6751                                    prog - obj->programs);
6752                 *prog_fd = -1;
6753                 return 0;
6754         }
6755
6756 retry_load:
6757         /* if log_level is zero, we don't request logs initially even if
6758          * custom log_buf is specified; if the program load fails, then we'll
6759          * bump log_level to 1 and use either custom log_buf or we'll allocate
6760          * our own and retry the load to get details on what failed
6761          */
6762         if (log_level) {
6763                 if (prog->log_buf) {
6764                         log_buf = prog->log_buf;
6765                         log_buf_size = prog->log_size;
6766                         own_log_buf = false;
6767                 } else if (obj->log_buf) {
6768                         log_buf = obj->log_buf;
6769                         log_buf_size = obj->log_size;
6770                         own_log_buf = false;
6771                 } else {
6772                         log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2);
6773                         tmp = realloc(log_buf, log_buf_size);
6774                         if (!tmp) {
6775                                 ret = -ENOMEM;
6776                                 goto out;
6777                         }
6778                         log_buf = tmp;
6779                         log_buf[0] = '\0';
6780                         own_log_buf = true;
6781                 }
6782         }
6783
6784         load_attr.log_buf = log_buf;
6785         load_attr.log_size = log_buf_size;
6786         load_attr.log_level = log_level;
6787
6788         ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
6789         if (ret >= 0) {
6790                 if (log_level && own_log_buf) {
6791                         pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
6792                                  prog->name, log_buf);
6793                 }
6794
6795                 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
6796                         struct bpf_map *map;
6797                         int i;
6798
6799                         for (i = 0; i < obj->nr_maps; i++) {
6800                                 map = &prog->obj->maps[i];
6801                                 if (map->libbpf_type != LIBBPF_MAP_RODATA)
6802                                         continue;
6803
6804                                 if (bpf_prog_bind_map(ret, bpf_map__fd(map), NULL)) {
6805                                         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6806                                         pr_warn("prog '%s': failed to bind map '%s': %s\n",
6807                                                 prog->name, map->real_name, cp);
6808                                         /* Don't fail hard if can't bind rodata. */
6809                                 }
6810                         }
6811                 }
6812
6813                 *prog_fd = ret;
6814                 ret = 0;
6815                 goto out;
6816         }
6817
6818         if (log_level == 0) {
6819                 log_level = 1;
6820                 goto retry_load;
6821         }
6822         /* On ENOSPC, increase log buffer size and retry, unless custom
6823          * log_buf is specified.
6824          * Be careful to not overflow u32, though. Kernel's log buf size limit
6825          * isn't part of UAPI so it can always be bumped to full 4GB. So don't
6826          * multiply by 2 unless we are sure we'll fit within 32 bits.
6827          * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
6828          */
6829         if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2)
6830                 goto retry_load;
6831
6832         ret = -errno;
6833
6834         /* post-process verifier log to improve error descriptions */
6835         fixup_verifier_log(prog, log_buf, log_buf_size);
6836
6837         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6838         pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
6839         pr_perm_msg(ret);
6840
6841         if (own_log_buf && log_buf && log_buf[0] != '\0') {
6842                 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
6843                         prog->name, log_buf);
6844         }
6845
6846 out:
6847         if (own_log_buf)
6848                 free(log_buf);
6849         return ret;
6850 }
6851
6852 static char *find_prev_line(char *buf, char *cur)
6853 {
6854         char *p;
6855
6856         if (cur == buf) /* end of a log buf */
6857                 return NULL;
6858
6859         p = cur - 1;
6860         while (p - 1 >= buf && *(p - 1) != '\n')
6861                 p--;
6862
6863         return p;
6864 }
6865
6866 static void patch_log(char *buf, size_t buf_sz, size_t log_sz,
6867                       char *orig, size_t orig_sz, const char *patch)
6868 {
6869         /* size of the remaining log content to the right from the to-be-replaced part */
6870         size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
6871         size_t patch_sz = strlen(patch);
6872
6873         if (patch_sz != orig_sz) {
6874                 /* If patch line(s) are longer than original piece of verifier log,
6875                  * shift log contents by (patch_sz - orig_sz) bytes to the right
6876                  * starting from after to-be-replaced part of the log.
6877                  *
6878                  * If patch line(s) are shorter than original piece of verifier log,
6879                  * shift log contents by (orig_sz - patch_sz) bytes to the left
6880                  * starting from after to-be-replaced part of the log
6881                  *
6882                  * We need to be careful about not overflowing available
6883                  * buf_sz capacity. If that's the case, we'll truncate the end
6884                  * of the original log, as necessary.
6885                  */
6886                 if (patch_sz > orig_sz) {
6887                         if (orig + patch_sz >= buf + buf_sz) {
6888                                 /* patch is big enough to cover remaining space completely */
6889                                 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
6890                                 rem_sz = 0;
6891                         } else if (patch_sz - orig_sz > buf_sz - log_sz) {
6892                                 /* patch causes part of remaining log to be truncated */
6893                                 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
6894                         }
6895                 }
6896                 /* shift remaining log to the right by calculated amount */
6897                 memmove(orig + patch_sz, orig + orig_sz, rem_sz);
6898         }
6899
6900         memcpy(orig, patch, patch_sz);
6901 }
6902
6903 static void fixup_log_failed_core_relo(struct bpf_program *prog,
6904                                        char *buf, size_t buf_sz, size_t log_sz,
6905                                        char *line1, char *line2, char *line3)
6906 {
6907         /* Expected log for failed and not properly guarded CO-RE relocation:
6908          * line1 -> 123: (85) call unknown#195896080
6909          * line2 -> invalid func unknown#195896080
6910          * line3 -> <anything else or end of buffer>
6911          *
6912          * "123" is the index of the instruction that was poisoned. We extract
6913          * instruction index to find corresponding CO-RE relocation and
6914          * replace this part of the log with more relevant information about
6915          * failed CO-RE relocation.
6916          */
6917         const struct bpf_core_relo *relo;
6918         struct bpf_core_spec spec;
6919         char patch[512], spec_buf[256];
6920         int insn_idx, err, spec_len;
6921
6922         if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1)
6923                 return;
6924
6925         relo = find_relo_core(prog, insn_idx);
6926         if (!relo)
6927                 return;
6928
6929         err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
6930         if (err)
6931                 return;
6932
6933         spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec);
6934         snprintf(patch, sizeof(patch),
6935                  "%d: <invalid CO-RE relocation>\n"
6936                  "failed to resolve CO-RE relocation %s%s\n",
6937                  insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : "");
6938
6939         patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
6940 }
6941
6942 static void fixup_log_missing_map_load(struct bpf_program *prog,
6943                                        char *buf, size_t buf_sz, size_t log_sz,
6944                                        char *line1, char *line2, char *line3)
6945 {
6946         /* Expected log for failed and not properly guarded CO-RE relocation:
6947          * line1 -> 123: (85) call unknown#2001000345
6948          * line2 -> invalid func unknown#2001000345
6949          * line3 -> <anything else or end of buffer>
6950          *
6951          * "123" is the index of the instruction that was poisoned.
6952          * "345" in "2001000345" are map index in obj->maps to fetch map name.
6953          */
6954         struct bpf_object *obj = prog->obj;
6955         const struct bpf_map *map;
6956         int insn_idx, map_idx;
6957         char patch[128];
6958
6959         if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2)
6960                 return;
6961
6962         map_idx -= MAP_LDIMM64_POISON_BASE;
6963         if (map_idx < 0 || map_idx >= obj->nr_maps)
6964                 return;
6965         map = &obj->maps[map_idx];
6966
6967         snprintf(patch, sizeof(patch),
6968                  "%d: <invalid BPF map reference>\n"
6969                  "BPF map '%s' is referenced but wasn't created\n",
6970                  insn_idx, map->name);
6971
6972         patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
6973 }
6974
6975 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz)
6976 {
6977         /* look for familiar error patterns in last N lines of the log */
6978         const size_t max_last_line_cnt = 10;
6979         char *prev_line, *cur_line, *next_line;
6980         size_t log_sz;
6981         int i;
6982
6983         if (!buf)
6984                 return;
6985
6986         log_sz = strlen(buf) + 1;
6987         next_line = buf + log_sz - 1;
6988
6989         for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) {
6990                 cur_line = find_prev_line(buf, next_line);
6991                 if (!cur_line)
6992                         return;
6993
6994                 /* failed CO-RE relocation case */
6995                 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) {
6996                         prev_line = find_prev_line(buf, cur_line);
6997                         if (!prev_line)
6998                                 continue;
6999
7000                         fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,
7001                                                    prev_line, cur_line, next_line);
7002                         return;
7003                 } else if (str_has_pfx(cur_line, "invalid func unknown#"MAP_LDIMM64_POISON_PFX)) {
7004                         prev_line = find_prev_line(buf, cur_line);
7005                         if (!prev_line)
7006                                 continue;
7007
7008                         fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,
7009                                                    prev_line, cur_line, next_line);
7010                         return;
7011                 }
7012         }
7013 }
7014
7015 static int bpf_program_record_relos(struct bpf_program *prog)
7016 {
7017         struct bpf_object *obj = prog->obj;
7018         int i;
7019
7020         for (i = 0; i < prog->nr_reloc; i++) {
7021                 struct reloc_desc *relo = &prog->reloc_desc[i];
7022                 struct extern_desc *ext = &obj->externs[relo->sym_off];
7023
7024                 switch (relo->type) {
7025                 case RELO_EXTERN_VAR:
7026                         if (ext->type != EXT_KSYM)
7027                                 continue;
7028                         bpf_gen__record_extern(obj->gen_loader, ext->name,
7029                                                ext->is_weak, !ext->ksym.type_id,
7030                                                BTF_KIND_VAR, relo->insn_idx);
7031                         break;
7032                 case RELO_EXTERN_FUNC:
7033                         bpf_gen__record_extern(obj->gen_loader, ext->name,
7034                                                ext->is_weak, false, BTF_KIND_FUNC,
7035                                                relo->insn_idx);
7036                         break;
7037                 case RELO_CORE: {
7038                         struct bpf_core_relo cr = {
7039                                 .insn_off = relo->insn_idx * 8,
7040                                 .type_id = relo->core_relo->type_id,
7041                                 .access_str_off = relo->core_relo->access_str_off,
7042                                 .kind = relo->core_relo->kind,
7043                         };
7044
7045                         bpf_gen__record_relo_core(obj->gen_loader, &cr);
7046                         break;
7047                 }
7048                 default:
7049                         continue;
7050                 }
7051         }
7052         return 0;
7053 }
7054
7055 static int
7056 bpf_object__load_progs(struct bpf_object *obj, int log_level)
7057 {
7058         struct bpf_program *prog;
7059         size_t i;
7060         int err;
7061
7062         for (i = 0; i < obj->nr_programs; i++) {
7063                 prog = &obj->programs[i];
7064                 err = bpf_object__sanitize_prog(obj, prog);
7065                 if (err)
7066                         return err;
7067         }
7068
7069         for (i = 0; i < obj->nr_programs; i++) {
7070                 prog = &obj->programs[i];
7071                 if (prog_is_subprog(obj, prog))
7072                         continue;
7073                 if (!prog->autoload) {
7074                         pr_debug("prog '%s': skipped loading\n", prog->name);
7075                         continue;
7076                 }
7077                 prog->log_level |= log_level;
7078
7079                 if (obj->gen_loader)
7080                         bpf_program_record_relos(prog);
7081
7082                 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7083                                            obj->license, obj->kern_version, &prog->fd);
7084                 if (err) {
7085                         pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7086                         return err;
7087                 }
7088         }
7089
7090         bpf_object__free_relocs(obj);
7091         return 0;
7092 }
7093
7094 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
7095
7096 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)
7097 {
7098         struct bpf_program *prog;
7099         int err;
7100
7101         bpf_object__for_each_program(prog, obj) {
7102                 prog->sec_def = find_sec_def(prog->sec_name);
7103                 if (!prog->sec_def) {
7104                         /* couldn't guess, but user might manually specify */
7105                         pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7106                                 prog->name, prog->sec_name);
7107                         continue;
7108                 }
7109
7110                 prog->type = prog->sec_def->prog_type;
7111                 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7112
7113                 /* sec_def can have custom callback which should be called
7114                  * after bpf_program is initialized to adjust its properties
7115                  */
7116                 if (prog->sec_def->prog_setup_fn) {
7117                         err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7118                         if (err < 0) {
7119                                 pr_warn("prog '%s': failed to initialize: %d\n",
7120                                         prog->name, err);
7121                                 return err;
7122                         }
7123                 }
7124         }
7125
7126         return 0;
7127 }
7128
7129 static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz,
7130                                           const struct bpf_object_open_opts *opts)
7131 {
7132         const char *obj_name, *kconfig, *btf_tmp_path;
7133         struct bpf_object *obj;
7134         char tmp_name[64];
7135         int err;
7136         char *log_buf;
7137         size_t log_size;
7138         __u32 log_level;
7139
7140         if (elf_version(EV_CURRENT) == EV_NONE) {
7141                 pr_warn("failed to init libelf for %s\n",
7142                         path ? : "(mem buf)");
7143                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7144         }
7145
7146         if (!OPTS_VALID(opts, bpf_object_open_opts))
7147                 return ERR_PTR(-EINVAL);
7148
7149         obj_name = OPTS_GET(opts, object_name, NULL);
7150         if (obj_buf) {
7151                 if (!obj_name) {
7152                         snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7153                                  (unsigned long)obj_buf,
7154                                  (unsigned long)obj_buf_sz);
7155                         obj_name = tmp_name;
7156                 }
7157                 path = obj_name;
7158                 pr_debug("loading object '%s' from buffer\n", obj_name);
7159         }
7160
7161         log_buf = OPTS_GET(opts, kernel_log_buf, NULL);
7162         log_size = OPTS_GET(opts, kernel_log_size, 0);
7163         log_level = OPTS_GET(opts, kernel_log_level, 0);
7164         if (log_size > UINT_MAX)
7165                 return ERR_PTR(-EINVAL);
7166         if (log_size && !log_buf)
7167                 return ERR_PTR(-EINVAL);
7168
7169         obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7170         if (IS_ERR(obj))
7171                 return obj;
7172
7173         obj->log_buf = log_buf;
7174         obj->log_size = log_size;
7175         obj->log_level = log_level;
7176
7177         btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL);
7178         if (btf_tmp_path) {
7179                 if (strlen(btf_tmp_path) >= PATH_MAX) {
7180                         err = -ENAMETOOLONG;
7181                         goto out;
7182                 }
7183                 obj->btf_custom_path = strdup(btf_tmp_path);
7184                 if (!obj->btf_custom_path) {
7185                         err = -ENOMEM;
7186                         goto out;
7187                 }
7188         }
7189
7190         kconfig = OPTS_GET(opts, kconfig, NULL);
7191         if (kconfig) {
7192                 obj->kconfig = strdup(kconfig);
7193                 if (!obj->kconfig) {
7194                         err = -ENOMEM;
7195                         goto out;
7196                 }
7197         }
7198
7199         err = bpf_object__elf_init(obj);
7200         err = err ? : bpf_object__check_endianness(obj);
7201         err = err ? : bpf_object__elf_collect(obj);
7202         err = err ? : bpf_object__collect_externs(obj);
7203         err = err ? : bpf_object__finalize_btf(obj);
7204         err = err ? : bpf_object__init_maps(obj, opts);
7205         err = err ? : bpf_object_init_progs(obj, opts);
7206         err = err ? : bpf_object__collect_relos(obj);
7207         if (err)
7208                 goto out;
7209
7210         bpf_object__elf_finish(obj);
7211
7212         return obj;
7213 out:
7214         bpf_object__close(obj);
7215         return ERR_PTR(err);
7216 }
7217
7218 struct bpf_object *
7219 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
7220 {
7221         if (!path)
7222                 return libbpf_err_ptr(-EINVAL);
7223
7224         pr_debug("loading %s\n", path);
7225
7226         return libbpf_ptr(bpf_object_open(path, NULL, 0, opts));
7227 }
7228
7229 struct bpf_object *bpf_object__open(const char *path)
7230 {
7231         return bpf_object__open_file(path, NULL);
7232 }
7233
7234 struct bpf_object *
7235 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
7236                      const struct bpf_object_open_opts *opts)
7237 {
7238         if (!obj_buf || obj_buf_sz == 0)
7239                 return libbpf_err_ptr(-EINVAL);
7240
7241         return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts));
7242 }
7243
7244 static int bpf_object_unload(struct bpf_object *obj)
7245 {
7246         size_t i;
7247
7248         if (!obj)
7249                 return libbpf_err(-EINVAL);
7250
7251         for (i = 0; i < obj->nr_maps; i++) {
7252                 zclose(obj->maps[i].fd);
7253                 if (obj->maps[i].st_ops)
7254                         zfree(&obj->maps[i].st_ops->kern_vdata);
7255         }
7256
7257         for (i = 0; i < obj->nr_programs; i++)
7258                 bpf_program__unload(&obj->programs[i]);
7259
7260         return 0;
7261 }
7262
7263 int bpf_object__unload(struct bpf_object *obj) __attribute__((alias("bpf_object_unload")));
7264
7265 static int bpf_object__sanitize_maps(struct bpf_object *obj)
7266 {
7267         struct bpf_map *m;
7268
7269         bpf_object__for_each_map(m, obj) {
7270                 if (!bpf_map__is_internal(m))
7271                         continue;
7272                 if (!kernel_supports(obj, FEAT_ARRAY_MMAP))
7273                         m->def.map_flags ^= BPF_F_MMAPABLE;
7274         }
7275
7276         return 0;
7277 }
7278
7279 int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx)
7280 {
7281         char sym_type, sym_name[500];
7282         unsigned long long sym_addr;
7283         int ret, err = 0;
7284         FILE *f;
7285
7286         f = fopen("/proc/kallsyms", "r");
7287         if (!f) {
7288                 err = -errno;
7289                 pr_warn("failed to open /proc/kallsyms: %d\n", err);
7290                 return err;
7291         }
7292
7293         while (true) {
7294                 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
7295                              &sym_addr, &sym_type, sym_name);
7296                 if (ret == EOF && feof(f))
7297                         break;
7298                 if (ret != 3) {
7299                         pr_warn("failed to read kallsyms entry: %d\n", ret);
7300                         err = -EINVAL;
7301                         break;
7302                 }
7303
7304                 err = cb(sym_addr, sym_type, sym_name, ctx);
7305                 if (err)
7306                         break;
7307         }
7308
7309         fclose(f);
7310         return err;
7311 }
7312
7313 static int kallsyms_cb(unsigned long long sym_addr, char sym_type,
7314                        const char *sym_name, void *ctx)
7315 {
7316         struct bpf_object *obj = ctx;
7317         const struct btf_type *t;
7318         struct extern_desc *ext;
7319
7320         ext = find_extern_by_name(obj, sym_name);
7321         if (!ext || ext->type != EXT_KSYM)
7322                 return 0;
7323
7324         t = btf__type_by_id(obj->btf, ext->btf_id);
7325         if (!btf_is_var(t))
7326                 return 0;
7327
7328         if (ext->is_set && ext->ksym.addr != sym_addr) {
7329                 pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n",
7330                         sym_name, ext->ksym.addr, sym_addr);
7331                 return -EINVAL;
7332         }
7333         if (!ext->is_set) {
7334                 ext->is_set = true;
7335                 ext->ksym.addr = sym_addr;
7336                 pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr);
7337         }
7338         return 0;
7339 }
7340
7341 static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
7342 {
7343         return libbpf_kallsyms_parse(kallsyms_cb, obj);
7344 }
7345
7346 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
7347                             __u16 kind, struct btf **res_btf,
7348                             struct module_btf **res_mod_btf)
7349 {
7350         struct module_btf *mod_btf;
7351         struct btf *btf;
7352         int i, id, err;
7353
7354         btf = obj->btf_vmlinux;
7355         mod_btf = NULL;
7356         id = btf__find_by_name_kind(btf, ksym_name, kind);
7357
7358         if (id == -ENOENT) {
7359                 err = load_module_btfs(obj);
7360                 if (err)
7361                         return err;
7362
7363                 for (i = 0; i < obj->btf_module_cnt; i++) {
7364                         /* we assume module_btf's BTF FD is always >0 */
7365                         mod_btf = &obj->btf_modules[i];
7366                         btf = mod_btf->btf;
7367                         id = btf__find_by_name_kind_own(btf, ksym_name, kind);
7368                         if (id != -ENOENT)
7369                                 break;
7370                 }
7371         }
7372         if (id <= 0)
7373                 return -ESRCH;
7374
7375         *res_btf = btf;
7376         *res_mod_btf = mod_btf;
7377         return id;
7378 }
7379
7380 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
7381                                                struct extern_desc *ext)
7382 {
7383         const struct btf_type *targ_var, *targ_type;
7384         __u32 targ_type_id, local_type_id;
7385         struct module_btf *mod_btf = NULL;
7386         const char *targ_var_name;
7387         struct btf *btf = NULL;
7388         int id, err;
7389
7390         id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
7391         if (id < 0) {
7392                 if (id == -ESRCH && ext->is_weak)
7393                         return 0;
7394                 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n",
7395                         ext->name);
7396                 return id;
7397         }
7398
7399         /* find local type_id */
7400         local_type_id = ext->ksym.type_id;
7401
7402         /* find target type_id */
7403         targ_var = btf__type_by_id(btf, id);
7404         targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
7405         targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
7406
7407         err = bpf_core_types_are_compat(obj->btf, local_type_id,
7408                                         btf, targ_type_id);
7409         if (err <= 0) {
7410                 const struct btf_type *local_type;
7411                 const char *targ_name, *local_name;
7412
7413                 local_type = btf__type_by_id(obj->btf, local_type_id);
7414                 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
7415                 targ_name = btf__name_by_offset(btf, targ_type->name_off);
7416
7417                 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7418                         ext->name, local_type_id,
7419                         btf_kind_str(local_type), local_name, targ_type_id,
7420                         btf_kind_str(targ_type), targ_name);
7421                 return -EINVAL;
7422         }
7423
7424         ext->is_set = true;
7425         ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
7426         ext->ksym.kernel_btf_id = id;
7427         pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
7428                  ext->name, id, btf_kind_str(targ_var), targ_var_name);
7429
7430         return 0;
7431 }
7432
7433 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
7434                                                 struct extern_desc *ext)
7435 {
7436         int local_func_proto_id, kfunc_proto_id, kfunc_id;
7437         struct module_btf *mod_btf = NULL;
7438         const struct btf_type *kern_func;
7439         struct btf *kern_btf = NULL;
7440         int ret;
7441
7442         local_func_proto_id = ext->ksym.type_id;
7443
7444         kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, &kern_btf, &mod_btf);
7445         if (kfunc_id < 0) {
7446                 if (kfunc_id == -ESRCH && ext->is_weak)
7447                         return 0;
7448                 pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n",
7449                         ext->name);
7450                 return kfunc_id;
7451         }
7452
7453         kern_func = btf__type_by_id(kern_btf, kfunc_id);
7454         kfunc_proto_id = kern_func->type;
7455
7456         ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
7457                                         kern_btf, kfunc_proto_id);
7458         if (ret <= 0) {
7459                 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n",
7460                         ext->name, local_func_proto_id, kfunc_proto_id);
7461                 return -EINVAL;
7462         }
7463
7464         /* set index for module BTF fd in fd_array, if unset */
7465         if (mod_btf && !mod_btf->fd_array_idx) {
7466                 /* insn->off is s16 */
7467                 if (obj->fd_array_cnt == INT16_MAX) {
7468                         pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n",
7469                                 ext->name, mod_btf->fd_array_idx);
7470                         return -E2BIG;
7471                 }
7472                 /* Cannot use index 0 for module BTF fd */
7473                 if (!obj->fd_array_cnt)
7474                         obj->fd_array_cnt = 1;
7475
7476                 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
7477                                         obj->fd_array_cnt + 1);
7478                 if (ret)
7479                         return ret;
7480                 mod_btf->fd_array_idx = obj->fd_array_cnt;
7481                 /* we assume module BTF FD is always >0 */
7482                 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
7483         }
7484
7485         ext->is_set = true;
7486         ext->ksym.kernel_btf_id = kfunc_id;
7487         ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
7488         pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n",
7489                  ext->name, kfunc_id);
7490
7491         return 0;
7492 }
7493
7494 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7495 {
7496         const struct btf_type *t;
7497         struct extern_desc *ext;
7498         int i, err;
7499
7500         for (i = 0; i < obj->nr_extern; i++) {
7501                 ext = &obj->externs[i];
7502                 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7503                         continue;
7504
7505                 if (obj->gen_loader) {
7506                         ext->is_set = true;
7507                         ext->ksym.kernel_btf_obj_fd = 0;
7508                         ext->ksym.kernel_btf_id = 0;
7509                         continue;
7510                 }
7511                 t = btf__type_by_id(obj->btf, ext->btf_id);
7512                 if (btf_is_var(t))
7513                         err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
7514                 else
7515                         err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
7516                 if (err)
7517                         return err;
7518         }
7519         return 0;
7520 }
7521
7522 static int bpf_object__resolve_externs(struct bpf_object *obj,
7523                                        const char *extra_kconfig)
7524 {
7525         bool need_config = false, need_kallsyms = false;
7526         bool need_vmlinux_btf = false;
7527         struct extern_desc *ext;
7528         void *kcfg_data = NULL;
7529         int err, i;
7530
7531         if (obj->nr_extern == 0)
7532                 return 0;
7533
7534         if (obj->kconfig_map_idx >= 0)
7535                 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7536
7537         for (i = 0; i < obj->nr_extern; i++) {
7538                 ext = &obj->externs[i];
7539
7540                 if (ext->type == EXT_KSYM) {
7541                         if (ext->ksym.type_id)
7542                                 need_vmlinux_btf = true;
7543                         else
7544                                 need_kallsyms = true;
7545                         continue;
7546                 } else if (ext->type == EXT_KCFG) {
7547                         void *ext_ptr = kcfg_data + ext->kcfg.data_off;
7548                         __u64 value = 0;
7549
7550                         /* Kconfig externs need actual /proc/config.gz */
7551                         if (str_has_pfx(ext->name, "CONFIG_")) {
7552                                 need_config = true;
7553                                 continue;
7554                         }
7555
7556                         /* Virtual kcfg externs are customly handled by libbpf */
7557                         if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
7558                                 value = get_kernel_version();
7559                                 if (!value) {
7560                                         pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
7561                                         return -EINVAL;
7562                                 }
7563                         } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
7564                                 value = kernel_supports(obj, FEAT_BPF_COOKIE);
7565                         } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
7566                                 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER);
7567                         } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
7568                                 /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed
7569                                  * __kconfig externs, where LINUX_ ones are virtual and filled out
7570                                  * customly by libbpf (their values don't come from Kconfig).
7571                                  * If LINUX_xxx variable is not recognized by libbpf, but is marked
7572                                  * __weak, it defaults to zero value, just like for CONFIG_xxx
7573                                  * externs.
7574                                  */
7575                                 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
7576                                 return -EINVAL;
7577                         }
7578
7579                         err = set_kcfg_value_num(ext, ext_ptr, value);
7580                         if (err)
7581                                 return err;
7582                         pr_debug("extern (kcfg) '%s': set to 0x%llx\n",
7583                                  ext->name, (long long)value);
7584                 } else {
7585                         pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
7586                         return -EINVAL;
7587                 }
7588         }
7589         if (need_config && extra_kconfig) {
7590                 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
7591                 if (err)
7592                         return -EINVAL;
7593                 need_config = false;
7594                 for (i = 0; i < obj->nr_extern; i++) {
7595                         ext = &obj->externs[i];
7596                         if (ext->type == EXT_KCFG && !ext->is_set) {
7597                                 need_config = true;
7598                                 break;
7599                         }
7600                 }
7601         }
7602         if (need_config) {
7603                 err = bpf_object__read_kconfig_file(obj, kcfg_data);
7604                 if (err)
7605                         return -EINVAL;
7606         }
7607         if (need_kallsyms) {
7608                 err = bpf_object__read_kallsyms_file(obj);
7609                 if (err)
7610                         return -EINVAL;
7611         }
7612         if (need_vmlinux_btf) {
7613                 err = bpf_object__resolve_ksyms_btf_id(obj);
7614                 if (err)
7615                         return -EINVAL;
7616         }
7617         for (i = 0; i < obj->nr_extern; i++) {
7618                 ext = &obj->externs[i];
7619
7620                 if (!ext->is_set && !ext->is_weak) {
7621                         pr_warn("extern '%s' (strong): not resolved\n", ext->name);
7622                         return -ESRCH;
7623                 } else if (!ext->is_set) {
7624                         pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n",
7625                                  ext->name);
7626                 }
7627         }
7628
7629         return 0;
7630 }
7631
7632 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path)
7633 {
7634         int err, i;
7635
7636         if (!obj)
7637                 return libbpf_err(-EINVAL);
7638
7639         if (obj->loaded) {
7640                 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
7641                 return libbpf_err(-EINVAL);
7642         }
7643
7644         if (obj->gen_loader)
7645                 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
7646
7647         err = bpf_object__probe_loading(obj);
7648         err = err ? : bpf_object__load_vmlinux_btf(obj, false);
7649         err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7650         err = err ? : bpf_object__sanitize_and_load_btf(obj);
7651         err = err ? : bpf_object__sanitize_maps(obj);
7652         err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7653         err = err ? : bpf_object__create_maps(obj);
7654         err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
7655         err = err ? : bpf_object__load_progs(obj, extra_log_level);
7656         err = err ? : bpf_object_init_prog_arrays(obj);
7657
7658         if (obj->gen_loader) {
7659                 /* reset FDs */
7660                 if (obj->btf)
7661                         btf__set_fd(obj->btf, -1);
7662                 for (i = 0; i < obj->nr_maps; i++)
7663                         obj->maps[i].fd = -1;
7664                 if (!err)
7665                         err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
7666         }
7667
7668         /* clean up fd_array */
7669         zfree(&obj->fd_array);
7670
7671         /* clean up module BTFs */
7672         for (i = 0; i < obj->btf_module_cnt; i++) {
7673                 close(obj->btf_modules[i].fd);
7674                 btf__free(obj->btf_modules[i].btf);
7675                 free(obj->btf_modules[i].name);
7676         }
7677         free(obj->btf_modules);
7678
7679         /* clean up vmlinux BTF */
7680         btf__free(obj->btf_vmlinux);
7681         obj->btf_vmlinux = NULL;
7682
7683         obj->loaded = true; /* doesn't matter if successfully or not */
7684
7685         if (err)
7686                 goto out;
7687
7688         return 0;
7689 out:
7690         /* unpin any maps that were auto-pinned during load */
7691         for (i = 0; i < obj->nr_maps; i++)
7692                 if (obj->maps[i].pinned && !obj->maps[i].reused)
7693                         bpf_map__unpin(&obj->maps[i], NULL);
7694
7695         bpf_object_unload(obj);
7696         pr_warn("failed to load object '%s'\n", obj->path);
7697         return libbpf_err(err);
7698 }
7699
7700 int bpf_object__load(struct bpf_object *obj)
7701 {
7702         return bpf_object_load(obj, 0, NULL);
7703 }
7704
7705 static int make_parent_dir(const char *path)
7706 {
7707         char *cp, errmsg[STRERR_BUFSIZE];
7708         char *dname, *dir;
7709         int err = 0;
7710
7711         dname = strdup(path);
7712         if (dname == NULL)
7713                 return -ENOMEM;
7714
7715         dir = dirname(dname);
7716         if (mkdir(dir, 0700) && errno != EEXIST)
7717                 err = -errno;
7718
7719         free(dname);
7720         if (err) {
7721                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7722                 pr_warn("failed to mkdir %s: %s\n", path, cp);
7723         }
7724         return err;
7725 }
7726
7727 static int check_path(const char *path)
7728 {
7729         char *cp, errmsg[STRERR_BUFSIZE];
7730         struct statfs st_fs;
7731         char *dname, *dir;
7732         int err = 0;
7733
7734         if (path == NULL)
7735                 return -EINVAL;
7736
7737         dname = strdup(path);
7738         if (dname == NULL)
7739                 return -ENOMEM;
7740
7741         dir = dirname(dname);
7742         if (statfs(dir, &st_fs)) {
7743                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7744                 pr_warn("failed to statfs %s: %s\n", dir, cp);
7745                 err = -errno;
7746         }
7747         free(dname);
7748
7749         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
7750                 pr_warn("specified path %s is not on BPF FS\n", path);
7751                 err = -EINVAL;
7752         }
7753
7754         return err;
7755 }
7756
7757 int bpf_program__pin(struct bpf_program *prog, const char *path)
7758 {
7759         char *cp, errmsg[STRERR_BUFSIZE];
7760         int err;
7761
7762         if (prog->fd < 0) {
7763                 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
7764                 return libbpf_err(-EINVAL);
7765         }
7766
7767         err = make_parent_dir(path);
7768         if (err)
7769                 return libbpf_err(err);
7770
7771         err = check_path(path);
7772         if (err)
7773                 return libbpf_err(err);
7774
7775         if (bpf_obj_pin(prog->fd, path)) {
7776                 err = -errno;
7777                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
7778                 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
7779                 return libbpf_err(err);
7780         }
7781
7782         pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
7783         return 0;
7784 }
7785
7786 int bpf_program__unpin(struct bpf_program *prog, const char *path)
7787 {
7788         int err;
7789
7790         if (prog->fd < 0) {
7791                 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
7792                 return libbpf_err(-EINVAL);
7793         }
7794
7795         err = check_path(path);
7796         if (err)
7797                 return libbpf_err(err);
7798
7799         err = unlink(path);
7800         if (err)
7801                 return libbpf_err(-errno);
7802
7803         pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
7804         return 0;
7805 }
7806
7807 int bpf_map__pin(struct bpf_map *map, const char *path)
7808 {
7809         char *cp, errmsg[STRERR_BUFSIZE];
7810         int err;
7811
7812         if (map == NULL) {
7813                 pr_warn("invalid map pointer\n");
7814                 return libbpf_err(-EINVAL);
7815         }
7816
7817         if (map->pin_path) {
7818                 if (path && strcmp(path, map->pin_path)) {
7819                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7820                                 bpf_map__name(map), map->pin_path, path);
7821                         return libbpf_err(-EINVAL);
7822                 } else if (map->pinned) {
7823                         pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
7824                                  bpf_map__name(map), map->pin_path);
7825                         return 0;
7826                 }
7827         } else {
7828                 if (!path) {
7829                         pr_warn("missing a path to pin map '%s' at\n",
7830                                 bpf_map__name(map));
7831                         return libbpf_err(-EINVAL);
7832                 } else if (map->pinned) {
7833                         pr_warn("map '%s' already pinned\n", bpf_map__name(map));
7834                         return libbpf_err(-EEXIST);
7835                 }
7836
7837                 map->pin_path = strdup(path);
7838                 if (!map->pin_path) {
7839                         err = -errno;
7840                         goto out_err;
7841                 }
7842         }
7843
7844         err = make_parent_dir(map->pin_path);
7845         if (err)
7846                 return libbpf_err(err);
7847
7848         err = check_path(map->pin_path);
7849         if (err)
7850                 return libbpf_err(err);
7851
7852         if (bpf_obj_pin(map->fd, map->pin_path)) {
7853                 err = -errno;
7854                 goto out_err;
7855         }
7856
7857         map->pinned = true;
7858         pr_debug("pinned map '%s'\n", map->pin_path);
7859
7860         return 0;
7861
7862 out_err:
7863         cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7864         pr_warn("failed to pin map: %s\n", cp);
7865         return libbpf_err(err);
7866 }
7867
7868 int bpf_map__unpin(struct bpf_map *map, const char *path)
7869 {
7870         int err;
7871
7872         if (map == NULL) {
7873                 pr_warn("invalid map pointer\n");
7874                 return libbpf_err(-EINVAL);
7875         }
7876
7877         if (map->pin_path) {
7878                 if (path && strcmp(path, map->pin_path)) {
7879                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7880                                 bpf_map__name(map), map->pin_path, path);
7881                         return libbpf_err(-EINVAL);
7882                 }
7883                 path = map->pin_path;
7884         } else if (!path) {
7885                 pr_warn("no path to unpin map '%s' from\n",
7886                         bpf_map__name(map));
7887                 return libbpf_err(-EINVAL);
7888         }
7889
7890         err = check_path(path);
7891         if (err)
7892                 return libbpf_err(err);
7893
7894         err = unlink(path);
7895         if (err != 0)
7896                 return libbpf_err(-errno);
7897
7898         map->pinned = false;
7899         pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
7900
7901         return 0;
7902 }
7903
7904 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
7905 {
7906         char *new = NULL;
7907
7908         if (path) {
7909                 new = strdup(path);
7910                 if (!new)
7911                         return libbpf_err(-errno);
7912         }
7913
7914         free(map->pin_path);
7915         map->pin_path = new;
7916         return 0;
7917 }
7918
7919 __alias(bpf_map__pin_path)
7920 const char *bpf_map__get_pin_path(const struct bpf_map *map);
7921
7922 const char *bpf_map__pin_path(const struct bpf_map *map)
7923 {
7924         return map->pin_path;
7925 }
7926
7927 bool bpf_map__is_pinned(const struct bpf_map *map)
7928 {
7929         return map->pinned;
7930 }
7931
7932 static void sanitize_pin_path(char *s)
7933 {
7934         /* bpffs disallows periods in path names */
7935         while (*s) {
7936                 if (*s == '.')
7937                         *s = '_';
7938                 s++;
7939         }
7940 }
7941
7942 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
7943 {
7944         struct bpf_map *map;
7945         int err;
7946
7947         if (!obj)
7948                 return libbpf_err(-ENOENT);
7949
7950         if (!obj->loaded) {
7951                 pr_warn("object not yet loaded; load it first\n");
7952                 return libbpf_err(-ENOENT);
7953         }
7954
7955         bpf_object__for_each_map(map, obj) {
7956                 char *pin_path = NULL;
7957                 char buf[PATH_MAX];
7958
7959                 if (!map->autocreate)
7960                         continue;
7961
7962                 if (path) {
7963                         int len;
7964
7965                         len = snprintf(buf, PATH_MAX, "%s/%s", path,
7966                                        bpf_map__name(map));
7967                         if (len < 0) {
7968                                 err = -EINVAL;
7969                                 goto err_unpin_maps;
7970                         } else if (len >= PATH_MAX) {
7971                                 err = -ENAMETOOLONG;
7972                                 goto err_unpin_maps;
7973                         }
7974                         sanitize_pin_path(buf);
7975                         pin_path = buf;
7976                 } else if (!map->pin_path) {
7977                         continue;
7978                 }
7979
7980                 err = bpf_map__pin(map, pin_path);
7981                 if (err)
7982                         goto err_unpin_maps;
7983         }
7984
7985         return 0;
7986
7987 err_unpin_maps:
7988         while ((map = bpf_object__prev_map(obj, map))) {
7989                 if (!map->pin_path)
7990                         continue;
7991
7992                 bpf_map__unpin(map, NULL);
7993         }
7994
7995         return libbpf_err(err);
7996 }
7997
7998 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
7999 {
8000         struct bpf_map *map;
8001         int err;
8002
8003         if (!obj)
8004                 return libbpf_err(-ENOENT);
8005
8006         bpf_object__for_each_map(map, obj) {
8007                 char *pin_path = NULL;
8008                 char buf[PATH_MAX];
8009
8010                 if (path) {
8011                         int len;
8012
8013                         len = snprintf(buf, PATH_MAX, "%s/%s", path,
8014                                        bpf_map__name(map));
8015                         if (len < 0)
8016                                 return libbpf_err(-EINVAL);
8017                         else if (len >= PATH_MAX)
8018                                 return libbpf_err(-ENAMETOOLONG);
8019                         sanitize_pin_path(buf);
8020                         pin_path = buf;
8021                 } else if (!map->pin_path) {
8022                         continue;
8023                 }
8024
8025                 err = bpf_map__unpin(map, pin_path);
8026                 if (err)
8027                         return libbpf_err(err);
8028         }
8029
8030         return 0;
8031 }
8032
8033 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
8034 {
8035         struct bpf_program *prog;
8036         int err;
8037
8038         if (!obj)
8039                 return libbpf_err(-ENOENT);
8040
8041         if (!obj->loaded) {
8042                 pr_warn("object not yet loaded; load it first\n");
8043                 return libbpf_err(-ENOENT);
8044         }
8045
8046         bpf_object__for_each_program(prog, obj) {
8047                 char buf[PATH_MAX];
8048                 int len;
8049
8050                 len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name);
8051                 if (len < 0) {
8052                         err = -EINVAL;
8053                         goto err_unpin_programs;
8054                 } else if (len >= PATH_MAX) {
8055                         err = -ENAMETOOLONG;
8056                         goto err_unpin_programs;
8057                 }
8058
8059                 err = bpf_program__pin(prog, buf);
8060                 if (err)
8061                         goto err_unpin_programs;
8062         }
8063
8064         return 0;
8065
8066 err_unpin_programs:
8067         while ((prog = bpf_object__prev_program(obj, prog))) {
8068                 char buf[PATH_MAX];
8069                 int len;
8070
8071                 len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name);
8072                 if (len < 0)
8073                         continue;
8074                 else if (len >= PATH_MAX)
8075                         continue;
8076
8077                 bpf_program__unpin(prog, buf);
8078         }
8079
8080         return libbpf_err(err);
8081 }
8082
8083 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8084 {
8085         struct bpf_program *prog;
8086         int err;
8087
8088         if (!obj)
8089                 return libbpf_err(-ENOENT);
8090
8091         bpf_object__for_each_program(prog, obj) {
8092                 char buf[PATH_MAX];
8093                 int len;
8094
8095                 len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name);
8096                 if (len < 0)
8097                         return libbpf_err(-EINVAL);
8098                 else if (len >= PATH_MAX)
8099                         return libbpf_err(-ENAMETOOLONG);
8100
8101                 err = bpf_program__unpin(prog, buf);
8102                 if (err)
8103                         return libbpf_err(err);
8104         }
8105
8106         return 0;
8107 }
8108
8109 int bpf_object__pin(struct bpf_object *obj, const char *path)
8110 {
8111         int err;
8112
8113         err = bpf_object__pin_maps(obj, path);
8114         if (err)
8115                 return libbpf_err(err);
8116
8117         err = bpf_object__pin_programs(obj, path);
8118         if (err) {
8119                 bpf_object__unpin_maps(obj, path);
8120                 return libbpf_err(err);
8121         }
8122
8123         return 0;
8124 }
8125
8126 static void bpf_map__destroy(struct bpf_map *map)
8127 {
8128         if (map->inner_map) {
8129                 bpf_map__destroy(map->inner_map);
8130                 zfree(&map->inner_map);
8131         }
8132
8133         zfree(&map->init_slots);
8134         map->init_slots_sz = 0;
8135
8136         if (map->mmaped) {
8137                 munmap(map->mmaped, bpf_map_mmap_sz(map));
8138                 map->mmaped = NULL;
8139         }
8140
8141         if (map->st_ops) {
8142                 zfree(&map->st_ops->data);
8143                 zfree(&map->st_ops->progs);
8144                 zfree(&map->st_ops->kern_func_off);
8145                 zfree(&map->st_ops);
8146         }
8147
8148         zfree(&map->name);
8149         zfree(&map->real_name);
8150         zfree(&map->pin_path);
8151
8152         if (map->fd >= 0)
8153                 zclose(map->fd);
8154 }
8155
8156 void bpf_object__close(struct bpf_object *obj)
8157 {
8158         size_t i;
8159
8160         if (IS_ERR_OR_NULL(obj))
8161                 return;
8162
8163         usdt_manager_free(obj->usdt_man);
8164         obj->usdt_man = NULL;
8165
8166         bpf_gen__free(obj->gen_loader);
8167         bpf_object__elf_finish(obj);
8168         bpf_object_unload(obj);
8169         btf__free(obj->btf);
8170         btf_ext__free(obj->btf_ext);
8171
8172         for (i = 0; i < obj->nr_maps; i++)
8173                 bpf_map__destroy(&obj->maps[i]);
8174
8175         zfree(&obj->btf_custom_path);
8176         zfree(&obj->kconfig);
8177         zfree(&obj->externs);
8178         obj->nr_extern = 0;
8179
8180         zfree(&obj->maps);
8181         obj->nr_maps = 0;
8182
8183         if (obj->programs && obj->nr_programs) {
8184                 for (i = 0; i < obj->nr_programs; i++)
8185                         bpf_program__exit(&obj->programs[i]);
8186         }
8187         zfree(&obj->programs);
8188
8189         free(obj);
8190 }
8191
8192 const char *bpf_object__name(const struct bpf_object *obj)
8193 {
8194         return obj ? obj->name : libbpf_err_ptr(-EINVAL);
8195 }
8196
8197 unsigned int bpf_object__kversion(const struct bpf_object *obj)
8198 {
8199         return obj ? obj->kern_version : 0;
8200 }
8201
8202 struct btf *bpf_object__btf(const struct bpf_object *obj)
8203 {
8204         return obj ? obj->btf : NULL;
8205 }
8206
8207 int bpf_object__btf_fd(const struct bpf_object *obj)
8208 {
8209         return obj->btf ? btf__fd(obj->btf) : -1;
8210 }
8211
8212 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
8213 {
8214         if (obj->loaded)
8215                 return libbpf_err(-EINVAL);
8216
8217         obj->kern_version = kern_version;
8218
8219         return 0;
8220 }
8221
8222 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
8223 {
8224         struct bpf_gen *gen;
8225
8226         if (!opts)
8227                 return -EFAULT;
8228         if (!OPTS_VALID(opts, gen_loader_opts))
8229                 return -EINVAL;
8230         gen = calloc(sizeof(*gen), 1);
8231         if (!gen)
8232                 return -ENOMEM;
8233         gen->opts = opts;
8234         obj->gen_loader = gen;
8235         return 0;
8236 }
8237
8238 static struct bpf_program *
8239 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
8240                     bool forward)
8241 {
8242         size_t nr_programs = obj->nr_programs;
8243         ssize_t idx;
8244
8245         if (!nr_programs)
8246                 return NULL;
8247
8248         if (!p)
8249                 /* Iter from the beginning */
8250                 return forward ? &obj->programs[0] :
8251                         &obj->programs[nr_programs - 1];
8252
8253         if (p->obj != obj) {
8254                 pr_warn("error: program handler doesn't match object\n");
8255                 return errno = EINVAL, NULL;
8256         }
8257
8258         idx = (p - obj->programs) + (forward ? 1 : -1);
8259         if (idx >= obj->nr_programs || idx < 0)
8260                 return NULL;
8261         return &obj->programs[idx];
8262 }
8263
8264 struct bpf_program *
8265 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
8266 {
8267         struct bpf_program *prog = prev;
8268
8269         do {
8270                 prog = __bpf_program__iter(prog, obj, true);
8271         } while (prog && prog_is_subprog(obj, prog));
8272
8273         return prog;
8274 }
8275
8276 struct bpf_program *
8277 bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)
8278 {
8279         struct bpf_program *prog = next;
8280
8281         do {
8282                 prog = __bpf_program__iter(prog, obj, false);
8283         } while (prog && prog_is_subprog(obj, prog));
8284
8285         return prog;
8286 }
8287
8288 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
8289 {
8290         prog->prog_ifindex = ifindex;
8291 }
8292
8293 const char *bpf_program__name(const struct bpf_program *prog)
8294 {
8295         return prog->name;
8296 }
8297
8298 const char *bpf_program__section_name(const struct bpf_program *prog)
8299 {
8300         return prog->sec_name;
8301 }
8302
8303 bool bpf_program__autoload(const struct bpf_program *prog)
8304 {
8305         return prog->autoload;
8306 }
8307
8308 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
8309 {
8310         if (prog->obj->loaded)
8311                 return libbpf_err(-EINVAL);
8312
8313         prog->autoload = autoload;
8314         return 0;
8315 }
8316
8317 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
8318 {
8319         return prog->insns;
8320 }
8321
8322 size_t bpf_program__insn_cnt(const struct bpf_program *prog)
8323 {
8324         return prog->insns_cnt;
8325 }
8326
8327 int bpf_program__set_insns(struct bpf_program *prog,
8328                            struct bpf_insn *new_insns, size_t new_insn_cnt)
8329 {
8330         struct bpf_insn *insns;
8331
8332         if (prog->obj->loaded)
8333                 return -EBUSY;
8334
8335         insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
8336         if (!insns) {
8337                 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
8338                 return -ENOMEM;
8339         }
8340         memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));
8341
8342         prog->insns = insns;
8343         prog->insns_cnt = new_insn_cnt;
8344         return 0;
8345 }
8346
8347 int bpf_program__fd(const struct bpf_program *prog)
8348 {
8349         if (!prog)
8350                 return libbpf_err(-EINVAL);
8351
8352         if (prog->fd < 0)
8353                 return libbpf_err(-ENOENT);
8354
8355         return prog->fd;
8356 }
8357
8358 __alias(bpf_program__type)
8359 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
8360
8361 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
8362 {
8363         return prog->type;
8364 }
8365
8366 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
8367 {
8368         if (prog->obj->loaded)
8369                 return libbpf_err(-EBUSY);
8370
8371         prog->type = type;
8372         return 0;
8373 }
8374
8375 __alias(bpf_program__expected_attach_type)
8376 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
8377
8378 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)
8379 {
8380         return prog->expected_attach_type;
8381 }
8382
8383 int bpf_program__set_expected_attach_type(struct bpf_program *prog,
8384                                            enum bpf_attach_type type)
8385 {
8386         if (prog->obj->loaded)
8387                 return libbpf_err(-EBUSY);
8388
8389         prog->expected_attach_type = type;
8390         return 0;
8391 }
8392
8393 __u32 bpf_program__flags(const struct bpf_program *prog)
8394 {
8395         return prog->prog_flags;
8396 }
8397
8398 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
8399 {
8400         if (prog->obj->loaded)
8401                 return libbpf_err(-EBUSY);
8402
8403         prog->prog_flags = flags;
8404         return 0;
8405 }
8406
8407 __u32 bpf_program__log_level(const struct bpf_program *prog)
8408 {
8409         return prog->log_level;
8410 }
8411
8412 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
8413 {
8414         if (prog->obj->loaded)
8415                 return libbpf_err(-EBUSY);
8416
8417         prog->log_level = log_level;
8418         return 0;
8419 }
8420
8421 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size)
8422 {
8423         *log_size = prog->log_size;
8424         return prog->log_buf;
8425 }
8426
8427 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
8428 {
8429         if (log_size && !log_buf)
8430                 return -EINVAL;
8431         if (prog->log_size > UINT_MAX)
8432                 return -EINVAL;
8433         if (prog->obj->loaded)
8434                 return -EBUSY;
8435
8436         prog->log_buf = log_buf;
8437         prog->log_size = log_size;
8438         return 0;
8439 }
8440
8441 #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) {                        \
8442         .sec = (char *)sec_pfx,                                             \
8443         .prog_type = BPF_PROG_TYPE_##ptype,                                 \
8444         .expected_attach_type = atype,                                      \
8445         .cookie = (long)(flags),                                            \
8446         .prog_prepare_load_fn = libbpf_prepare_prog_load,                   \
8447         __VA_ARGS__                                                         \
8448 }
8449
8450 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8451 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8452 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8453 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8454 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8455 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8456 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8457 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8458 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8459 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8460
8461 static const struct bpf_sec_def section_defs[] = {
8462         SEC_DEF("socket",               SOCKET_FILTER, 0, SEC_NONE),
8463         SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE),
8464         SEC_DEF("sk_reuseport",         SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE),
8465         SEC_DEF("kprobe+",              KPROBE, 0, SEC_NONE, attach_kprobe),
8466         SEC_DEF("uprobe+",              KPROBE, 0, SEC_NONE, attach_uprobe),
8467         SEC_DEF("uprobe.s+",            KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
8468         SEC_DEF("kretprobe+",           KPROBE, 0, SEC_NONE, attach_kprobe),
8469         SEC_DEF("uretprobe+",           KPROBE, 0, SEC_NONE, attach_uprobe),
8470         SEC_DEF("uretprobe.s+",         KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
8471         SEC_DEF("kprobe.multi+",        KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
8472         SEC_DEF("kretprobe.multi+",     KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
8473         SEC_DEF("ksyscall+",            KPROBE, 0, SEC_NONE, attach_ksyscall),
8474         SEC_DEF("kretsyscall+",         KPROBE, 0, SEC_NONE, attach_ksyscall),
8475         SEC_DEF("usdt+",                KPROBE, 0, SEC_NONE, attach_usdt),
8476         SEC_DEF("tc",                   SCHED_CLS, 0, SEC_NONE),
8477         SEC_DEF("classifier",           SCHED_CLS, 0, SEC_NONE),
8478         SEC_DEF("action",               SCHED_ACT, 0, SEC_NONE),
8479         SEC_DEF("tracepoint+",          TRACEPOINT, 0, SEC_NONE, attach_tp),
8480         SEC_DEF("tp+",                  TRACEPOINT, 0, SEC_NONE, attach_tp),
8481         SEC_DEF("raw_tracepoint+",      RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
8482         SEC_DEF("raw_tp+",              RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
8483         SEC_DEF("raw_tracepoint.w+",    RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
8484         SEC_DEF("raw_tp.w+",            RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
8485         SEC_DEF("tp_btf+",              TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace),
8486         SEC_DEF("fentry+",              TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace),
8487         SEC_DEF("fmod_ret+",            TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace),
8488         SEC_DEF("fexit+",               TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace),
8489         SEC_DEF("fentry.s+",            TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8490         SEC_DEF("fmod_ret.s+",          TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8491         SEC_DEF("fexit.s+",             TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8492         SEC_DEF("freplace+",            EXT, 0, SEC_ATTACH_BTF, attach_trace),
8493         SEC_DEF("lsm+",                 LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
8494         SEC_DEF("lsm.s+",               LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
8495         SEC_DEF("lsm_cgroup+",          LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF),
8496         SEC_DEF("iter+",                TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
8497         SEC_DEF("iter.s+",              TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
8498         SEC_DEF("syscall",              SYSCALL, 0, SEC_SLEEPABLE),
8499         SEC_DEF("xdp.frags/devmap",     XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS),
8500         SEC_DEF("xdp/devmap",           XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
8501         SEC_DEF("xdp.frags/cpumap",     XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS),
8502         SEC_DEF("xdp/cpumap",           XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
8503         SEC_DEF("xdp.frags",            XDP, BPF_XDP, SEC_XDP_FRAGS),
8504         SEC_DEF("xdp",                  XDP, BPF_XDP, SEC_ATTACHABLE_OPT),
8505         SEC_DEF("perf_event",           PERF_EVENT, 0, SEC_NONE),
8506         SEC_DEF("lwt_in",               LWT_IN, 0, SEC_NONE),
8507         SEC_DEF("lwt_out",              LWT_OUT, 0, SEC_NONE),
8508         SEC_DEF("lwt_xmit",             LWT_XMIT, 0, SEC_NONE),
8509         SEC_DEF("lwt_seg6local",        LWT_SEG6LOCAL, 0, SEC_NONE),
8510         SEC_DEF("sockops",              SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
8511         SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
8512         SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
8513         SEC_DEF("sk_skb",               SK_SKB, 0, SEC_NONE),
8514         SEC_DEF("sk_msg",               SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
8515         SEC_DEF("lirc_mode2",           LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
8516         SEC_DEF("flow_dissector",       FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT),
8517         SEC_DEF("cgroup_skb/ingress",   CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT),
8518         SEC_DEF("cgroup_skb/egress",    CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT),
8519         SEC_DEF("cgroup/skb",           CGROUP_SKB, 0, SEC_NONE),
8520         SEC_DEF("cgroup/sock_create",   CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE),
8521         SEC_DEF("cgroup/sock_release",  CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE),
8522         SEC_DEF("cgroup/sock",          CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT),
8523         SEC_DEF("cgroup/post_bind4",    CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE),
8524         SEC_DEF("cgroup/post_bind6",    CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE),
8525         SEC_DEF("cgroup/bind4",         CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE),
8526         SEC_DEF("cgroup/bind6",         CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE),
8527         SEC_DEF("cgroup/connect4",      CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE),
8528         SEC_DEF("cgroup/connect6",      CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE),
8529         SEC_DEF("cgroup/sendmsg4",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE),
8530         SEC_DEF("cgroup/sendmsg6",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE),
8531         SEC_DEF("cgroup/recvmsg4",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE),
8532         SEC_DEF("cgroup/recvmsg6",      CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE),
8533         SEC_DEF("cgroup/getpeername4",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE),
8534         SEC_DEF("cgroup/getpeername6",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE),
8535         SEC_DEF("cgroup/getsockname4",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE),
8536         SEC_DEF("cgroup/getsockname6",  CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE),
8537         SEC_DEF("cgroup/sysctl",        CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE),
8538         SEC_DEF("cgroup/getsockopt",    CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE),
8539         SEC_DEF("cgroup/setsockopt",    CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE),
8540         SEC_DEF("cgroup/dev",           CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),
8541         SEC_DEF("struct_ops+",          STRUCT_OPS, 0, SEC_NONE),
8542         SEC_DEF("sk_lookup",            SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE),
8543 };
8544
8545 static size_t custom_sec_def_cnt;
8546 static struct bpf_sec_def *custom_sec_defs;
8547 static struct bpf_sec_def custom_fallback_def;
8548 static bool has_custom_fallback_def;
8549
8550 static int last_custom_sec_def_handler_id;
8551
8552 int libbpf_register_prog_handler(const char *sec,
8553                                  enum bpf_prog_type prog_type,
8554                                  enum bpf_attach_type exp_attach_type,
8555                                  const struct libbpf_prog_handler_opts *opts)
8556 {
8557         struct bpf_sec_def *sec_def;
8558
8559         if (!OPTS_VALID(opts, libbpf_prog_handler_opts))
8560                 return libbpf_err(-EINVAL);
8561
8562         if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */
8563                 return libbpf_err(-E2BIG);
8564
8565         if (sec) {
8566                 sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1,
8567                                               sizeof(*sec_def));
8568                 if (!sec_def)
8569                         return libbpf_err(-ENOMEM);
8570
8571                 custom_sec_defs = sec_def;
8572                 sec_def = &custom_sec_defs[custom_sec_def_cnt];
8573         } else {
8574                 if (has_custom_fallback_def)
8575                         return libbpf_err(-EBUSY);
8576
8577                 sec_def = &custom_fallback_def;
8578         }
8579
8580         sec_def->sec = sec ? strdup(sec) : NULL;
8581         if (sec && !sec_def->sec)
8582                 return libbpf_err(-ENOMEM);
8583
8584         sec_def->prog_type = prog_type;
8585         sec_def->expected_attach_type = exp_attach_type;
8586         sec_def->cookie = OPTS_GET(opts, cookie, 0);
8587
8588         sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
8589         sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
8590         sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
8591
8592         sec_def->handler_id = ++last_custom_sec_def_handler_id;
8593
8594         if (sec)
8595                 custom_sec_def_cnt++;
8596         else
8597                 has_custom_fallback_def = true;
8598
8599         return sec_def->handler_id;
8600 }
8601
8602 int libbpf_unregister_prog_handler(int handler_id)
8603 {
8604         struct bpf_sec_def *sec_defs;
8605         int i;
8606
8607         if (handler_id <= 0)
8608                 return libbpf_err(-EINVAL);
8609
8610         if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) {
8611                 memset(&custom_fallback_def, 0, sizeof(custom_fallback_def));
8612                 has_custom_fallback_def = false;
8613                 return 0;
8614         }
8615
8616         for (i = 0; i < custom_sec_def_cnt; i++) {
8617                 if (custom_sec_defs[i].handler_id == handler_id)
8618                         break;
8619         }
8620
8621         if (i == custom_sec_def_cnt)
8622                 return libbpf_err(-ENOENT);
8623
8624         free(custom_sec_defs[i].sec);
8625         for (i = i + 1; i < custom_sec_def_cnt; i++)
8626                 custom_sec_defs[i - 1] = custom_sec_defs[i];
8627         custom_sec_def_cnt--;
8628
8629         /* try to shrink the array, but it's ok if we couldn't */
8630         sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs));
8631         if (sec_defs)
8632                 custom_sec_defs = sec_defs;
8633
8634         return 0;
8635 }
8636
8637 static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name)
8638 {
8639         size_t len = strlen(sec_def->sec);
8640
8641         /* "type/" always has to have proper SEC("type/extras") form */
8642         if (sec_def->sec[len - 1] == '/') {
8643                 if (str_has_pfx(sec_name, sec_def->sec))
8644                         return true;
8645                 return false;
8646         }
8647
8648         /* "type+" means it can be either exact SEC("type") or
8649          * well-formed SEC("type/extras") with proper '/' separator
8650          */
8651         if (sec_def->sec[len - 1] == '+') {
8652                 len--;
8653                 /* not even a prefix */
8654                 if (strncmp(sec_name, sec_def->sec, len) != 0)
8655                         return false;
8656                 /* exact match or has '/' separator */
8657                 if (sec_name[len] == '\0' || sec_name[len] == '/')
8658                         return true;
8659                 return false;
8660         }
8661
8662         return strcmp(sec_name, sec_def->sec) == 0;
8663 }
8664
8665 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
8666 {
8667         const struct bpf_sec_def *sec_def;
8668         int i, n;
8669
8670         n = custom_sec_def_cnt;
8671         for (i = 0; i < n; i++) {
8672                 sec_def = &custom_sec_defs[i];
8673                 if (sec_def_matches(sec_def, sec_name))
8674                         return sec_def;
8675         }
8676
8677         n = ARRAY_SIZE(section_defs);
8678         for (i = 0; i < n; i++) {
8679                 sec_def = &section_defs[i];
8680                 if (sec_def_matches(sec_def, sec_name))
8681                         return sec_def;
8682         }
8683
8684         if (has_custom_fallback_def)
8685                 return &custom_fallback_def;
8686
8687         return NULL;
8688 }
8689
8690 #define MAX_TYPE_NAME_SIZE 32
8691
8692 static char *libbpf_get_type_names(bool attach_type)
8693 {
8694         int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
8695         char *buf;
8696
8697         buf = malloc(len);
8698         if (!buf)
8699                 return NULL;
8700
8701         buf[0] = '\0';
8702         /* Forge string buf with all available names */
8703         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8704                 const struct bpf_sec_def *sec_def = &section_defs[i];
8705
8706                 if (attach_type) {
8707                         if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
8708                                 continue;
8709
8710                         if (!(sec_def->cookie & SEC_ATTACHABLE))
8711                                 continue;
8712                 }
8713
8714                 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
8715                         free(buf);
8716                         return NULL;
8717                 }
8718                 strcat(buf, " ");
8719                 strcat(buf, section_defs[i].sec);
8720         }
8721
8722         return buf;
8723 }
8724
8725 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
8726                              enum bpf_attach_type *expected_attach_type)
8727 {
8728         const struct bpf_sec_def *sec_def;
8729         char *type_names;
8730
8731         if (!name)
8732                 return libbpf_err(-EINVAL);
8733
8734         sec_def = find_sec_def(name);
8735         if (sec_def) {
8736                 *prog_type = sec_def->prog_type;
8737                 *expected_attach_type = sec_def->expected_attach_type;
8738                 return 0;
8739         }
8740
8741         pr_debug("failed to guess program type from ELF section '%s'\n", name);
8742         type_names = libbpf_get_type_names(false);
8743         if (type_names != NULL) {
8744                 pr_debug("supported section(type) names are:%s\n", type_names);
8745                 free(type_names);
8746         }
8747
8748         return libbpf_err(-ESRCH);
8749 }
8750
8751 const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t)
8752 {
8753         if (t < 0 || t >= ARRAY_SIZE(attach_type_name))
8754                 return NULL;
8755
8756         return attach_type_name[t];
8757 }
8758
8759 const char *libbpf_bpf_link_type_str(enum bpf_link_type t)
8760 {
8761         if (t < 0 || t >= ARRAY_SIZE(link_type_name))
8762                 return NULL;
8763
8764         return link_type_name[t];
8765 }
8766
8767 const char *libbpf_bpf_map_type_str(enum bpf_map_type t)
8768 {
8769         if (t < 0 || t >= ARRAY_SIZE(map_type_name))
8770                 return NULL;
8771
8772         return map_type_name[t];
8773 }
8774
8775 const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t)
8776 {
8777         if (t < 0 || t >= ARRAY_SIZE(prog_type_name))
8778                 return NULL;
8779
8780         return prog_type_name[t];
8781 }
8782
8783 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
8784                                                      size_t offset)
8785 {
8786         struct bpf_map *map;
8787         size_t i;
8788
8789         for (i = 0; i < obj->nr_maps; i++) {
8790                 map = &obj->maps[i];
8791                 if (!bpf_map__is_struct_ops(map))
8792                         continue;
8793                 if (map->sec_offset <= offset &&
8794                     offset - map->sec_offset < map->def.value_size)
8795                         return map;
8796         }
8797
8798         return NULL;
8799 }
8800
8801 /* Collect the reloc from ELF and populate the st_ops->progs[] */
8802 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
8803                                             Elf64_Shdr *shdr, Elf_Data *data)
8804 {
8805         const struct btf_member *member;
8806         struct bpf_struct_ops *st_ops;
8807         struct bpf_program *prog;
8808         unsigned int shdr_idx;
8809         const struct btf *btf;
8810         struct bpf_map *map;
8811         unsigned int moff, insn_idx;
8812         const char *name;
8813         __u32 member_idx;
8814         Elf64_Sym *sym;
8815         Elf64_Rel *rel;
8816         int i, nrels;
8817
8818         btf = obj->btf;
8819         nrels = shdr->sh_size / shdr->sh_entsize;
8820         for (i = 0; i < nrels; i++) {
8821                 rel = elf_rel_by_idx(data, i);
8822                 if (!rel) {
8823                         pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
8824                         return -LIBBPF_ERRNO__FORMAT;
8825                 }
8826
8827                 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
8828                 if (!sym) {
8829                         pr_warn("struct_ops reloc: symbol %zx not found\n",
8830                                 (size_t)ELF64_R_SYM(rel->r_info));
8831                         return -LIBBPF_ERRNO__FORMAT;
8832                 }
8833
8834                 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
8835                 map = find_struct_ops_map_by_offset(obj, rel->r_offset);
8836                 if (!map) {
8837                         pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
8838                                 (size_t)rel->r_offset);
8839                         return -EINVAL;
8840                 }
8841
8842                 moff = rel->r_offset - map->sec_offset;
8843                 shdr_idx = sym->st_shndx;
8844                 st_ops = map->st_ops;
8845                 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
8846                          map->name,
8847                          (long long)(rel->r_info >> 32),
8848                          (long long)sym->st_value,
8849                          shdr_idx, (size_t)rel->r_offset,
8850                          map->sec_offset, sym->st_name, name);
8851
8852                 if (shdr_idx >= SHN_LORESERVE) {
8853                         pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
8854                                 map->name, (size_t)rel->r_offset, shdr_idx);
8855                         return -LIBBPF_ERRNO__RELOC;
8856                 }
8857                 if (sym->st_value % BPF_INSN_SZ) {
8858                         pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
8859                                 map->name, (unsigned long long)sym->st_value);
8860                         return -LIBBPF_ERRNO__FORMAT;
8861                 }
8862                 insn_idx = sym->st_value / BPF_INSN_SZ;
8863
8864                 member = find_member_by_offset(st_ops->type, moff * 8);
8865                 if (!member) {
8866                         pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
8867                                 map->name, moff);
8868                         return -EINVAL;
8869                 }
8870                 member_idx = member - btf_members(st_ops->type);
8871                 name = btf__name_by_offset(btf, member->name_off);
8872
8873                 if (!resolve_func_ptr(btf, member->type, NULL)) {
8874                         pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
8875                                 map->name, name);
8876                         return -EINVAL;
8877                 }
8878
8879                 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
8880                 if (!prog) {
8881                         pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
8882                                 map->name, shdr_idx, name);
8883                         return -EINVAL;
8884                 }
8885
8886                 /* prevent the use of BPF prog with invalid type */
8887                 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
8888                         pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
8889                                 map->name, prog->name);
8890                         return -EINVAL;
8891                 }
8892
8893                 /* if we haven't yet processed this BPF program, record proper
8894                  * attach_btf_id and member_idx
8895                  */
8896                 if (!prog->attach_btf_id) {
8897                         prog->attach_btf_id = st_ops->type_id;
8898                         prog->expected_attach_type = member_idx;
8899                 }
8900
8901                 /* struct_ops BPF prog can be re-used between multiple
8902                  * .struct_ops as long as it's the same struct_ops struct
8903                  * definition and the same function pointer field
8904                  */
8905                 if (prog->attach_btf_id != st_ops->type_id ||
8906                     prog->expected_attach_type != member_idx) {
8907                         pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8908                                 map->name, prog->name, prog->sec_name, prog->type,
8909                                 prog->attach_btf_id, prog->expected_attach_type, name);
8910                         return -EINVAL;
8911                 }
8912
8913                 st_ops->progs[member_idx] = prog;
8914         }
8915
8916         return 0;
8917 }
8918
8919 #define BTF_TRACE_PREFIX "btf_trace_"
8920 #define BTF_LSM_PREFIX "bpf_lsm_"
8921 #define BTF_ITER_PREFIX "bpf_iter_"
8922 #define BTF_MAX_NAME_SIZE 128
8923
8924 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
8925                                 const char **prefix, int *kind)
8926 {
8927         switch (attach_type) {
8928         case BPF_TRACE_RAW_TP:
8929                 *prefix = BTF_TRACE_PREFIX;
8930                 *kind = BTF_KIND_TYPEDEF;
8931                 break;
8932         case BPF_LSM_MAC:
8933         case BPF_LSM_CGROUP:
8934                 *prefix = BTF_LSM_PREFIX;
8935                 *kind = BTF_KIND_FUNC;
8936                 break;
8937         case BPF_TRACE_ITER:
8938                 *prefix = BTF_ITER_PREFIX;
8939                 *kind = BTF_KIND_FUNC;
8940                 break;
8941         default:
8942                 *prefix = "";
8943                 *kind = BTF_KIND_FUNC;
8944         }
8945 }
8946
8947 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
8948                                    const char *name, __u32 kind)
8949 {
8950         char btf_type_name[BTF_MAX_NAME_SIZE];
8951         int ret;
8952
8953         ret = snprintf(btf_type_name, sizeof(btf_type_name),
8954                        "%s%s", prefix, name);
8955         /* snprintf returns the number of characters written excluding the
8956          * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
8957          * indicates truncation.
8958          */
8959         if (ret < 0 || ret >= sizeof(btf_type_name))
8960                 return -ENAMETOOLONG;
8961         return btf__find_by_name_kind(btf, btf_type_name, kind);
8962 }
8963
8964 static inline int find_attach_btf_id(struct btf *btf, const char *name,
8965                                      enum bpf_attach_type attach_type)
8966 {
8967         const char *prefix;
8968         int kind;
8969
8970         btf_get_kernel_prefix_kind(attach_type, &prefix, &kind);
8971         return find_btf_by_prefix_kind(btf, prefix, name, kind);
8972 }
8973
8974 int libbpf_find_vmlinux_btf_id(const char *name,
8975                                enum bpf_attach_type attach_type)
8976 {
8977         struct btf *btf;
8978         int err;
8979
8980         btf = btf__load_vmlinux_btf();
8981         err = libbpf_get_error(btf);
8982         if (err) {
8983                 pr_warn("vmlinux BTF is not found\n");
8984                 return libbpf_err(err);
8985         }
8986
8987         err = find_attach_btf_id(btf, name, attach_type);
8988         if (err <= 0)
8989                 pr_warn("%s is not found in vmlinux BTF\n", name);
8990
8991         btf__free(btf);
8992         return libbpf_err(err);
8993 }
8994
8995 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
8996 {
8997         struct bpf_prog_info info = {};
8998         __u32 info_len = sizeof(info);
8999         struct btf *btf;
9000         int err;
9001
9002         err = bpf_obj_get_info_by_fd(attach_prog_fd, &info, &info_len);
9003         if (err) {
9004                 pr_warn("failed bpf_obj_get_info_by_fd for FD %d: %d\n",
9005                         attach_prog_fd, err);
9006                 return err;
9007         }
9008
9009         err = -EINVAL;
9010         if (!info.btf_id) {
9011                 pr_warn("The target program doesn't have BTF\n");
9012                 goto out;
9013         }
9014         btf = btf__load_from_kernel_by_id(info.btf_id);
9015         err = libbpf_get_error(btf);
9016         if (err) {
9017                 pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err);
9018                 goto out;
9019         }
9020         err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
9021         btf__free(btf);
9022         if (err <= 0) {
9023                 pr_warn("%s is not found in prog's BTF\n", name);
9024                 goto out;
9025         }
9026 out:
9027         return err;
9028 }
9029
9030 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
9031                               enum bpf_attach_type attach_type,
9032                               int *btf_obj_fd, int *btf_type_id)
9033 {
9034         int ret, i;
9035
9036         ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
9037         if (ret > 0) {
9038                 *btf_obj_fd = 0; /* vmlinux BTF */
9039                 *btf_type_id = ret;
9040                 return 0;
9041         }
9042         if (ret != -ENOENT)
9043                 return ret;
9044
9045         ret = load_module_btfs(obj);
9046         if (ret)
9047                 return ret;
9048
9049         for (i = 0; i < obj->btf_module_cnt; i++) {
9050                 const struct module_btf *mod = &obj->btf_modules[i];
9051
9052                 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
9053                 if (ret > 0) {
9054                         *btf_obj_fd = mod->fd;
9055                         *btf_type_id = ret;
9056                         return 0;
9057                 }
9058                 if (ret == -ENOENT)
9059                         continue;
9060
9061                 return ret;
9062         }
9063
9064         return -ESRCH;
9065 }
9066
9067 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
9068                                      int *btf_obj_fd, int *btf_type_id)
9069 {
9070         enum bpf_attach_type attach_type = prog->expected_attach_type;
9071         __u32 attach_prog_fd = prog->attach_prog_fd;
9072         int err = 0;
9073
9074         /* BPF program's BTF ID */
9075         if (attach_prog_fd) {
9076                 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
9077                 if (err < 0) {
9078                         pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
9079                                  attach_prog_fd, attach_name, err);
9080                         return err;
9081                 }
9082                 *btf_obj_fd = 0;
9083                 *btf_type_id = err;
9084                 return 0;
9085         }
9086
9087         /* kernel/module BTF ID */
9088         if (prog->obj->gen_loader) {
9089                 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9090                 *btf_obj_fd = 0;
9091                 *btf_type_id = 1;
9092         } else {
9093                 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9094         }
9095         if (err) {
9096                 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err);
9097                 return err;
9098         }
9099         return 0;
9100 }
9101
9102 int libbpf_attach_type_by_name(const char *name,
9103                                enum bpf_attach_type *attach_type)
9104 {
9105         char *type_names;
9106         const struct bpf_sec_def *sec_def;
9107
9108         if (!name)
9109                 return libbpf_err(-EINVAL);
9110
9111         sec_def = find_sec_def(name);
9112         if (!sec_def) {
9113                 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
9114                 type_names = libbpf_get_type_names(true);
9115                 if (type_names != NULL) {
9116                         pr_debug("attachable section(type) names are:%s\n", type_names);
9117                         free(type_names);
9118                 }
9119
9120                 return libbpf_err(-EINVAL);
9121         }
9122
9123         if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9124                 return libbpf_err(-EINVAL);
9125         if (!(sec_def->cookie & SEC_ATTACHABLE))
9126                 return libbpf_err(-EINVAL);
9127
9128         *attach_type = sec_def->expected_attach_type;
9129         return 0;
9130 }
9131
9132 int bpf_map__fd(const struct bpf_map *map)
9133 {
9134         return map ? map->fd : libbpf_err(-EINVAL);
9135 }
9136
9137 static bool map_uses_real_name(const struct bpf_map *map)
9138 {
9139         /* Since libbpf started to support custom .data.* and .rodata.* maps,
9140          * their user-visible name differs from kernel-visible name. Users see
9141          * such map's corresponding ELF section name as a map name.
9142          * This check distinguishes .data/.rodata from .data.* and .rodata.*
9143          * maps to know which name has to be returned to the user.
9144          */
9145         if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
9146                 return true;
9147         if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
9148                 return true;
9149         return false;
9150 }
9151
9152 const char *bpf_map__name(const struct bpf_map *map)
9153 {
9154         if (!map)
9155                 return NULL;
9156
9157         if (map_uses_real_name(map))
9158                 return map->real_name;
9159
9160         return map->name;
9161 }
9162
9163 enum bpf_map_type bpf_map__type(const struct bpf_map *map)
9164 {
9165         return map->def.type;
9166 }
9167
9168 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
9169 {
9170         if (map->fd >= 0)
9171                 return libbpf_err(-EBUSY);
9172         map->def.type = type;
9173         return 0;
9174 }
9175
9176 __u32 bpf_map__map_flags(const struct bpf_map *map)
9177 {
9178         return map->def.map_flags;
9179 }
9180
9181 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
9182 {
9183         if (map->fd >= 0)
9184                 return libbpf_err(-EBUSY);
9185         map->def.map_flags = flags;
9186         return 0;
9187 }
9188
9189 __u64 bpf_map__map_extra(const struct bpf_map *map)
9190 {
9191         return map->map_extra;
9192 }
9193
9194 int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra)
9195 {
9196         if (map->fd >= 0)
9197                 return libbpf_err(-EBUSY);
9198         map->map_extra = map_extra;
9199         return 0;
9200 }
9201
9202 __u32 bpf_map__numa_node(const struct bpf_map *map)
9203 {
9204         return map->numa_node;
9205 }
9206
9207 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
9208 {
9209         if (map->fd >= 0)
9210                 return libbpf_err(-EBUSY);
9211         map->numa_node = numa_node;
9212         return 0;
9213 }
9214
9215 __u32 bpf_map__key_size(const struct bpf_map *map)
9216 {
9217         return map->def.key_size;
9218 }
9219
9220 int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
9221 {
9222         if (map->fd >= 0)
9223                 return libbpf_err(-EBUSY);
9224         map->def.key_size = size;
9225         return 0;
9226 }
9227
9228 __u32 bpf_map__value_size(const struct bpf_map *map)
9229 {
9230         return map->def.value_size;
9231 }
9232
9233 int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
9234 {
9235         if (map->fd >= 0)
9236                 return libbpf_err(-EBUSY);
9237         map->def.value_size = size;
9238         return 0;
9239 }
9240
9241 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
9242 {
9243         return map ? map->btf_key_type_id : 0;
9244 }
9245
9246 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
9247 {
9248         return map ? map->btf_value_type_id : 0;
9249 }
9250
9251 int bpf_map__set_initial_value(struct bpf_map *map,
9252                                const void *data, size_t size)
9253 {
9254         if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
9255             size != map->def.value_size || map->fd >= 0)
9256                 return libbpf_err(-EINVAL);
9257
9258         memcpy(map->mmaped, data, size);
9259         return 0;
9260 }
9261
9262 const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize)
9263 {
9264         if (!map->mmaped)
9265                 return NULL;
9266         *psize = map->def.value_size;
9267         return map->mmaped;
9268 }
9269
9270 bool bpf_map__is_internal(const struct bpf_map *map)
9271 {
9272         return map->libbpf_type != LIBBPF_MAP_UNSPEC;
9273 }
9274
9275 __u32 bpf_map__ifindex(const struct bpf_map *map)
9276 {
9277         return map->map_ifindex;
9278 }
9279
9280 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
9281 {
9282         if (map->fd >= 0)
9283                 return libbpf_err(-EBUSY);
9284         map->map_ifindex = ifindex;
9285         return 0;
9286 }
9287
9288 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
9289 {
9290         if (!bpf_map_type__is_map_in_map(map->def.type)) {
9291                 pr_warn("error: unsupported map type\n");
9292                 return libbpf_err(-EINVAL);
9293         }
9294         if (map->inner_map_fd != -1) {
9295                 pr_warn("error: inner_map_fd already specified\n");
9296                 return libbpf_err(-EINVAL);
9297         }
9298         if (map->inner_map) {
9299                 bpf_map__destroy(map->inner_map);
9300                 zfree(&map->inner_map);
9301         }
9302         map->inner_map_fd = fd;
9303         return 0;
9304 }
9305
9306 static struct bpf_map *
9307 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
9308 {
9309         ssize_t idx;
9310         struct bpf_map *s, *e;
9311
9312         if (!obj || !obj->maps)
9313                 return errno = EINVAL, NULL;
9314
9315         s = obj->maps;
9316         e = obj->maps + obj->nr_maps;
9317
9318         if ((m < s) || (m >= e)) {
9319                 pr_warn("error in %s: map handler doesn't belong to object\n",
9320                          __func__);
9321                 return errno = EINVAL, NULL;
9322         }
9323
9324         idx = (m - obj->maps) + i;
9325         if (idx >= obj->nr_maps || idx < 0)
9326                 return NULL;
9327         return &obj->maps[idx];
9328 }
9329
9330 struct bpf_map *
9331 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
9332 {
9333         if (prev == NULL)
9334                 return obj->maps;
9335
9336         return __bpf_map__iter(prev, obj, 1);
9337 }
9338
9339 struct bpf_map *
9340 bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next)
9341 {
9342         if (next == NULL) {
9343                 if (!obj->nr_maps)
9344                         return NULL;
9345                 return obj->maps + obj->nr_maps - 1;
9346         }
9347
9348         return __bpf_map__iter(next, obj, -1);
9349 }
9350
9351 struct bpf_map *
9352 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
9353 {
9354         struct bpf_map *pos;
9355
9356         bpf_object__for_each_map(pos, obj) {
9357                 /* if it's a special internal map name (which always starts
9358                  * with dot) then check if that special name matches the
9359                  * real map name (ELF section name)
9360                  */
9361                 if (name[0] == '.') {
9362                         if (pos->real_name && strcmp(pos->real_name, name) == 0)
9363                                 return pos;
9364                         continue;
9365                 }
9366                 /* otherwise map name has to be an exact match */
9367                 if (map_uses_real_name(pos)) {
9368                         if (strcmp(pos->real_name, name) == 0)
9369                                 return pos;
9370                         continue;
9371                 }
9372                 if (strcmp(pos->name, name) == 0)
9373                         return pos;
9374         }
9375         return errno = ENOENT, NULL;
9376 }
9377
9378 int
9379 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
9380 {
9381         return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
9382 }
9383
9384 static int validate_map_op(const struct bpf_map *map, size_t key_sz,
9385                            size_t value_sz, bool check_value_sz)
9386 {
9387         if (map->fd <= 0)
9388                 return -ENOENT;
9389
9390         if (map->def.key_size != key_sz) {
9391                 pr_warn("map '%s': unexpected key size %zu provided, expected %u\n",
9392                         map->name, key_sz, map->def.key_size);
9393                 return -EINVAL;
9394         }
9395
9396         if (!check_value_sz)
9397                 return 0;
9398
9399         switch (map->def.type) {
9400         case BPF_MAP_TYPE_PERCPU_ARRAY:
9401         case BPF_MAP_TYPE_PERCPU_HASH:
9402         case BPF_MAP_TYPE_LRU_PERCPU_HASH:
9403         case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: {
9404                 int num_cpu = libbpf_num_possible_cpus();
9405                 size_t elem_sz = roundup(map->def.value_size, 8);
9406
9407                 if (value_sz != num_cpu * elem_sz) {
9408                         pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
9409                                 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
9410                         return -EINVAL;
9411                 }
9412                 break;
9413         }
9414         default:
9415                 if (map->def.value_size != value_sz) {
9416                         pr_warn("map '%s': unexpected value size %zu provided, expected %u\n",
9417                                 map->name, value_sz, map->def.value_size);
9418                         return -EINVAL;
9419                 }
9420                 break;
9421         }
9422         return 0;
9423 }
9424
9425 int bpf_map__lookup_elem(const struct bpf_map *map,
9426                          const void *key, size_t key_sz,
9427                          void *value, size_t value_sz, __u64 flags)
9428 {
9429         int err;
9430
9431         err = validate_map_op(map, key_sz, value_sz, true);
9432         if (err)
9433                 return libbpf_err(err);
9434
9435         return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
9436 }
9437
9438 int bpf_map__update_elem(const struct bpf_map *map,
9439                          const void *key, size_t key_sz,
9440                          const void *value, size_t value_sz, __u64 flags)
9441 {
9442         int err;
9443
9444         err = validate_map_op(map, key_sz, value_sz, true);
9445         if (err)
9446                 return libbpf_err(err);
9447
9448         return bpf_map_update_elem(map->fd, key, value, flags);
9449 }
9450
9451 int bpf_map__delete_elem(const struct bpf_map *map,
9452                          const void *key, size_t key_sz, __u64 flags)
9453 {
9454         int err;
9455
9456         err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
9457         if (err)
9458                 return libbpf_err(err);
9459
9460         return bpf_map_delete_elem_flags(map->fd, key, flags);
9461 }
9462
9463 int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,
9464                                     const void *key, size_t key_sz,
9465                                     void *value, size_t value_sz, __u64 flags)
9466 {
9467         int err;
9468
9469         err = validate_map_op(map, key_sz, value_sz, true);
9470         if (err)
9471                 return libbpf_err(err);
9472
9473         return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
9474 }
9475
9476 int bpf_map__get_next_key(const struct bpf_map *map,
9477                           const void *cur_key, void *next_key, size_t key_sz)
9478 {
9479         int err;
9480
9481         err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
9482         if (err)
9483                 return libbpf_err(err);
9484
9485         return bpf_map_get_next_key(map->fd, cur_key, next_key);
9486 }
9487
9488 long libbpf_get_error(const void *ptr)
9489 {
9490         if (!IS_ERR_OR_NULL(ptr))
9491                 return 0;
9492
9493         if (IS_ERR(ptr))
9494                 errno = -PTR_ERR(ptr);
9495
9496         /* If ptr == NULL, then errno should be already set by the failing
9497          * API, because libbpf never returns NULL on success and it now always
9498          * sets errno on error. So no extra errno handling for ptr == NULL
9499          * case.
9500          */
9501         return -errno;
9502 }
9503
9504 /* Replace link's underlying BPF program with the new one */
9505 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
9506 {
9507         int ret;
9508
9509         ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
9510         return libbpf_err_errno(ret);
9511 }
9512
9513 /* Release "ownership" of underlying BPF resource (typically, BPF program
9514  * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
9515  * link, when destructed through bpf_link__destroy() call won't attempt to
9516  * detach/unregisted that BPF resource. This is useful in situations where,
9517  * say, attached BPF program has to outlive userspace program that attached it
9518  * in the system. Depending on type of BPF program, though, there might be
9519  * additional steps (like pinning BPF program in BPF FS) necessary to ensure
9520  * exit of userspace program doesn't trigger automatic detachment and clean up
9521  * inside the kernel.
9522  */
9523 void bpf_link__disconnect(struct bpf_link *link)
9524 {
9525         link->disconnected = true;
9526 }
9527
9528 int bpf_link__destroy(struct bpf_link *link)
9529 {
9530         int err = 0;
9531
9532         if (IS_ERR_OR_NULL(link))
9533                 return 0;
9534
9535         if (!link->disconnected && link->detach)
9536                 err = link->detach(link);
9537         if (link->pin_path)
9538                 free(link->pin_path);
9539         if (link->dealloc)
9540                 link->dealloc(link);
9541         else
9542                 free(link);
9543
9544         return libbpf_err(err);
9545 }
9546
9547 int bpf_link__fd(const struct bpf_link *link)
9548 {
9549         return link->fd;
9550 }
9551
9552 const char *bpf_link__pin_path(const struct bpf_link *link)
9553 {
9554         return link->pin_path;
9555 }
9556
9557 static int bpf_link__detach_fd(struct bpf_link *link)
9558 {
9559         return libbpf_err_errno(close(link->fd));
9560 }
9561
9562 struct bpf_link *bpf_link__open(const char *path)
9563 {
9564         struct bpf_link *link;
9565         int fd;
9566
9567         fd = bpf_obj_get(path);
9568         if (fd < 0) {
9569                 fd = -errno;
9570                 pr_warn("failed to open link at %s: %d\n", path, fd);
9571                 return libbpf_err_ptr(fd);
9572         }
9573
9574         link = calloc(1, sizeof(*link));
9575         if (!link) {
9576                 close(fd);
9577                 return libbpf_err_ptr(-ENOMEM);
9578         }
9579         link->detach = &bpf_link__detach_fd;
9580         link->fd = fd;
9581
9582         link->pin_path = strdup(path);
9583         if (!link->pin_path) {
9584                 bpf_link__destroy(link);
9585                 return libbpf_err_ptr(-ENOMEM);
9586         }
9587
9588         return link;
9589 }
9590
9591 int bpf_link__detach(struct bpf_link *link)
9592 {
9593         return bpf_link_detach(link->fd) ? -errno : 0;
9594 }
9595
9596 int bpf_link__pin(struct bpf_link *link, const char *path)
9597 {
9598         int err;
9599
9600         if (link->pin_path)
9601                 return libbpf_err(-EBUSY);
9602         err = make_parent_dir(path);
9603         if (err)
9604                 return libbpf_err(err);
9605         err = check_path(path);
9606         if (err)
9607                 return libbpf_err(err);
9608
9609         link->pin_path = strdup(path);
9610         if (!link->pin_path)
9611                 return libbpf_err(-ENOMEM);
9612
9613         if (bpf_obj_pin(link->fd, link->pin_path)) {
9614                 err = -errno;
9615                 zfree(&link->pin_path);
9616                 return libbpf_err(err);
9617         }
9618
9619         pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
9620         return 0;
9621 }
9622
9623 int bpf_link__unpin(struct bpf_link *link)
9624 {
9625         int err;
9626
9627         if (!link->pin_path)
9628                 return libbpf_err(-EINVAL);
9629
9630         err = unlink(link->pin_path);
9631         if (err != 0)
9632                 return -errno;
9633
9634         pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
9635         zfree(&link->pin_path);
9636         return 0;
9637 }
9638
9639 struct bpf_link_perf {
9640         struct bpf_link link;
9641         int perf_event_fd;
9642         /* legacy kprobe support: keep track of probe identifier and type */
9643         char *legacy_probe_name;
9644         bool legacy_is_kprobe;
9645         bool legacy_is_retprobe;
9646 };
9647
9648 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe);
9649 static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe);
9650
9651 static int bpf_link_perf_detach(struct bpf_link *link)
9652 {
9653         struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9654         int err = 0;
9655
9656         if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
9657                 err = -errno;
9658
9659         if (perf_link->perf_event_fd != link->fd)
9660                 close(perf_link->perf_event_fd);
9661         close(link->fd);
9662
9663         /* legacy uprobe/kprobe needs to be removed after perf event fd closure */
9664         if (perf_link->legacy_probe_name) {
9665                 if (perf_link->legacy_is_kprobe) {
9666                         err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
9667                                                          perf_link->legacy_is_retprobe);
9668                 } else {
9669                         err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
9670                                                          perf_link->legacy_is_retprobe);
9671                 }
9672         }
9673
9674         return err;
9675 }
9676
9677 static void bpf_link_perf_dealloc(struct bpf_link *link)
9678 {
9679         struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9680
9681         free(perf_link->legacy_probe_name);
9682         free(perf_link);
9683 }
9684
9685 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
9686                                                      const struct bpf_perf_event_opts *opts)
9687 {
9688         char errmsg[STRERR_BUFSIZE];
9689         struct bpf_link_perf *link;
9690         int prog_fd, link_fd = -1, err;
9691
9692         if (!OPTS_VALID(opts, bpf_perf_event_opts))
9693                 return libbpf_err_ptr(-EINVAL);
9694
9695         if (pfd < 0) {
9696                 pr_warn("prog '%s': invalid perf event FD %d\n",
9697                         prog->name, pfd);
9698                 return libbpf_err_ptr(-EINVAL);
9699         }
9700         prog_fd = bpf_program__fd(prog);
9701         if (prog_fd < 0) {
9702                 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
9703                         prog->name);
9704                 return libbpf_err_ptr(-EINVAL);
9705         }
9706
9707         link = calloc(1, sizeof(*link));
9708         if (!link)
9709                 return libbpf_err_ptr(-ENOMEM);
9710         link->link.detach = &bpf_link_perf_detach;
9711         link->link.dealloc = &bpf_link_perf_dealloc;
9712         link->perf_event_fd = pfd;
9713
9714         if (kernel_supports(prog->obj, FEAT_PERF_LINK)) {
9715                 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts,
9716                         .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0));
9717
9718                 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);
9719                 if (link_fd < 0) {
9720                         err = -errno;
9721                         pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",
9722                                 prog->name, pfd,
9723                                 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9724                         goto err_out;
9725                 }
9726                 link->link.fd = link_fd;
9727         } else {
9728                 if (OPTS_GET(opts, bpf_cookie, 0)) {
9729                         pr_warn("prog '%s': user context value is not supported\n", prog->name);
9730                         err = -EOPNOTSUPP;
9731                         goto err_out;
9732                 }
9733
9734                 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
9735                         err = -errno;
9736                         pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
9737                                 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9738                         if (err == -EPROTO)
9739                                 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
9740                                         prog->name, pfd);
9741                         goto err_out;
9742                 }
9743                 link->link.fd = pfd;
9744         }
9745         if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9746                 err = -errno;
9747                 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
9748                         prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9749                 goto err_out;
9750         }
9751
9752         return &link->link;
9753 err_out:
9754         if (link_fd >= 0)
9755                 close(link_fd);
9756         free(link);
9757         return libbpf_err_ptr(err);
9758 }
9759
9760 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)
9761 {
9762         return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
9763 }
9764
9765 /*
9766  * this function is expected to parse integer in the range of [0, 2^31-1] from
9767  * given file using scanf format string fmt. If actual parsed value is
9768  * negative, the result might be indistinguishable from error
9769  */
9770 static int parse_uint_from_file(const char *file, const char *fmt)
9771 {
9772         char buf[STRERR_BUFSIZE];
9773         int err, ret;
9774         FILE *f;
9775
9776         f = fopen(file, "r");
9777         if (!f) {
9778                 err = -errno;
9779                 pr_debug("failed to open '%s': %s\n", file,
9780                          libbpf_strerror_r(err, buf, sizeof(buf)));
9781                 return err;
9782         }
9783         err = fscanf(f, fmt, &ret);
9784         if (err != 1) {
9785                 err = err == EOF ? -EIO : -errno;
9786                 pr_debug("failed to parse '%s': %s\n", file,
9787                         libbpf_strerror_r(err, buf, sizeof(buf)));
9788                 fclose(f);
9789                 return err;
9790         }
9791         fclose(f);
9792         return ret;
9793 }
9794
9795 static int determine_kprobe_perf_type(void)
9796 {
9797         const char *file = "/sys/bus/event_source/devices/kprobe/type";
9798
9799         return parse_uint_from_file(file, "%d\n");
9800 }
9801
9802 static int determine_uprobe_perf_type(void)
9803 {
9804         const char *file = "/sys/bus/event_source/devices/uprobe/type";
9805
9806         return parse_uint_from_file(file, "%d\n");
9807 }
9808
9809 static int determine_kprobe_retprobe_bit(void)
9810 {
9811         const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
9812
9813         return parse_uint_from_file(file, "config:%d\n");
9814 }
9815
9816 static int determine_uprobe_retprobe_bit(void)
9817 {
9818         const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
9819
9820         return parse_uint_from_file(file, "config:%d\n");
9821 }
9822
9823 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32
9824 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32
9825
9826 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
9827                                  uint64_t offset, int pid, size_t ref_ctr_off)
9828 {
9829         struct perf_event_attr attr = {};
9830         char errmsg[STRERR_BUFSIZE];
9831         int type, pfd;
9832
9833         if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
9834                 return -EINVAL;
9835
9836         type = uprobe ? determine_uprobe_perf_type()
9837                       : determine_kprobe_perf_type();
9838         if (type < 0) {
9839                 pr_warn("failed to determine %s perf type: %s\n",
9840                         uprobe ? "uprobe" : "kprobe",
9841                         libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9842                 return type;
9843         }
9844         if (retprobe) {
9845                 int bit = uprobe ? determine_uprobe_retprobe_bit()
9846                                  : determine_kprobe_retprobe_bit();
9847
9848                 if (bit < 0) {
9849                         pr_warn("failed to determine %s retprobe bit: %s\n",
9850                                 uprobe ? "uprobe" : "kprobe",
9851                                 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
9852                         return bit;
9853                 }
9854                 attr.config |= 1 << bit;
9855         }
9856         attr.size = sizeof(attr);
9857         attr.type = type;
9858         attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
9859         attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
9860         attr.config2 = offset;           /* kprobe_addr or probe_offset */
9861
9862         /* pid filter is meaningful only for uprobes */
9863         pfd = syscall(__NR_perf_event_open, &attr,
9864                       pid < 0 ? -1 : pid /* pid */,
9865                       pid == -1 ? 0 : -1 /* cpu */,
9866                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9867         return pfd >= 0 ? pfd : -errno;
9868 }
9869
9870 static int append_to_file(const char *file, const char *fmt, ...)
9871 {
9872         int fd, n, err = 0;
9873         va_list ap;
9874
9875         fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0);
9876         if (fd < 0)
9877                 return -errno;
9878
9879         va_start(ap, fmt);
9880         n = vdprintf(fd, fmt, ap);
9881         va_end(ap);
9882
9883         if (n < 0)
9884                 err = -errno;
9885
9886         close(fd);
9887         return err;
9888 }
9889
9890 #define DEBUGFS "/sys/kernel/debug/tracing"
9891 #define TRACEFS "/sys/kernel/tracing"
9892
9893 static bool use_debugfs(void)
9894 {
9895         static int has_debugfs = -1;
9896
9897         if (has_debugfs < 0)
9898                 has_debugfs = access(DEBUGFS, F_OK) == 0;
9899
9900         return has_debugfs == 1;
9901 }
9902
9903 static const char *tracefs_path(void)
9904 {
9905         return use_debugfs() ? DEBUGFS : TRACEFS;
9906 }
9907
9908 static const char *tracefs_kprobe_events(void)
9909 {
9910         return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events";
9911 }
9912
9913 static const char *tracefs_uprobe_events(void)
9914 {
9915         return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
9916 }
9917
9918 static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
9919                                          const char *kfunc_name, size_t offset)
9920 {
9921         static int index = 0;
9922
9923         snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset,
9924                  __sync_fetch_and_add(&index, 1));
9925 }
9926
9927 static int add_kprobe_event_legacy(const char *probe_name, bool retprobe,
9928                                    const char *kfunc_name, size_t offset)
9929 {
9930         return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx",
9931                               retprobe ? 'r' : 'p',
9932                               retprobe ? "kretprobes" : "kprobes",
9933                               probe_name, kfunc_name, offset);
9934 }
9935
9936 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe)
9937 {
9938         return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
9939                               retprobe ? "kretprobes" : "kprobes", probe_name);
9940 }
9941
9942 static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe)
9943 {
9944         char file[256];
9945
9946         snprintf(file, sizeof(file), "%s/events/%s/%s/id",
9947                  tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name);
9948
9949         return parse_uint_from_file(file, "%d\n");
9950 }
9951
9952 static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
9953                                          const char *kfunc_name, size_t offset, int pid)
9954 {
9955         struct perf_event_attr attr = {};
9956         char errmsg[STRERR_BUFSIZE];
9957         int type, pfd, err;
9958
9959         err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);
9960         if (err < 0) {
9961                 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",
9962                         kfunc_name, offset,
9963                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9964                 return err;
9965         }
9966         type = determine_kprobe_perf_type_legacy(probe_name, retprobe);
9967         if (type < 0) {
9968                 err = type;
9969                 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",
9970                         kfunc_name, offset,
9971                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9972                 goto err_clean_legacy;
9973         }
9974         attr.size = sizeof(attr);
9975         attr.config = type;
9976         attr.type = PERF_TYPE_TRACEPOINT;
9977
9978         pfd = syscall(__NR_perf_event_open, &attr,
9979                       pid < 0 ? -1 : pid, /* pid */
9980                       pid == -1 ? 0 : -1, /* cpu */
9981                       -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);
9982         if (pfd < 0) {
9983                 err = -errno;
9984                 pr_warn("legacy kprobe perf_event_open() failed: %s\n",
9985                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9986                 goto err_clean_legacy;
9987         }
9988         return pfd;
9989
9990 err_clean_legacy:
9991         /* Clear the newly added legacy kprobe_event */
9992         remove_kprobe_event_legacy(probe_name, retprobe);
9993         return err;
9994 }
9995
9996 static const char *arch_specific_syscall_pfx(void)
9997 {
9998 #if defined(__x86_64__)
9999         return "x64";
10000 #elif defined(__i386__)
10001         return "ia32";
10002 #elif defined(__s390x__)
10003         return "s390x";
10004 #elif defined(__s390__)
10005         return "s390";
10006 #elif defined(__arm__)
10007         return "arm";
10008 #elif defined(__aarch64__)
10009         return "arm64";
10010 #elif defined(__mips__)
10011         return "mips";
10012 #elif defined(__riscv)
10013         return "riscv";
10014 #elif defined(__powerpc__)
10015         return "powerpc";
10016 #elif defined(__powerpc64__)
10017         return "powerpc64";
10018 #else
10019         return NULL;
10020 #endif
10021 }
10022
10023 static int probe_kern_syscall_wrapper(void)
10024 {
10025         char syscall_name[64];
10026         const char *ksys_pfx;
10027
10028         ksys_pfx = arch_specific_syscall_pfx();
10029         if (!ksys_pfx)
10030                 return 0;
10031
10032         snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx);
10033
10034         if (determine_kprobe_perf_type() >= 0) {
10035                 int pfd;
10036
10037                 pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0);
10038                 if (pfd >= 0)
10039                         close(pfd);
10040
10041                 return pfd >= 0 ? 1 : 0;
10042         } else { /* legacy mode */
10043                 char probe_name[128];
10044
10045                 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);
10046                 if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0)
10047                         return 0;
10048
10049                 (void)remove_kprobe_event_legacy(probe_name, false);
10050                 return 1;
10051         }
10052 }
10053
10054 struct bpf_link *
10055 bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
10056                                 const char *func_name,
10057                                 const struct bpf_kprobe_opts *opts)
10058 {
10059         DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
10060         char errmsg[STRERR_BUFSIZE];
10061         char *legacy_probe = NULL;
10062         struct bpf_link *link;
10063         size_t offset;
10064         bool retprobe, legacy;
10065         int pfd, err;
10066
10067         if (!OPTS_VALID(opts, bpf_kprobe_opts))
10068                 return libbpf_err_ptr(-EINVAL);
10069
10070         retprobe = OPTS_GET(opts, retprobe, false);
10071         offset = OPTS_GET(opts, offset, 0);
10072         pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10073
10074         legacy = determine_kprobe_perf_type() < 0;
10075         if (!legacy) {
10076                 pfd = perf_event_open_probe(false /* uprobe */, retprobe,
10077                                             func_name, offset,
10078                                             -1 /* pid */, 0 /* ref_ctr_off */);
10079         } else {
10080                 char probe_name[256];
10081
10082                 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name),
10083                                              func_name, offset);
10084
10085                 legacy_probe = strdup(probe_name);
10086                 if (!legacy_probe)
10087                         return libbpf_err_ptr(-ENOMEM);
10088
10089                 pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name,
10090                                                     offset, -1 /* pid */);
10091         }
10092         if (pfd < 0) {
10093                 err = -errno;
10094                 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
10095                         prog->name, retprobe ? "kretprobe" : "kprobe",
10096                         func_name, offset,
10097                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10098                 goto err_out;
10099         }
10100         link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10101         err = libbpf_get_error(link);
10102         if (err) {
10103                 close(pfd);
10104                 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
10105                         prog->name, retprobe ? "kretprobe" : "kprobe",
10106                         func_name, offset,
10107                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10108                 goto err_clean_legacy;
10109         }
10110         if (legacy) {
10111                 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10112
10113                 perf_link->legacy_probe_name = legacy_probe;
10114                 perf_link->legacy_is_kprobe = true;
10115                 perf_link->legacy_is_retprobe = retprobe;
10116         }
10117
10118         return link;
10119
10120 err_clean_legacy:
10121         if (legacy)
10122                 remove_kprobe_event_legacy(legacy_probe, retprobe);
10123 err_out:
10124         free(legacy_probe);
10125         return libbpf_err_ptr(err);
10126 }
10127
10128 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
10129                                             bool retprobe,
10130                                             const char *func_name)
10131 {
10132         DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts,
10133                 .retprobe = retprobe,
10134         );
10135
10136         return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
10137 }
10138
10139 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
10140                                               const char *syscall_name,
10141                                               const struct bpf_ksyscall_opts *opts)
10142 {
10143         LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
10144         char func_name[128];
10145
10146         if (!OPTS_VALID(opts, bpf_ksyscall_opts))
10147                 return libbpf_err_ptr(-EINVAL);
10148
10149         if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
10150                 /* arch_specific_syscall_pfx() should never return NULL here
10151                  * because it is guarded by kernel_supports(). However, since
10152                  * compiler does not know that we have an explicit conditional
10153                  * as well.
10154                  */
10155                 snprintf(func_name, sizeof(func_name), "__%s_sys_%s",
10156                          arch_specific_syscall_pfx() ? : "", syscall_name);
10157         } else {
10158                 snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name);
10159         }
10160
10161         kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false);
10162         kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10163
10164         return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
10165 }
10166
10167 /* Adapted from perf/util/string.c */
10168 static bool glob_match(const char *str, const char *pat)
10169 {
10170         while (*str && *pat && *pat != '*') {
10171                 if (*pat == '?') {      /* Matches any single character */
10172                         str++;
10173                         pat++;
10174                         continue;
10175                 }
10176                 if (*str != *pat)
10177                         return false;
10178                 str++;
10179                 pat++;
10180         }
10181         /* Check wild card */
10182         if (*pat == '*') {
10183                 while (*pat == '*')
10184                         pat++;
10185                 if (!*pat) /* Tail wild card matches all */
10186                         return true;
10187                 while (*str)
10188                         if (glob_match(str++, pat))
10189                                 return true;
10190         }
10191         return !*str && !*pat;
10192 }
10193
10194 struct kprobe_multi_resolve {
10195         const char *pattern;
10196         unsigned long *addrs;
10197         size_t cap;
10198         size_t cnt;
10199 };
10200
10201 static int
10202 resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
10203                         const char *sym_name, void *ctx)
10204 {
10205         struct kprobe_multi_resolve *res = ctx;
10206         int err;
10207
10208         if (!glob_match(sym_name, res->pattern))
10209                 return 0;
10210
10211         err = libbpf_ensure_mem((void **) &res->addrs, &res->cap, sizeof(unsigned long),
10212                                 res->cnt + 1);
10213         if (err)
10214                 return err;
10215
10216         res->addrs[res->cnt++] = (unsigned long) sym_addr;
10217         return 0;
10218 }
10219
10220 struct bpf_link *
10221 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
10222                                       const char *pattern,
10223                                       const struct bpf_kprobe_multi_opts *opts)
10224 {
10225         LIBBPF_OPTS(bpf_link_create_opts, lopts);
10226         struct kprobe_multi_resolve res = {
10227                 .pattern = pattern,
10228         };
10229         struct bpf_link *link = NULL;
10230         char errmsg[STRERR_BUFSIZE];
10231         const unsigned long *addrs;
10232         int err, link_fd, prog_fd;
10233         const __u64 *cookies;
10234         const char **syms;
10235         bool retprobe;
10236         size_t cnt;
10237
10238         if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))
10239                 return libbpf_err_ptr(-EINVAL);
10240
10241         syms    = OPTS_GET(opts, syms, false);
10242         addrs   = OPTS_GET(opts, addrs, false);
10243         cnt     = OPTS_GET(opts, cnt, false);
10244         cookies = OPTS_GET(opts, cookies, false);
10245
10246         if (!pattern && !addrs && !syms)
10247                 return libbpf_err_ptr(-EINVAL);
10248         if (pattern && (addrs || syms || cookies || cnt))
10249                 return libbpf_err_ptr(-EINVAL);
10250         if (!pattern && !cnt)
10251                 return libbpf_err_ptr(-EINVAL);
10252         if (addrs && syms)
10253                 return libbpf_err_ptr(-EINVAL);
10254
10255         if (pattern) {
10256                 err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
10257                 if (err)
10258                         goto error;
10259                 if (!res.cnt) {
10260                         err = -ENOENT;
10261                         goto error;
10262                 }
10263                 addrs = res.addrs;
10264                 cnt = res.cnt;
10265         }
10266
10267         retprobe = OPTS_GET(opts, retprobe, false);
10268
10269         lopts.kprobe_multi.syms = syms;
10270         lopts.kprobe_multi.addrs = addrs;
10271         lopts.kprobe_multi.cookies = cookies;
10272         lopts.kprobe_multi.cnt = cnt;
10273         lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0;
10274
10275         link = calloc(1, sizeof(*link));
10276         if (!link) {
10277                 err = -ENOMEM;
10278                 goto error;
10279         }
10280         link->detach = &bpf_link__detach_fd;
10281
10282         prog_fd = bpf_program__fd(prog);
10283         link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts);
10284         if (link_fd < 0) {
10285                 err = -errno;
10286                 pr_warn("prog '%s': failed to attach: %s\n",
10287                         prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10288                 goto error;
10289         }
10290         link->fd = link_fd;
10291         free(res.addrs);
10292         return link;
10293
10294 error:
10295         free(link);
10296         free(res.addrs);
10297         return libbpf_err_ptr(err);
10298 }
10299
10300 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10301 {
10302         DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);
10303         unsigned long offset = 0;
10304         const char *func_name;
10305         char *func;
10306         int n;
10307
10308         *link = NULL;
10309
10310         /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
10311         if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
10312                 return 0;
10313
10314         opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
10315         if (opts.retprobe)
10316                 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
10317         else
10318                 func_name = prog->sec_name + sizeof("kprobe/") - 1;
10319
10320         n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
10321         if (n < 1) {
10322                 pr_warn("kprobe name is invalid: %s\n", func_name);
10323                 return -EINVAL;
10324         }
10325         if (opts.retprobe && offset != 0) {
10326                 free(func);
10327                 pr_warn("kretprobes do not support offset specification\n");
10328                 return -EINVAL;
10329         }
10330
10331         opts.offset = offset;
10332         *link = bpf_program__attach_kprobe_opts(prog, func, &opts);
10333         free(func);
10334         return libbpf_get_error(*link);
10335 }
10336
10337 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10338 {
10339         LIBBPF_OPTS(bpf_ksyscall_opts, opts);
10340         const char *syscall_name;
10341
10342         *link = NULL;
10343
10344         /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
10345         if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
10346                 return 0;
10347
10348         opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
10349         if (opts.retprobe)
10350                 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
10351         else
10352                 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
10353
10354         *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
10355         return *link ? 0 : -errno;
10356 }
10357
10358 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10359 {
10360         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
10361         const char *spec;
10362         char *pattern;
10363         int n;
10364
10365         *link = NULL;
10366
10367         /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
10368         if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
10369             strcmp(prog->sec_name, "kretprobe.multi") == 0)
10370                 return 0;
10371
10372         opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
10373         if (opts.retprobe)
10374                 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
10375         else
10376                 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
10377
10378         n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
10379         if (n < 1) {
10380                 pr_warn("kprobe multi pattern is invalid: %s\n", pattern);
10381                 return -EINVAL;
10382         }
10383
10384         *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
10385         free(pattern);
10386         return libbpf_get_error(*link);
10387 }
10388
10389 static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz,
10390                                          const char *binary_path, uint64_t offset)
10391 {
10392         int i;
10393
10394         snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset);
10395
10396         /* sanitize binary_path in the probe name */
10397         for (i = 0; buf[i]; i++) {
10398                 if (!isalnum(buf[i]))
10399                         buf[i] = '_';
10400         }
10401 }
10402
10403 static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,
10404                                           const char *binary_path, size_t offset)
10405 {
10406         return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx",
10407                               retprobe ? 'r' : 'p',
10408                               retprobe ? "uretprobes" : "uprobes",
10409                               probe_name, binary_path, offset);
10410 }
10411
10412 static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe)
10413 {
10414         return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
10415                               retprobe ? "uretprobes" : "uprobes", probe_name);
10416 }
10417
10418 static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe)
10419 {
10420         char file[512];
10421
10422         snprintf(file, sizeof(file), "%s/events/%s/%s/id",
10423                  tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name);
10424
10425         return parse_uint_from_file(file, "%d\n");
10426 }
10427
10428 static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
10429                                          const char *binary_path, size_t offset, int pid)
10430 {
10431         struct perf_event_attr attr;
10432         int type, pfd, err;
10433
10434         err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);
10435         if (err < 0) {
10436                 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n",
10437                         binary_path, (size_t)offset, err);
10438                 return err;
10439         }
10440         type = determine_uprobe_perf_type_legacy(probe_name, retprobe);
10441         if (type < 0) {
10442                 err = type;
10443                 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n",
10444                         binary_path, offset, err);
10445                 goto err_clean_legacy;
10446         }
10447
10448         memset(&attr, 0, sizeof(attr));
10449         attr.size = sizeof(attr);
10450         attr.config = type;
10451         attr.type = PERF_TYPE_TRACEPOINT;
10452
10453         pfd = syscall(__NR_perf_event_open, &attr,
10454                       pid < 0 ? -1 : pid, /* pid */
10455                       pid == -1 ? 0 : -1, /* cpu */
10456                       -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);
10457         if (pfd < 0) {
10458                 err = -errno;
10459                 pr_warn("legacy uprobe perf_event_open() failed: %d\n", err);
10460                 goto err_clean_legacy;
10461         }
10462         return pfd;
10463
10464 err_clean_legacy:
10465         /* Clear the newly added legacy uprobe_event */
10466         remove_uprobe_event_legacy(probe_name, retprobe);
10467         return err;
10468 }
10469
10470 /* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */
10471 static Elf_Scn *elf_find_next_scn_by_type(Elf *elf, int sh_type, Elf_Scn *scn)
10472 {
10473         while ((scn = elf_nextscn(elf, scn)) != NULL) {
10474                 GElf_Shdr sh;
10475
10476                 if (!gelf_getshdr(scn, &sh))
10477                         continue;
10478                 if (sh.sh_type == sh_type)
10479                         return scn;
10480         }
10481         return NULL;
10482 }
10483
10484 /* Find offset of function name in object specified by path.  "name" matches
10485  * symbol name or name@@LIB for library functions.
10486  */
10487 static long elf_find_func_offset(const char *binary_path, const char *name)
10488 {
10489         int fd, i, sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB };
10490         bool is_shared_lib, is_name_qualified;
10491         char errmsg[STRERR_BUFSIZE];
10492         long ret = -ENOENT;
10493         size_t name_len;
10494         GElf_Ehdr ehdr;
10495         Elf *elf;
10496
10497         fd = open(binary_path, O_RDONLY | O_CLOEXEC);
10498         if (fd < 0) {
10499                 ret = -errno;
10500                 pr_warn("failed to open %s: %s\n", binary_path,
10501                         libbpf_strerror_r(ret, errmsg, sizeof(errmsg)));
10502                 return ret;
10503         }
10504         elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
10505         if (!elf) {
10506                 pr_warn("elf: could not read elf from %s: %s\n", binary_path, elf_errmsg(-1));
10507                 close(fd);
10508                 return -LIBBPF_ERRNO__FORMAT;
10509         }
10510         if (!gelf_getehdr(elf, &ehdr)) {
10511                 pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1));
10512                 ret = -LIBBPF_ERRNO__FORMAT;
10513                 goto out;
10514         }
10515         /* for shared lib case, we do not need to calculate relative offset */
10516         is_shared_lib = ehdr.e_type == ET_DYN;
10517
10518         name_len = strlen(name);
10519         /* Does name specify "@@LIB"? */
10520         is_name_qualified = strstr(name, "@@") != NULL;
10521
10522         /* Search SHT_DYNSYM, SHT_SYMTAB for symbol.  This search order is used because if
10523          * a binary is stripped, it may only have SHT_DYNSYM, and a fully-statically
10524          * linked binary may not have SHT_DYMSYM, so absence of a section should not be
10525          * reported as a warning/error.
10526          */
10527         for (i = 0; i < ARRAY_SIZE(sh_types); i++) {
10528                 size_t nr_syms, strtabidx, idx;
10529                 Elf_Data *symbols = NULL;
10530                 Elf_Scn *scn = NULL;
10531                 int last_bind = -1;
10532                 const char *sname;
10533                 GElf_Shdr sh;
10534
10535                 scn = elf_find_next_scn_by_type(elf, sh_types[i], NULL);
10536                 if (!scn) {
10537                         pr_debug("elf: failed to find symbol table ELF sections in '%s'\n",
10538                                  binary_path);
10539                         continue;
10540                 }
10541                 if (!gelf_getshdr(scn, &sh))
10542                         continue;
10543                 strtabidx = sh.sh_link;
10544                 symbols = elf_getdata(scn, 0);
10545                 if (!symbols) {
10546                         pr_warn("elf: failed to get symbols for symtab section in '%s': %s\n",
10547                                 binary_path, elf_errmsg(-1));
10548                         ret = -LIBBPF_ERRNO__FORMAT;
10549                         goto out;
10550                 }
10551                 nr_syms = symbols->d_size / sh.sh_entsize;
10552
10553                 for (idx = 0; idx < nr_syms; idx++) {
10554                         int curr_bind;
10555                         GElf_Sym sym;
10556                         Elf_Scn *sym_scn;
10557                         GElf_Shdr sym_sh;
10558
10559                         if (!gelf_getsym(symbols, idx, &sym))
10560                                 continue;
10561
10562                         if (GELF_ST_TYPE(sym.st_info) != STT_FUNC)
10563                                 continue;
10564
10565                         sname = elf_strptr(elf, strtabidx, sym.st_name);
10566                         if (!sname)
10567                                 continue;
10568
10569                         curr_bind = GELF_ST_BIND(sym.st_info);
10570
10571                         /* User can specify func, func@@LIB or func@@LIB_VERSION. */
10572                         if (strncmp(sname, name, name_len) != 0)
10573                                 continue;
10574                         /* ...but we don't want a search for "foo" to match 'foo2" also, so any
10575                          * additional characters in sname should be of the form "@@LIB".
10576                          */
10577                         if (!is_name_qualified && sname[name_len] != '\0' && sname[name_len] != '@')
10578                                 continue;
10579
10580                         if (ret >= 0) {
10581                                 /* handle multiple matches */
10582                                 if (last_bind != STB_WEAK && curr_bind != STB_WEAK) {
10583                                         /* Only accept one non-weak bind. */
10584                                         pr_warn("elf: ambiguous match for '%s', '%s' in '%s'\n",
10585                                                 sname, name, binary_path);
10586                                         ret = -LIBBPF_ERRNO__FORMAT;
10587                                         goto out;
10588                                 } else if (curr_bind == STB_WEAK) {
10589                                         /* already have a non-weak bind, and
10590                                          * this is a weak bind, so ignore.
10591                                          */
10592                                         continue;
10593                                 }
10594                         }
10595
10596                         /* Transform symbol's virtual address (absolute for
10597                          * binaries and relative for shared libs) into file
10598                          * offset, which is what kernel is expecting for
10599                          * uprobe/uretprobe attachment.
10600                          * See Documentation/trace/uprobetracer.rst for more
10601                          * details.
10602                          * This is done by looking up symbol's containing
10603                          * section's header and using it's virtual address
10604                          * (sh_addr) and corresponding file offset (sh_offset)
10605                          * to transform sym.st_value (virtual address) into
10606                          * desired final file offset.
10607                          */
10608                         sym_scn = elf_getscn(elf, sym.st_shndx);
10609                         if (!sym_scn)
10610                                 continue;
10611                         if (!gelf_getshdr(sym_scn, &sym_sh))
10612                                 continue;
10613
10614                         ret = sym.st_value - sym_sh.sh_addr + sym_sh.sh_offset;
10615                         last_bind = curr_bind;
10616                 }
10617                 if (ret > 0)
10618                         break;
10619         }
10620
10621         if (ret > 0) {
10622                 pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path,
10623                          ret);
10624         } else {
10625                 if (ret == 0) {
10626                         pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path,
10627                                 is_shared_lib ? "should not be 0 in a shared library" :
10628                                                 "try using shared library path instead");
10629                         ret = -ENOENT;
10630                 } else {
10631                         pr_warn("elf: failed to find symbol '%s' in '%s'\n", name, binary_path);
10632                 }
10633         }
10634 out:
10635         elf_end(elf);
10636         close(fd);
10637         return ret;
10638 }
10639
10640 static const char *arch_specific_lib_paths(void)
10641 {
10642         /*
10643          * Based on https://packages.debian.org/sid/libc6.
10644          *
10645          * Assume that the traced program is built for the same architecture
10646          * as libbpf, which should cover the vast majority of cases.
10647          */
10648 #if defined(__x86_64__)
10649         return "/lib/x86_64-linux-gnu";
10650 #elif defined(__i386__)
10651         return "/lib/i386-linux-gnu";
10652 #elif defined(__s390x__)
10653         return "/lib/s390x-linux-gnu";
10654 #elif defined(__s390__)
10655         return "/lib/s390-linux-gnu";
10656 #elif defined(__arm__) && defined(__SOFTFP__)
10657         return "/lib/arm-linux-gnueabi";
10658 #elif defined(__arm__) && !defined(__SOFTFP__)
10659         return "/lib/arm-linux-gnueabihf";
10660 #elif defined(__aarch64__)
10661         return "/lib/aarch64-linux-gnu";
10662 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64
10663         return "/lib/mips64el-linux-gnuabi64";
10664 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32
10665         return "/lib/mipsel-linux-gnu";
10666 #elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
10667         return "/lib/powerpc64le-linux-gnu";
10668 #elif defined(__sparc__) && defined(__arch64__)
10669         return "/lib/sparc64-linux-gnu";
10670 #elif defined(__riscv) && __riscv_xlen == 64
10671         return "/lib/riscv64-linux-gnu";
10672 #else
10673         return NULL;
10674 #endif
10675 }
10676
10677 /* Get full path to program/shared library. */
10678 static int resolve_full_path(const char *file, char *result, size_t result_sz)
10679 {
10680         const char *search_paths[3] = {};
10681         int i, perm;
10682
10683         if (str_has_sfx(file, ".so") || strstr(file, ".so.")) {
10684                 search_paths[0] = getenv("LD_LIBRARY_PATH");
10685                 search_paths[1] = "/usr/lib64:/usr/lib";
10686                 search_paths[2] = arch_specific_lib_paths();
10687                 perm = R_OK;
10688         } else {
10689                 search_paths[0] = getenv("PATH");
10690                 search_paths[1] = "/usr/bin:/usr/sbin";
10691                 perm = R_OK | X_OK;
10692         }
10693
10694         for (i = 0; i < ARRAY_SIZE(search_paths); i++) {
10695                 const char *s;
10696
10697                 if (!search_paths[i])
10698                         continue;
10699                 for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) {
10700                         char *next_path;
10701                         int seg_len;
10702
10703                         if (s[0] == ':')
10704                                 s++;
10705                         next_path = strchr(s, ':');
10706                         seg_len = next_path ? next_path - s : strlen(s);
10707                         if (!seg_len)
10708                                 continue;
10709                         snprintf(result, result_sz, "%.*s/%s", seg_len, s, file);
10710                         /* ensure it has required permissions */
10711                         if (access(result, perm) < 0)
10712                                 continue;
10713                         pr_debug("resolved '%s' to '%s'\n", file, result);
10714                         return 0;
10715                 }
10716         }
10717         return -ENOENT;
10718 }
10719
10720 LIBBPF_API struct bpf_link *
10721 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
10722                                 const char *binary_path, size_t func_offset,
10723                                 const struct bpf_uprobe_opts *opts)
10724 {
10725         DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
10726         char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL;
10727         char full_binary_path[PATH_MAX];
10728         struct bpf_link *link;
10729         size_t ref_ctr_off;
10730         int pfd, err;
10731         bool retprobe, legacy;
10732         const char *func_name;
10733
10734         if (!OPTS_VALID(opts, bpf_uprobe_opts))
10735                 return libbpf_err_ptr(-EINVAL);
10736
10737         retprobe = OPTS_GET(opts, retprobe, false);
10738         ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0);
10739         pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10740
10741         if (!binary_path)
10742                 return libbpf_err_ptr(-EINVAL);
10743
10744         if (!strchr(binary_path, '/')) {
10745                 err = resolve_full_path(binary_path, full_binary_path,
10746                                         sizeof(full_binary_path));
10747                 if (err) {
10748                         pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
10749                                 prog->name, binary_path, err);
10750                         return libbpf_err_ptr(err);
10751                 }
10752                 binary_path = full_binary_path;
10753         }
10754         func_name = OPTS_GET(opts, func_name, NULL);
10755         if (func_name) {
10756                 long sym_off;
10757
10758                 sym_off = elf_find_func_offset(binary_path, func_name);
10759                 if (sym_off < 0)
10760                         return libbpf_err_ptr(sym_off);
10761                 func_offset += sym_off;
10762         }
10763
10764         legacy = determine_uprobe_perf_type() < 0;
10765         if (!legacy) {
10766                 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path,
10767                                             func_offset, pid, ref_ctr_off);
10768         } else {
10769                 char probe_name[PATH_MAX + 64];
10770
10771                 if (ref_ctr_off)
10772                         return libbpf_err_ptr(-EINVAL);
10773
10774                 gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name),
10775                                              binary_path, func_offset);
10776
10777                 legacy_probe = strdup(probe_name);
10778                 if (!legacy_probe)
10779                         return libbpf_err_ptr(-ENOMEM);
10780
10781                 pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe,
10782                                                     binary_path, func_offset, pid);
10783         }
10784         if (pfd < 0) {
10785                 err = -errno;
10786                 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
10787                         prog->name, retprobe ? "uretprobe" : "uprobe",
10788                         binary_path, func_offset,
10789                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10790                 goto err_out;
10791         }
10792
10793         link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10794         err = libbpf_get_error(link);
10795         if (err) {
10796                 close(pfd);
10797                 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
10798                         prog->name, retprobe ? "uretprobe" : "uprobe",
10799                         binary_path, func_offset,
10800                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10801                 goto err_clean_legacy;
10802         }
10803         if (legacy) {
10804                 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10805
10806                 perf_link->legacy_probe_name = legacy_probe;
10807                 perf_link->legacy_is_kprobe = false;
10808                 perf_link->legacy_is_retprobe = retprobe;
10809         }
10810         return link;
10811
10812 err_clean_legacy:
10813         if (legacy)
10814                 remove_uprobe_event_legacy(legacy_probe, retprobe);
10815 err_out:
10816         free(legacy_probe);
10817         return libbpf_err_ptr(err);
10818 }
10819
10820 /* Format of u[ret]probe section definition supporting auto-attach:
10821  * u[ret]probe/binary:function[+offset]
10822  *
10823  * binary can be an absolute/relative path or a filename; the latter is resolved to a
10824  * full binary path via bpf_program__attach_uprobe_opts.
10825  *
10826  * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be
10827  * specified (and auto-attach is not possible) or the above format is specified for
10828  * auto-attach.
10829  */
10830 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10831 {
10832         DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts);
10833         char *probe_type = NULL, *binary_path = NULL, *func_name = NULL;
10834         int n, ret = -EINVAL;
10835         long offset = 0;
10836
10837         *link = NULL;
10838
10839         n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[a-zA-Z0-9_.]+%li",
10840                    &probe_type, &binary_path, &func_name, &offset);
10841         switch (n) {
10842         case 1:
10843                 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
10844                 ret = 0;
10845                 break;
10846         case 2:
10847                 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",
10848                         prog->name, prog->sec_name);
10849                 break;
10850         case 3:
10851         case 4:
10852                 opts.retprobe = strcmp(probe_type, "uretprobe") == 0 ||
10853                                 strcmp(probe_type, "uretprobe.s") == 0;
10854                 if (opts.retprobe && offset != 0) {
10855                         pr_warn("prog '%s': uretprobes do not support offset specification\n",
10856                                 prog->name);
10857                         break;
10858                 }
10859                 opts.func_name = func_name;
10860                 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
10861                 ret = libbpf_get_error(*link);
10862                 break;
10863         default:
10864                 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
10865                         prog->sec_name);
10866                 break;
10867         }
10868         free(probe_type);
10869         free(binary_path);
10870         free(func_name);
10871
10872         return ret;
10873 }
10874
10875 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,
10876                                             bool retprobe, pid_t pid,
10877                                             const char *binary_path,
10878                                             size_t func_offset)
10879 {
10880         DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe);
10881
10882         return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
10883 }
10884
10885 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
10886                                           pid_t pid, const char *binary_path,
10887                                           const char *usdt_provider, const char *usdt_name,
10888                                           const struct bpf_usdt_opts *opts)
10889 {
10890         char resolved_path[512];
10891         struct bpf_object *obj = prog->obj;
10892         struct bpf_link *link;
10893         __u64 usdt_cookie;
10894         int err;
10895
10896         if (!OPTS_VALID(opts, bpf_uprobe_opts))
10897                 return libbpf_err_ptr(-EINVAL);
10898
10899         if (bpf_program__fd(prog) < 0) {
10900                 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
10901                         prog->name);
10902                 return libbpf_err_ptr(-EINVAL);
10903         }
10904
10905         if (!binary_path)
10906                 return libbpf_err_ptr(-EINVAL);
10907
10908         if (!strchr(binary_path, '/')) {
10909                 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));
10910                 if (err) {
10911                         pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
10912                                 prog->name, binary_path, err);
10913                         return libbpf_err_ptr(err);
10914                 }
10915                 binary_path = resolved_path;
10916         }
10917
10918         /* USDT manager is instantiated lazily on first USDT attach. It will
10919          * be destroyed together with BPF object in bpf_object__close().
10920          */
10921         if (IS_ERR(obj->usdt_man))
10922                 return libbpf_ptr(obj->usdt_man);
10923         if (!obj->usdt_man) {
10924                 obj->usdt_man = usdt_manager_new(obj);
10925                 if (IS_ERR(obj->usdt_man))
10926                         return libbpf_ptr(obj->usdt_man);
10927         }
10928
10929         usdt_cookie = OPTS_GET(opts, usdt_cookie, 0);
10930         link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
10931                                         usdt_provider, usdt_name, usdt_cookie);
10932         err = libbpf_get_error(link);
10933         if (err)
10934                 return libbpf_err_ptr(err);
10935         return link;
10936 }
10937
10938 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10939 {
10940         char *path = NULL, *provider = NULL, *name = NULL;
10941         const char *sec_name;
10942         int n, err;
10943
10944         sec_name = bpf_program__section_name(prog);
10945         if (strcmp(sec_name, "usdt") == 0) {
10946                 /* no auto-attach for just SEC("usdt") */
10947                 *link = NULL;
10948                 return 0;
10949         }
10950
10951         n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name);
10952         if (n != 3) {
10953                 pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n",
10954                         sec_name);
10955                 err = -EINVAL;
10956         } else {
10957                 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
10958                                                  provider, name, NULL);
10959                 err = libbpf_get_error(*link);
10960         }
10961         free(path);
10962         free(provider);
10963         free(name);
10964         return err;
10965 }
10966
10967 static int determine_tracepoint_id(const char *tp_category,
10968                                    const char *tp_name)
10969 {
10970         char file[PATH_MAX];
10971         int ret;
10972
10973         ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id",
10974                        tracefs_path(), tp_category, tp_name);
10975         if (ret < 0)
10976                 return -errno;
10977         if (ret >= sizeof(file)) {
10978                 pr_debug("tracepoint %s/%s path is too long\n",
10979                          tp_category, tp_name);
10980                 return -E2BIG;
10981         }
10982         return parse_uint_from_file(file, "%d\n");
10983 }
10984
10985 static int perf_event_open_tracepoint(const char *tp_category,
10986                                       const char *tp_name)
10987 {
10988         struct perf_event_attr attr = {};
10989         char errmsg[STRERR_BUFSIZE];
10990         int tp_id, pfd, err;
10991
10992         tp_id = determine_tracepoint_id(tp_category, tp_name);
10993         if (tp_id < 0) {
10994                 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
10995                         tp_category, tp_name,
10996                         libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
10997                 return tp_id;
10998         }
10999
11000         attr.type = PERF_TYPE_TRACEPOINT;
11001         attr.size = sizeof(attr);
11002         attr.config = tp_id;
11003
11004         pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
11005                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11006         if (pfd < 0) {
11007                 err = -errno;
11008                 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
11009                         tp_category, tp_name,
11010                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11011                 return err;
11012         }
11013         return pfd;
11014 }
11015
11016 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
11017                                                      const char *tp_category,
11018                                                      const char *tp_name,
11019                                                      const struct bpf_tracepoint_opts *opts)
11020 {
11021         DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
11022         char errmsg[STRERR_BUFSIZE];
11023         struct bpf_link *link;
11024         int pfd, err;
11025
11026         if (!OPTS_VALID(opts, bpf_tracepoint_opts))
11027                 return libbpf_err_ptr(-EINVAL);
11028
11029         pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
11030
11031         pfd = perf_event_open_tracepoint(tp_category, tp_name);
11032         if (pfd < 0) {
11033                 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
11034                         prog->name, tp_category, tp_name,
11035                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11036                 return libbpf_err_ptr(pfd);
11037         }
11038         link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11039         err = libbpf_get_error(link);
11040         if (err) {
11041                 close(pfd);
11042                 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
11043                         prog->name, tp_category, tp_name,
11044                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11045                 return libbpf_err_ptr(err);
11046         }
11047         return link;
11048 }
11049
11050 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,
11051                                                 const char *tp_category,
11052                                                 const char *tp_name)
11053 {
11054         return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
11055 }
11056
11057 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11058 {
11059         char *sec_name, *tp_cat, *tp_name;
11060
11061         *link = NULL;
11062
11063         /* no auto-attach for SEC("tp") or SEC("tracepoint") */
11064         if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
11065                 return 0;
11066
11067         sec_name = strdup(prog->sec_name);
11068         if (!sec_name)
11069                 return -ENOMEM;
11070
11071         /* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */
11072         if (str_has_pfx(prog->sec_name, "tp/"))
11073                 tp_cat = sec_name + sizeof("tp/") - 1;
11074         else
11075                 tp_cat = sec_name + sizeof("tracepoint/") - 1;
11076         tp_name = strchr(tp_cat, '/');
11077         if (!tp_name) {
11078                 free(sec_name);
11079                 return -EINVAL;
11080         }
11081         *tp_name = '\0';
11082         tp_name++;
11083
11084         *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
11085         free(sec_name);
11086         return libbpf_get_error(*link);
11087 }
11088
11089 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
11090                                                     const char *tp_name)
11091 {
11092         char errmsg[STRERR_BUFSIZE];
11093         struct bpf_link *link;
11094         int prog_fd, pfd;
11095
11096         prog_fd = bpf_program__fd(prog);
11097         if (prog_fd < 0) {
11098                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11099                 return libbpf_err_ptr(-EINVAL);
11100         }
11101
11102         link = calloc(1, sizeof(*link));
11103         if (!link)
11104                 return libbpf_err_ptr(-ENOMEM);
11105         link->detach = &bpf_link__detach_fd;
11106
11107         pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
11108         if (pfd < 0) {
11109                 pfd = -errno;
11110                 free(link);
11111                 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
11112                         prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11113                 return libbpf_err_ptr(pfd);
11114         }
11115         link->fd = pfd;
11116         return link;
11117 }
11118
11119 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11120 {
11121         static const char *const prefixes[] = {
11122                 "raw_tp",
11123                 "raw_tracepoint",
11124                 "raw_tp.w",
11125                 "raw_tracepoint.w",
11126         };
11127         size_t i;
11128         const char *tp_name = NULL;
11129
11130         *link = NULL;
11131
11132         for (i = 0; i < ARRAY_SIZE(prefixes); i++) {
11133                 size_t pfx_len;
11134
11135                 if (!str_has_pfx(prog->sec_name, prefixes[i]))
11136                         continue;
11137
11138                 pfx_len = strlen(prefixes[i]);
11139                 /* no auto-attach case of, e.g., SEC("raw_tp") */
11140                 if (prog->sec_name[pfx_len] == '\0')
11141                         return 0;
11142
11143                 if (prog->sec_name[pfx_len] != '/')
11144                         continue;
11145
11146                 tp_name = prog->sec_name + pfx_len + 1;
11147                 break;
11148         }
11149
11150         if (!tp_name) {
11151                 pr_warn("prog '%s': invalid section name '%s'\n",
11152                         prog->name, prog->sec_name);
11153                 return -EINVAL;
11154         }
11155
11156         *link = bpf_program__attach_raw_tracepoint(prog, tp_name);
11157         return libbpf_get_error(link);
11158 }
11159
11160 /* Common logic for all BPF program types that attach to a btf_id */
11161 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog,
11162                                                    const struct bpf_trace_opts *opts)
11163 {
11164         LIBBPF_OPTS(bpf_link_create_opts, link_opts);
11165         char errmsg[STRERR_BUFSIZE];
11166         struct bpf_link *link;
11167         int prog_fd, pfd;
11168
11169         if (!OPTS_VALID(opts, bpf_trace_opts))
11170                 return libbpf_err_ptr(-EINVAL);
11171
11172         prog_fd = bpf_program__fd(prog);
11173         if (prog_fd < 0) {
11174                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11175                 return libbpf_err_ptr(-EINVAL);
11176         }
11177
11178         link = calloc(1, sizeof(*link));
11179         if (!link)
11180                 return libbpf_err_ptr(-ENOMEM);
11181         link->detach = &bpf_link__detach_fd;
11182
11183         /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */
11184         link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0);
11185         pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);
11186         if (pfd < 0) {
11187                 pfd = -errno;
11188                 free(link);
11189                 pr_warn("prog '%s': failed to attach: %s\n",
11190                         prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11191                 return libbpf_err_ptr(pfd);
11192         }
11193         link->fd = pfd;
11194         return link;
11195 }
11196
11197 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
11198 {
11199         return bpf_program__attach_btf_id(prog, NULL);
11200 }
11201
11202 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
11203                                                 const struct bpf_trace_opts *opts)
11204 {
11205         return bpf_program__attach_btf_id(prog, opts);
11206 }
11207
11208 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
11209 {
11210         return bpf_program__attach_btf_id(prog, NULL);
11211 }
11212
11213 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11214 {
11215         *link = bpf_program__attach_trace(prog);
11216         return libbpf_get_error(*link);
11217 }
11218
11219 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11220 {
11221         *link = bpf_program__attach_lsm(prog);
11222         return libbpf_get_error(*link);
11223 }
11224
11225 static struct bpf_link *
11226 bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id,
11227                        const char *target_name)
11228 {
11229         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
11230                             .target_btf_id = btf_id);
11231         enum bpf_attach_type attach_type;
11232         char errmsg[STRERR_BUFSIZE];
11233         struct bpf_link *link;
11234         int prog_fd, link_fd;
11235
11236         prog_fd = bpf_program__fd(prog);
11237         if (prog_fd < 0) {
11238                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11239                 return libbpf_err_ptr(-EINVAL);
11240         }
11241
11242         link = calloc(1, sizeof(*link));
11243         if (!link)
11244                 return libbpf_err_ptr(-ENOMEM);
11245         link->detach = &bpf_link__detach_fd;
11246
11247         attach_type = bpf_program__expected_attach_type(prog);
11248         link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
11249         if (link_fd < 0) {
11250                 link_fd = -errno;
11251                 free(link);
11252                 pr_warn("prog '%s': failed to attach to %s: %s\n",
11253                         prog->name, target_name,
11254                         libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11255                 return libbpf_err_ptr(link_fd);
11256         }
11257         link->fd = link_fd;
11258         return link;
11259 }
11260
11261 struct bpf_link *
11262 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
11263 {
11264         return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
11265 }
11266
11267 struct bpf_link *
11268 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
11269 {
11270         return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
11271 }
11272
11273 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
11274 {
11275         /* target_fd/target_ifindex use the same field in LINK_CREATE */
11276         return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
11277 }
11278
11279 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
11280                                               int target_fd,
11281                                               const char *attach_func_name)
11282 {
11283         int btf_id;
11284
11285         if (!!target_fd != !!attach_func_name) {
11286                 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
11287                         prog->name);
11288                 return libbpf_err_ptr(-EINVAL);
11289         }
11290
11291         if (prog->type != BPF_PROG_TYPE_EXT) {
11292                 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
11293                         prog->name);
11294                 return libbpf_err_ptr(-EINVAL);
11295         }
11296
11297         if (target_fd) {
11298                 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
11299                 if (btf_id < 0)
11300                         return libbpf_err_ptr(btf_id);
11301
11302                 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
11303         } else {
11304                 /* no target, so use raw_tracepoint_open for compatibility
11305                  * with old kernels
11306                  */
11307                 return bpf_program__attach_trace(prog);
11308         }
11309 }
11310
11311 struct bpf_link *
11312 bpf_program__attach_iter(const struct bpf_program *prog,
11313                          const struct bpf_iter_attach_opts *opts)
11314 {
11315         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
11316         char errmsg[STRERR_BUFSIZE];
11317         struct bpf_link *link;
11318         int prog_fd, link_fd;
11319         __u32 target_fd = 0;
11320
11321         if (!OPTS_VALID(opts, bpf_iter_attach_opts))
11322                 return libbpf_err_ptr(-EINVAL);
11323
11324         link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
11325         link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
11326
11327         prog_fd = bpf_program__fd(prog);
11328         if (prog_fd < 0) {
11329                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11330                 return libbpf_err_ptr(-EINVAL);
11331         }
11332
11333         link = calloc(1, sizeof(*link));
11334         if (!link)
11335                 return libbpf_err_ptr(-ENOMEM);
11336         link->detach = &bpf_link__detach_fd;
11337
11338         link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
11339                                   &link_create_opts);
11340         if (link_fd < 0) {
11341                 link_fd = -errno;
11342                 free(link);
11343                 pr_warn("prog '%s': failed to attach to iterator: %s\n",
11344                         prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11345                 return libbpf_err_ptr(link_fd);
11346         }
11347         link->fd = link_fd;
11348         return link;
11349 }
11350
11351 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11352 {
11353         *link = bpf_program__attach_iter(prog, NULL);
11354         return libbpf_get_error(*link);
11355 }
11356
11357 struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
11358 {
11359         struct bpf_link *link = NULL;
11360         int err;
11361
11362         if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
11363                 return libbpf_err_ptr(-EOPNOTSUPP);
11364
11365         err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
11366         if (err)
11367                 return libbpf_err_ptr(err);
11368
11369         /* When calling bpf_program__attach() explicitly, auto-attach support
11370          * is expected to work, so NULL returned link is considered an error.
11371          * This is different for skeleton's attach, see comment in
11372          * bpf_object__attach_skeleton().
11373          */
11374         if (!link)
11375                 return libbpf_err_ptr(-EOPNOTSUPP);
11376
11377         return link;
11378 }
11379
11380 static int bpf_link__detach_struct_ops(struct bpf_link *link)
11381 {
11382         __u32 zero = 0;
11383
11384         if (bpf_map_delete_elem(link->fd, &zero))
11385                 return -errno;
11386
11387         return 0;
11388 }
11389
11390 struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)
11391 {
11392         struct bpf_struct_ops *st_ops;
11393         struct bpf_link *link;
11394         __u32 i, zero = 0;
11395         int err;
11396
11397         if (!bpf_map__is_struct_ops(map) || map->fd == -1)
11398                 return libbpf_err_ptr(-EINVAL);
11399
11400         link = calloc(1, sizeof(*link));
11401         if (!link)
11402                 return libbpf_err_ptr(-EINVAL);
11403
11404         st_ops = map->st_ops;
11405         for (i = 0; i < btf_vlen(st_ops->type); i++) {
11406                 struct bpf_program *prog = st_ops->progs[i];
11407                 void *kern_data;
11408                 int prog_fd;
11409
11410                 if (!prog)
11411                         continue;
11412
11413                 prog_fd = bpf_program__fd(prog);
11414                 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
11415                 *(unsigned long *)kern_data = prog_fd;
11416         }
11417
11418         err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
11419         if (err) {
11420                 err = -errno;
11421                 free(link);
11422                 return libbpf_err_ptr(err);
11423         }
11424
11425         link->detach = bpf_link__detach_struct_ops;
11426         link->fd = map->fd;
11427
11428         return link;
11429 }
11430
11431 typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr,
11432                                                           void *private_data);
11433
11434 static enum bpf_perf_event_ret
11435 perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
11436                        void **copy_mem, size_t *copy_size,
11437                        bpf_perf_event_print_t fn, void *private_data)
11438 {
11439         struct perf_event_mmap_page *header = mmap_mem;
11440         __u64 data_head = ring_buffer_read_head(header);
11441         __u64 data_tail = header->data_tail;
11442         void *base = ((__u8 *)header) + page_size;
11443         int ret = LIBBPF_PERF_EVENT_CONT;
11444         struct perf_event_header *ehdr;
11445         size_t ehdr_size;
11446
11447         while (data_head != data_tail) {
11448                 ehdr = base + (data_tail & (mmap_size - 1));
11449                 ehdr_size = ehdr->size;
11450
11451                 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
11452                         void *copy_start = ehdr;
11453                         size_t len_first = base + mmap_size - copy_start;
11454                         size_t len_secnd = ehdr_size - len_first;
11455
11456                         if (*copy_size < ehdr_size) {
11457                                 free(*copy_mem);
11458                                 *copy_mem = malloc(ehdr_size);
11459                                 if (!*copy_mem) {
11460                                         *copy_size = 0;
11461                                         ret = LIBBPF_PERF_EVENT_ERROR;
11462                                         break;
11463                                 }
11464                                 *copy_size = ehdr_size;
11465                         }
11466
11467                         memcpy(*copy_mem, copy_start, len_first);
11468                         memcpy(*copy_mem + len_first, base, len_secnd);
11469                         ehdr = *copy_mem;
11470                 }
11471
11472                 ret = fn(ehdr, private_data);
11473                 data_tail += ehdr_size;
11474                 if (ret != LIBBPF_PERF_EVENT_CONT)
11475                         break;
11476         }
11477
11478         ring_buffer_write_tail(header, data_tail);
11479         return libbpf_err(ret);
11480 }
11481
11482 struct perf_buffer;
11483
11484 struct perf_buffer_params {
11485         struct perf_event_attr *attr;
11486         /* if event_cb is specified, it takes precendence */
11487         perf_buffer_event_fn event_cb;
11488         /* sample_cb and lost_cb are higher-level common-case callbacks */
11489         perf_buffer_sample_fn sample_cb;
11490         perf_buffer_lost_fn lost_cb;
11491         void *ctx;
11492         int cpu_cnt;
11493         int *cpus;
11494         int *map_keys;
11495 };
11496
11497 struct perf_cpu_buf {
11498         struct perf_buffer *pb;
11499         void *base; /* mmap()'ed memory */
11500         void *buf; /* for reconstructing segmented data */
11501         size_t buf_size;
11502         int fd;
11503         int cpu;
11504         int map_key;
11505 };
11506
11507 struct perf_buffer {
11508         perf_buffer_event_fn event_cb;
11509         perf_buffer_sample_fn sample_cb;
11510         perf_buffer_lost_fn lost_cb;
11511         void *ctx; /* passed into callbacks */
11512
11513         size_t page_size;
11514         size_t mmap_size;
11515         struct perf_cpu_buf **cpu_bufs;
11516         struct epoll_event *events;
11517         int cpu_cnt; /* number of allocated CPU buffers */
11518         int epoll_fd; /* perf event FD */
11519         int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
11520 };
11521
11522 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
11523                                       struct perf_cpu_buf *cpu_buf)
11524 {
11525         if (!cpu_buf)
11526                 return;
11527         if (cpu_buf->base &&
11528             munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
11529                 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
11530         if (cpu_buf->fd >= 0) {
11531                 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
11532                 close(cpu_buf->fd);
11533         }
11534         free(cpu_buf->buf);
11535         free(cpu_buf);
11536 }
11537
11538 void perf_buffer__free(struct perf_buffer *pb)
11539 {
11540         int i;
11541
11542         if (IS_ERR_OR_NULL(pb))
11543                 return;
11544         if (pb->cpu_bufs) {
11545                 for (i = 0; i < pb->cpu_cnt; i++) {
11546                         struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
11547
11548                         if (!cpu_buf)
11549                                 continue;
11550
11551                         bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
11552                         perf_buffer__free_cpu_buf(pb, cpu_buf);
11553                 }
11554                 free(pb->cpu_bufs);
11555         }
11556         if (pb->epoll_fd >= 0)
11557                 close(pb->epoll_fd);
11558         free(pb->events);
11559         free(pb);
11560 }
11561
11562 static struct perf_cpu_buf *
11563 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
11564                           int cpu, int map_key)
11565 {
11566         struct perf_cpu_buf *cpu_buf;
11567         char msg[STRERR_BUFSIZE];
11568         int err;
11569
11570         cpu_buf = calloc(1, sizeof(*cpu_buf));
11571         if (!cpu_buf)
11572                 return ERR_PTR(-ENOMEM);
11573
11574         cpu_buf->pb = pb;
11575         cpu_buf->cpu = cpu;
11576         cpu_buf->map_key = map_key;
11577
11578         cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
11579                               -1, PERF_FLAG_FD_CLOEXEC);
11580         if (cpu_buf->fd < 0) {
11581                 err = -errno;
11582                 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
11583                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11584                 goto error;
11585         }
11586
11587         cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
11588                              PROT_READ | PROT_WRITE, MAP_SHARED,
11589                              cpu_buf->fd, 0);
11590         if (cpu_buf->base == MAP_FAILED) {
11591                 cpu_buf->base = NULL;
11592                 err = -errno;
11593                 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
11594                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11595                 goto error;
11596         }
11597
11598         if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
11599                 err = -errno;
11600                 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
11601                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11602                 goto error;
11603         }
11604
11605         return cpu_buf;
11606
11607 error:
11608         perf_buffer__free_cpu_buf(pb, cpu_buf);
11609         return (struct perf_cpu_buf *)ERR_PTR(err);
11610 }
11611
11612 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
11613                                               struct perf_buffer_params *p);
11614
11615 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
11616                                      perf_buffer_sample_fn sample_cb,
11617                                      perf_buffer_lost_fn lost_cb,
11618                                      void *ctx,
11619                                      const struct perf_buffer_opts *opts)
11620 {
11621         struct perf_buffer_params p = {};
11622         struct perf_event_attr attr = {};
11623
11624         if (!OPTS_VALID(opts, perf_buffer_opts))
11625                 return libbpf_err_ptr(-EINVAL);
11626
11627         attr.config = PERF_COUNT_SW_BPF_OUTPUT;
11628         attr.type = PERF_TYPE_SOFTWARE;
11629         attr.sample_type = PERF_SAMPLE_RAW;
11630         attr.sample_period = 1;
11631         attr.wakeup_events = 1;
11632
11633         p.attr = &attr;
11634         p.sample_cb = sample_cb;
11635         p.lost_cb = lost_cb;
11636         p.ctx = ctx;
11637
11638         return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
11639 }
11640
11641 struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt,
11642                                          struct perf_event_attr *attr,
11643                                          perf_buffer_event_fn event_cb, void *ctx,
11644                                          const struct perf_buffer_raw_opts *opts)
11645 {
11646         struct perf_buffer_params p = {};
11647
11648         if (!attr)
11649                 return libbpf_err_ptr(-EINVAL);
11650
11651         if (!OPTS_VALID(opts, perf_buffer_raw_opts))
11652                 return libbpf_err_ptr(-EINVAL);
11653
11654         p.attr = attr;
11655         p.event_cb = event_cb;
11656         p.ctx = ctx;
11657         p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0);
11658         p.cpus = OPTS_GET(opts, cpus, NULL);
11659         p.map_keys = OPTS_GET(opts, map_keys, NULL);
11660
11661         return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
11662 }
11663
11664 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
11665                                               struct perf_buffer_params *p)
11666 {
11667         const char *online_cpus_file = "/sys/devices/system/cpu/online";
11668         struct bpf_map_info map;
11669         char msg[STRERR_BUFSIZE];
11670         struct perf_buffer *pb;
11671         bool *online = NULL;
11672         __u32 map_info_len;
11673         int err, i, j, n;
11674
11675         if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
11676                 pr_warn("page count should be power of two, but is %zu\n",
11677                         page_cnt);
11678                 return ERR_PTR(-EINVAL);
11679         }
11680
11681         /* best-effort sanity checks */
11682         memset(&map, 0, sizeof(map));
11683         map_info_len = sizeof(map);
11684         err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
11685         if (err) {
11686                 err = -errno;
11687                 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
11688                  * -EBADFD, -EFAULT, or -E2BIG on real error
11689                  */
11690                 if (err != -EINVAL) {
11691                         pr_warn("failed to get map info for map FD %d: %s\n",
11692                                 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
11693                         return ERR_PTR(err);
11694                 }
11695                 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
11696                          map_fd);
11697         } else {
11698                 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
11699                         pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
11700                                 map.name);
11701                         return ERR_PTR(-EINVAL);
11702                 }
11703         }
11704
11705         pb = calloc(1, sizeof(*pb));
11706         if (!pb)
11707                 return ERR_PTR(-ENOMEM);
11708
11709         pb->event_cb = p->event_cb;
11710         pb->sample_cb = p->sample_cb;
11711         pb->lost_cb = p->lost_cb;
11712         pb->ctx = p->ctx;
11713
11714         pb->page_size = getpagesize();
11715         pb->mmap_size = pb->page_size * page_cnt;
11716         pb->map_fd = map_fd;
11717
11718         pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
11719         if (pb->epoll_fd < 0) {
11720                 err = -errno;
11721                 pr_warn("failed to create epoll instance: %s\n",
11722                         libbpf_strerror_r(err, msg, sizeof(msg)));
11723                 goto error;
11724         }
11725
11726         if (p->cpu_cnt > 0) {
11727                 pb->cpu_cnt = p->cpu_cnt;
11728         } else {
11729                 pb->cpu_cnt = libbpf_num_possible_cpus();
11730                 if (pb->cpu_cnt < 0) {
11731                         err = pb->cpu_cnt;
11732                         goto error;
11733                 }
11734                 if (map.max_entries && map.max_entries < pb->cpu_cnt)
11735                         pb->cpu_cnt = map.max_entries;
11736         }
11737
11738         pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
11739         if (!pb->events) {
11740                 err = -ENOMEM;
11741                 pr_warn("failed to allocate events: out of memory\n");
11742                 goto error;
11743         }
11744         pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
11745         if (!pb->cpu_bufs) {
11746                 err = -ENOMEM;
11747                 pr_warn("failed to allocate buffers: out of memory\n");
11748                 goto error;
11749         }
11750
11751         err = parse_cpu_mask_file(online_cpus_file, &online, &n);
11752         if (err) {
11753                 pr_warn("failed to get online CPU mask: %d\n", err);
11754                 goto error;
11755         }
11756
11757         for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
11758                 struct perf_cpu_buf *cpu_buf;
11759                 int cpu, map_key;
11760
11761                 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
11762                 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
11763
11764                 /* in case user didn't explicitly requested particular CPUs to
11765                  * be attached to, skip offline/not present CPUs
11766                  */
11767                 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
11768                         continue;
11769
11770                 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
11771                 if (IS_ERR(cpu_buf)) {
11772                         err = PTR_ERR(cpu_buf);
11773                         goto error;
11774                 }
11775
11776                 pb->cpu_bufs[j] = cpu_buf;
11777
11778                 err = bpf_map_update_elem(pb->map_fd, &map_key,
11779                                           &cpu_buf->fd, 0);
11780                 if (err) {
11781                         err = -errno;
11782                         pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
11783                                 cpu, map_key, cpu_buf->fd,
11784                                 libbpf_strerror_r(err, msg, sizeof(msg)));
11785                         goto error;
11786                 }
11787
11788                 pb->events[j].events = EPOLLIN;
11789                 pb->events[j].data.ptr = cpu_buf;
11790                 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
11791                               &pb->events[j]) < 0) {
11792                         err = -errno;
11793                         pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
11794                                 cpu, cpu_buf->fd,
11795                                 libbpf_strerror_r(err, msg, sizeof(msg)));
11796                         goto error;
11797                 }
11798                 j++;
11799         }
11800         pb->cpu_cnt = j;
11801         free(online);
11802
11803         return pb;
11804
11805 error:
11806         free(online);
11807         if (pb)
11808                 perf_buffer__free(pb);
11809         return ERR_PTR(err);
11810 }
11811
11812 struct perf_sample_raw {
11813         struct perf_event_header header;
11814         uint32_t size;
11815         char data[];
11816 };
11817
11818 struct perf_sample_lost {
11819         struct perf_event_header header;
11820         uint64_t id;
11821         uint64_t lost;
11822         uint64_t sample_id;
11823 };
11824
11825 static enum bpf_perf_event_ret
11826 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
11827 {
11828         struct perf_cpu_buf *cpu_buf = ctx;
11829         struct perf_buffer *pb = cpu_buf->pb;
11830         void *data = e;
11831
11832         /* user wants full control over parsing perf event */
11833         if (pb->event_cb)
11834                 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
11835
11836         switch (e->type) {
11837         case PERF_RECORD_SAMPLE: {
11838                 struct perf_sample_raw *s = data;
11839
11840                 if (pb->sample_cb)
11841                         pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
11842                 break;
11843         }
11844         case PERF_RECORD_LOST: {
11845                 struct perf_sample_lost *s = data;
11846
11847                 if (pb->lost_cb)
11848                         pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
11849                 break;
11850         }
11851         default:
11852                 pr_warn("unknown perf sample type %d\n", e->type);
11853                 return LIBBPF_PERF_EVENT_ERROR;
11854         }
11855         return LIBBPF_PERF_EVENT_CONT;
11856 }
11857
11858 static int perf_buffer__process_records(struct perf_buffer *pb,
11859                                         struct perf_cpu_buf *cpu_buf)
11860 {
11861         enum bpf_perf_event_ret ret;
11862
11863         ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
11864                                      pb->page_size, &cpu_buf->buf,
11865                                      &cpu_buf->buf_size,
11866                                      perf_buffer__process_record, cpu_buf);
11867         if (ret != LIBBPF_PERF_EVENT_CONT)
11868                 return ret;
11869         return 0;
11870 }
11871
11872 int perf_buffer__epoll_fd(const struct perf_buffer *pb)
11873 {
11874         return pb->epoll_fd;
11875 }
11876
11877 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
11878 {
11879         int i, cnt, err;
11880
11881         cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
11882         if (cnt < 0)
11883                 return -errno;
11884
11885         for (i = 0; i < cnt; i++) {
11886                 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
11887
11888                 err = perf_buffer__process_records(pb, cpu_buf);
11889                 if (err) {
11890                         pr_warn("error while processing records: %d\n", err);
11891                         return libbpf_err(err);
11892                 }
11893         }
11894         return cnt;
11895 }
11896
11897 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
11898  * manager.
11899  */
11900 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
11901 {
11902         return pb->cpu_cnt;
11903 }
11904
11905 /*
11906  * Return perf_event FD of a ring buffer in *buf_idx* slot of
11907  * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
11908  * select()/poll()/epoll() Linux syscalls.
11909  */
11910 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
11911 {
11912         struct perf_cpu_buf *cpu_buf;
11913
11914         if (buf_idx >= pb->cpu_cnt)
11915                 return libbpf_err(-EINVAL);
11916
11917         cpu_buf = pb->cpu_bufs[buf_idx];
11918         if (!cpu_buf)
11919                 return libbpf_err(-ENOENT);
11920
11921         return cpu_buf->fd;
11922 }
11923
11924 int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size)
11925 {
11926         struct perf_cpu_buf *cpu_buf;
11927
11928         if (buf_idx >= pb->cpu_cnt)
11929                 return libbpf_err(-EINVAL);
11930
11931         cpu_buf = pb->cpu_bufs[buf_idx];
11932         if (!cpu_buf)
11933                 return libbpf_err(-ENOENT);
11934
11935         *buf = cpu_buf->base;
11936         *buf_size = pb->mmap_size;
11937         return 0;
11938 }
11939
11940 /*
11941  * Consume data from perf ring buffer corresponding to slot *buf_idx* in
11942  * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
11943  * consume, do nothing and return success.
11944  * Returns:
11945  *   - 0 on success;
11946  *   - <0 on failure.
11947  */
11948 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
11949 {
11950         struct perf_cpu_buf *cpu_buf;
11951
11952         if (buf_idx >= pb->cpu_cnt)
11953                 return libbpf_err(-EINVAL);
11954
11955         cpu_buf = pb->cpu_bufs[buf_idx];
11956         if (!cpu_buf)
11957                 return libbpf_err(-ENOENT);
11958
11959         return perf_buffer__process_records(pb, cpu_buf);
11960 }
11961
11962 int perf_buffer__consume(struct perf_buffer *pb)
11963 {
11964         int i, err;
11965
11966         for (i = 0; i < pb->cpu_cnt; i++) {
11967                 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
11968
11969                 if (!cpu_buf)
11970                         continue;
11971
11972                 err = perf_buffer__process_records(pb, cpu_buf);
11973                 if (err) {
11974                         pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
11975                         return libbpf_err(err);
11976                 }
11977         }
11978         return 0;
11979 }
11980
11981 int bpf_program__set_attach_target(struct bpf_program *prog,
11982                                    int attach_prog_fd,
11983                                    const char *attach_func_name)
11984 {
11985         int btf_obj_fd = 0, btf_id = 0, err;
11986
11987         if (!prog || attach_prog_fd < 0)
11988                 return libbpf_err(-EINVAL);
11989
11990         if (prog->obj->loaded)
11991                 return libbpf_err(-EINVAL);
11992
11993         if (attach_prog_fd && !attach_func_name) {
11994                 /* remember attach_prog_fd and let bpf_program__load() find
11995                  * BTF ID during the program load
11996                  */
11997                 prog->attach_prog_fd = attach_prog_fd;
11998                 return 0;
11999         }
12000
12001         if (attach_prog_fd) {
12002                 btf_id = libbpf_find_prog_btf_id(attach_func_name,
12003                                                  attach_prog_fd);
12004                 if (btf_id < 0)
12005                         return libbpf_err(btf_id);
12006         } else {
12007                 if (!attach_func_name)
12008                         return libbpf_err(-EINVAL);
12009
12010                 /* load btf_vmlinux, if not yet */
12011                 err = bpf_object__load_vmlinux_btf(prog->obj, true);
12012                 if (err)
12013                         return libbpf_err(err);
12014                 err = find_kernel_btf_id(prog->obj, attach_func_name,
12015                                          prog->expected_attach_type,
12016                                          &btf_obj_fd, &btf_id);
12017                 if (err)
12018                         return libbpf_err(err);
12019         }
12020
12021         prog->attach_btf_id = btf_id;
12022         prog->attach_btf_obj_fd = btf_obj_fd;
12023         prog->attach_prog_fd = attach_prog_fd;
12024         return 0;
12025 }
12026
12027 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
12028 {
12029         int err = 0, n, len, start, end = -1;
12030         bool *tmp;
12031
12032         *mask = NULL;
12033         *mask_sz = 0;
12034
12035         /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
12036         while (*s) {
12037                 if (*s == ',' || *s == '\n') {
12038                         s++;
12039                         continue;
12040                 }
12041                 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
12042                 if (n <= 0 || n > 2) {
12043                         pr_warn("Failed to get CPU range %s: %d\n", s, n);
12044                         err = -EINVAL;
12045                         goto cleanup;
12046                 } else if (n == 1) {
12047                         end = start;
12048                 }
12049                 if (start < 0 || start > end) {
12050                         pr_warn("Invalid CPU range [%d,%d] in %s\n",
12051                                 start, end, s);
12052                         err = -EINVAL;
12053                         goto cleanup;
12054                 }
12055                 tmp = realloc(*mask, end + 1);
12056                 if (!tmp) {
12057                         err = -ENOMEM;
12058                         goto cleanup;
12059                 }
12060                 *mask = tmp;
12061                 memset(tmp + *mask_sz, 0, start - *mask_sz);
12062                 memset(tmp + start, 1, end - start + 1);
12063                 *mask_sz = end + 1;
12064                 s += len;
12065         }
12066         if (!*mask_sz) {
12067                 pr_warn("Empty CPU range\n");
12068                 return -EINVAL;
12069         }
12070         return 0;
12071 cleanup:
12072         free(*mask);
12073         *mask = NULL;
12074         return err;
12075 }
12076
12077 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
12078 {
12079         int fd, err = 0, len;
12080         char buf[128];
12081
12082         fd = open(fcpu, O_RDONLY | O_CLOEXEC);
12083         if (fd < 0) {
12084                 err = -errno;
12085                 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
12086                 return err;
12087         }
12088         len = read(fd, buf, sizeof(buf));
12089         close(fd);
12090         if (len <= 0) {
12091                 err = len ? -errno : -EINVAL;
12092                 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
12093                 return err;
12094         }
12095         if (len >= sizeof(buf)) {
12096                 pr_warn("CPU mask is too big in file %s\n", fcpu);
12097                 return -E2BIG;
12098         }
12099         buf[len] = '\0';
12100
12101         return parse_cpu_mask_str(buf, mask, mask_sz);
12102 }
12103
12104 int libbpf_num_possible_cpus(void)
12105 {
12106         static const char *fcpu = "/sys/devices/system/cpu/possible";
12107         static int cpus;
12108         int err, n, i, tmp_cpus;
12109         bool *mask;
12110
12111         tmp_cpus = READ_ONCE(cpus);
12112         if (tmp_cpus > 0)
12113                 return tmp_cpus;
12114
12115         err = parse_cpu_mask_file(fcpu, &mask, &n);
12116         if (err)
12117                 return libbpf_err(err);
12118
12119         tmp_cpus = 0;
12120         for (i = 0; i < n; i++) {
12121                 if (mask[i])
12122                         tmp_cpus++;
12123         }
12124         free(mask);
12125
12126         WRITE_ONCE(cpus, tmp_cpus);
12127         return tmp_cpus;
12128 }
12129
12130 static int populate_skeleton_maps(const struct bpf_object *obj,
12131                                   struct bpf_map_skeleton *maps,
12132                                   size_t map_cnt)
12133 {
12134         int i;
12135
12136         for (i = 0; i < map_cnt; i++) {
12137                 struct bpf_map **map = maps[i].map;
12138                 const char *name = maps[i].name;
12139                 void **mmaped = maps[i].mmaped;
12140
12141                 *map = bpf_object__find_map_by_name(obj, name);
12142                 if (!*map) {
12143                         pr_warn("failed to find skeleton map '%s'\n", name);
12144                         return -ESRCH;
12145                 }
12146
12147                 /* externs shouldn't be pre-setup from user code */
12148                 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
12149                         *mmaped = (*map)->mmaped;
12150         }
12151         return 0;
12152 }
12153
12154 static int populate_skeleton_progs(const struct bpf_object *obj,
12155                                    struct bpf_prog_skeleton *progs,
12156                                    size_t prog_cnt)
12157 {
12158         int i;
12159
12160         for (i = 0; i < prog_cnt; i++) {
12161                 struct bpf_program **prog = progs[i].prog;
12162                 const char *name = progs[i].name;
12163
12164                 *prog = bpf_object__find_program_by_name(obj, name);
12165                 if (!*prog) {
12166                         pr_warn("failed to find skeleton program '%s'\n", name);
12167                         return -ESRCH;
12168                 }
12169         }
12170         return 0;
12171 }
12172
12173 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
12174                               const struct bpf_object_open_opts *opts)
12175 {
12176         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
12177                 .object_name = s->name,
12178         );
12179         struct bpf_object *obj;
12180         int err;
12181
12182         /* Attempt to preserve opts->object_name, unless overriden by user
12183          * explicitly. Overwriting object name for skeletons is discouraged,
12184          * as it breaks global data maps, because they contain object name
12185          * prefix as their own map name prefix. When skeleton is generated,
12186          * bpftool is making an assumption that this name will stay the same.
12187          */
12188         if (opts) {
12189                 memcpy(&skel_opts, opts, sizeof(*opts));
12190                 if (!opts->object_name)
12191                         skel_opts.object_name = s->name;
12192         }
12193
12194         obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
12195         err = libbpf_get_error(obj);
12196         if (err) {
12197                 pr_warn("failed to initialize skeleton BPF object '%s': %d\n",
12198                         s->name, err);
12199                 return libbpf_err(err);
12200         }
12201
12202         *s->obj = obj;
12203         err = populate_skeleton_maps(obj, s->maps, s->map_cnt);
12204         if (err) {
12205                 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
12206                 return libbpf_err(err);
12207         }
12208
12209         err = populate_skeleton_progs(obj, s->progs, s->prog_cnt);
12210         if (err) {
12211                 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
12212                 return libbpf_err(err);
12213         }
12214
12215         return 0;
12216 }
12217
12218 int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s)
12219 {
12220         int err, len, var_idx, i;
12221         const char *var_name;
12222         const struct bpf_map *map;
12223         struct btf *btf;
12224         __u32 map_type_id;
12225         const struct btf_type *map_type, *var_type;
12226         const struct bpf_var_skeleton *var_skel;
12227         struct btf_var_secinfo *var;
12228
12229         if (!s->obj)
12230                 return libbpf_err(-EINVAL);
12231
12232         btf = bpf_object__btf(s->obj);
12233         if (!btf) {
12234                 pr_warn("subskeletons require BTF at runtime (object %s)\n",
12235                         bpf_object__name(s->obj));
12236                 return libbpf_err(-errno);
12237         }
12238
12239         err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt);
12240         if (err) {
12241                 pr_warn("failed to populate subskeleton maps: %d\n", err);
12242                 return libbpf_err(err);
12243         }
12244
12245         err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt);
12246         if (err) {
12247                 pr_warn("failed to populate subskeleton maps: %d\n", err);
12248                 return libbpf_err(err);
12249         }
12250
12251         for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
12252                 var_skel = &s->vars[var_idx];
12253                 map = *var_skel->map;
12254                 map_type_id = bpf_map__btf_value_type_id(map);
12255                 map_type = btf__type_by_id(btf, map_type_id);
12256
12257                 if (!btf_is_datasec(map_type)) {
12258                         pr_warn("type for map '%1$s' is not a datasec: %2$s",
12259                                 bpf_map__name(map),
12260                                 __btf_kind_str(btf_kind(map_type)));
12261                         return libbpf_err(-EINVAL);
12262                 }
12263
12264                 len = btf_vlen(map_type);
12265                 var = btf_var_secinfos(map_type);
12266                 for (i = 0; i < len; i++, var++) {
12267                         var_type = btf__type_by_id(btf, var->type);
12268                         var_name = btf__name_by_offset(btf, var_type->name_off);
12269                         if (strcmp(var_name, var_skel->name) == 0) {
12270                                 *var_skel->addr = map->mmaped + var->offset;
12271                                 break;
12272                         }
12273                 }
12274         }
12275         return 0;
12276 }
12277
12278 void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s)
12279 {
12280         if (!s)
12281                 return;
12282         free(s->maps);
12283         free(s->progs);
12284         free(s->vars);
12285         free(s);
12286 }
12287
12288 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
12289 {
12290         int i, err;
12291
12292         err = bpf_object__load(*s->obj);
12293         if (err) {
12294                 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
12295                 return libbpf_err(err);
12296         }
12297
12298         for (i = 0; i < s->map_cnt; i++) {
12299                 struct bpf_map *map = *s->maps[i].map;
12300                 size_t mmap_sz = bpf_map_mmap_sz(map);
12301                 int prot, map_fd = bpf_map__fd(map);
12302                 void **mmaped = s->maps[i].mmaped;
12303
12304                 if (!mmaped)
12305                         continue;
12306
12307                 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
12308                         *mmaped = NULL;
12309                         continue;
12310                 }
12311
12312                 if (map->def.map_flags & BPF_F_RDONLY_PROG)
12313                         prot = PROT_READ;
12314                 else
12315                         prot = PROT_READ | PROT_WRITE;
12316
12317                 /* Remap anonymous mmap()-ed "map initialization image" as
12318                  * a BPF map-backed mmap()-ed memory, but preserving the same
12319                  * memory address. This will cause kernel to change process'
12320                  * page table to point to a different piece of kernel memory,
12321                  * but from userspace point of view memory address (and its
12322                  * contents, being identical at this point) will stay the
12323                  * same. This mapping will be released by bpf_object__close()
12324                  * as per normal clean up procedure, so we don't need to worry
12325                  * about it from skeleton's clean up perspective.
12326                  */
12327                 *mmaped = mmap(map->mmaped, mmap_sz, prot,
12328                                 MAP_SHARED | MAP_FIXED, map_fd, 0);
12329                 if (*mmaped == MAP_FAILED) {
12330                         err = -errno;
12331                         *mmaped = NULL;
12332                         pr_warn("failed to re-mmap() map '%s': %d\n",
12333                                  bpf_map__name(map), err);
12334                         return libbpf_err(err);
12335                 }
12336         }
12337
12338         return 0;
12339 }
12340
12341 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
12342 {
12343         int i, err;
12344
12345         for (i = 0; i < s->prog_cnt; i++) {
12346                 struct bpf_program *prog = *s->progs[i].prog;
12347                 struct bpf_link **link = s->progs[i].link;
12348
12349                 if (!prog->autoload)
12350                         continue;
12351
12352                 /* auto-attaching not supported for this program */
12353                 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12354                         continue;
12355
12356                 /* if user already set the link manually, don't attempt auto-attach */
12357                 if (*link)
12358                         continue;
12359
12360                 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
12361                 if (err) {
12362                         pr_warn("prog '%s': failed to auto-attach: %d\n",
12363                                 bpf_program__name(prog), err);
12364                         return libbpf_err(err);
12365                 }
12366
12367                 /* It's possible that for some SEC() definitions auto-attach
12368                  * is supported in some cases (e.g., if definition completely
12369                  * specifies target information), but is not in other cases.
12370                  * SEC("uprobe") is one such case. If user specified target
12371                  * binary and function name, such BPF program can be
12372                  * auto-attached. But if not, it shouldn't trigger skeleton's
12373                  * attach to fail. It should just be skipped.
12374                  * attach_fn signals such case with returning 0 (no error) and
12375                  * setting link to NULL.
12376                  */
12377         }
12378
12379         return 0;
12380 }
12381
12382 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
12383 {
12384         int i;
12385
12386         for (i = 0; i < s->prog_cnt; i++) {
12387                 struct bpf_link **link = s->progs[i].link;
12388
12389                 bpf_link__destroy(*link);
12390                 *link = NULL;
12391         }
12392 }
12393
12394 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
12395 {
12396         if (!s)
12397                 return;
12398
12399         if (s->progs)
12400                 bpf_object__detach_skeleton(s);
12401         if (s->obj)
12402                 bpf_object__close(*s->obj);
12403         free(s->maps);
12404         free(s->progs);
12405         free(s);
12406 }