1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
4 * Common eBPF ELF object loading operations.
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8 * Copyright (C) 2015 Huawei Inc.
9 * Copyright (C) 2017 Nicira, Inc.
10 * Copyright (C) 2019 Isovalent, Inc.
28 #include <asm/unistd.h>
29 #include <linux/err.h>
30 #include <linux/kernel.h>
31 #include <linux/bpf.h>
32 #include <linux/btf.h>
33 #include <linux/filter.h>
34 #include <linux/limits.h>
35 #include <linux/perf_event.h>
36 #include <linux/ring_buffer.h>
37 #include <linux/version.h>
38 #include <sys/epoll.h>
39 #include <sys/ioctl.h>
42 #include <sys/types.h>
44 #include <sys/utsname.h>
45 #include <sys/resource.h>
53 #include "str_error.h"
54 #include "libbpf_internal.h"
56 #include "bpf_gen_internal.h"
59 #define BPF_FS_MAGIC 0xcafe4a11
62 #define BPF_INSN_SZ (sizeof(struct bpf_insn))
64 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
65 * compilation if user enables corresponding warning. Disable it explicitly.
67 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
69 #define __printf(a, b) __attribute__((format(printf, a, b)))
71 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
72 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
74 static const char * const attach_type_name[] = {
75 [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress",
76 [BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress",
77 [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create",
78 [BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release",
79 [BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops",
80 [BPF_CGROUP_DEVICE] = "cgroup_device",
81 [BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind",
82 [BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind",
83 [BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect",
84 [BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect",
85 [BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind",
86 [BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind",
87 [BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername",
88 [BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername",
89 [BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname",
90 [BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname",
91 [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg",
92 [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg",
93 [BPF_CGROUP_SYSCTL] = "cgroup_sysctl",
94 [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg",
95 [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg",
96 [BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt",
97 [BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt",
98 [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",
99 [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
100 [BPF_SK_SKB_VERDICT] = "sk_skb_verdict",
101 [BPF_SK_MSG_VERDICT] = "sk_msg_verdict",
102 [BPF_LIRC_MODE2] = "lirc_mode2",
103 [BPF_FLOW_DISSECTOR] = "flow_dissector",
104 [BPF_TRACE_RAW_TP] = "trace_raw_tp",
105 [BPF_TRACE_FENTRY] = "trace_fentry",
106 [BPF_TRACE_FEXIT] = "trace_fexit",
107 [BPF_MODIFY_RETURN] = "modify_return",
108 [BPF_LSM_MAC] = "lsm_mac",
109 [BPF_LSM_CGROUP] = "lsm_cgroup",
110 [BPF_SK_LOOKUP] = "sk_lookup",
111 [BPF_TRACE_ITER] = "trace_iter",
112 [BPF_XDP_DEVMAP] = "xdp_devmap",
113 [BPF_XDP_CPUMAP] = "xdp_cpumap",
115 [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select",
116 [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate",
117 [BPF_PERF_EVENT] = "perf_event",
118 [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi",
121 static const char * const link_type_name[] = {
122 [BPF_LINK_TYPE_UNSPEC] = "unspec",
123 [BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
124 [BPF_LINK_TYPE_TRACING] = "tracing",
125 [BPF_LINK_TYPE_CGROUP] = "cgroup",
126 [BPF_LINK_TYPE_ITER] = "iter",
127 [BPF_LINK_TYPE_NETNS] = "netns",
128 [BPF_LINK_TYPE_XDP] = "xdp",
129 [BPF_LINK_TYPE_PERF_EVENT] = "perf_event",
130 [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi",
131 [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops",
134 static const char * const map_type_name[] = {
135 [BPF_MAP_TYPE_UNSPEC] = "unspec",
136 [BPF_MAP_TYPE_HASH] = "hash",
137 [BPF_MAP_TYPE_ARRAY] = "array",
138 [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array",
139 [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array",
140 [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash",
141 [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array",
142 [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace",
143 [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array",
144 [BPF_MAP_TYPE_LRU_HASH] = "lru_hash",
145 [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash",
146 [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie",
147 [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps",
148 [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps",
149 [BPF_MAP_TYPE_DEVMAP] = "devmap",
150 [BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash",
151 [BPF_MAP_TYPE_SOCKMAP] = "sockmap",
152 [BPF_MAP_TYPE_CPUMAP] = "cpumap",
153 [BPF_MAP_TYPE_XSKMAP] = "xskmap",
154 [BPF_MAP_TYPE_SOCKHASH] = "sockhash",
155 [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage",
156 [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray",
157 [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage",
158 [BPF_MAP_TYPE_QUEUE] = "queue",
159 [BPF_MAP_TYPE_STACK] = "stack",
160 [BPF_MAP_TYPE_SK_STORAGE] = "sk_storage",
161 [BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops",
162 [BPF_MAP_TYPE_RINGBUF] = "ringbuf",
163 [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage",
164 [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage",
165 [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter",
166 [BPF_MAP_TYPE_USER_RINGBUF] = "user_ringbuf",
169 static const char * const prog_type_name[] = {
170 [BPF_PROG_TYPE_UNSPEC] = "unspec",
171 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter",
172 [BPF_PROG_TYPE_KPROBE] = "kprobe",
173 [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls",
174 [BPF_PROG_TYPE_SCHED_ACT] = "sched_act",
175 [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint",
176 [BPF_PROG_TYPE_XDP] = "xdp",
177 [BPF_PROG_TYPE_PERF_EVENT] = "perf_event",
178 [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb",
179 [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock",
180 [BPF_PROG_TYPE_LWT_IN] = "lwt_in",
181 [BPF_PROG_TYPE_LWT_OUT] = "lwt_out",
182 [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit",
183 [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops",
184 [BPF_PROG_TYPE_SK_SKB] = "sk_skb",
185 [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device",
186 [BPF_PROG_TYPE_SK_MSG] = "sk_msg",
187 [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
188 [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr",
189 [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local",
190 [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2",
191 [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport",
192 [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector",
193 [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl",
194 [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
195 [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt",
196 [BPF_PROG_TYPE_TRACING] = "tracing",
197 [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops",
198 [BPF_PROG_TYPE_EXT] = "ext",
199 [BPF_PROG_TYPE_LSM] = "lsm",
200 [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup",
201 [BPF_PROG_TYPE_SYSCALL] = "syscall",
204 static int __base_pr(enum libbpf_print_level level, const char *format,
207 if (level == LIBBPF_DEBUG)
210 return vfprintf(stderr, format, args);
213 static libbpf_print_fn_t __libbpf_pr = __base_pr;
215 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
217 libbpf_print_fn_t old_print_fn = __libbpf_pr;
224 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
234 va_start(args, format);
235 __libbpf_pr(level, format, args);
241 static void pr_perm_msg(int err)
246 if (err != -EPERM || geteuid() != 0)
249 err = getrlimit(RLIMIT_MEMLOCK, &limit);
253 if (limit.rlim_cur == RLIM_INFINITY)
256 if (limit.rlim_cur < 1024)
257 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
258 else if (limit.rlim_cur < 1024*1024)
259 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
261 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
263 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
267 #define STRERR_BUFSIZE 128
269 /* Copied from tools/perf/util/util.h */
271 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
275 # define zclose(fd) ({ \
278 ___err = close((fd)); \
283 static inline __u64 ptr_to_u64(const void *ptr)
285 return (__u64) (unsigned long) ptr;
288 int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
290 /* as of v1.0 libbpf_set_strict_mode() is a no-op */
294 __u32 libbpf_major_version(void)
296 return LIBBPF_MAJOR_VERSION;
299 __u32 libbpf_minor_version(void)
301 return LIBBPF_MINOR_VERSION;
304 const char *libbpf_version_string(void)
308 return "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);
324 enum reloc_type type;
327 const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */
335 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
338 /* expected_attach_type is optional, if kernel doesn't support that */
339 SEC_EXP_ATTACH_OPT = 1,
340 /* legacy, only used by libbpf_get_type_names() and
341 * libbpf_attach_type_by_name(), not used by libbpf itself at all.
342 * This used to be associated with cgroup (and few other) BPF programs
343 * that were attachable through BPF_PROG_ATTACH command. Pretty
344 * meaningless nowadays, though.
347 SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT,
348 /* attachment target is specified through BTF ID in either kernel or
349 * other BPF program's BTF object */
351 /* BPF program type allows sleeping/blocking in kernel */
353 /* BPF program support non-linear XDP buffer */
359 enum bpf_prog_type prog_type;
360 enum bpf_attach_type expected_attach_type;
364 libbpf_prog_setup_fn_t prog_setup_fn;
365 libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;
366 libbpf_prog_attach_fn_t prog_attach_fn;
370 * bpf_prog should be a better name but it has been used in
377 const struct bpf_sec_def *sec_def;
378 /* this program's instruction offset (in number of instructions)
379 * within its containing ELF section
382 /* number of original instructions in ELF section belonging to this
383 * program, not taking into account subprogram instructions possible
384 * appended later during relocation
387 /* Offset (in number of instructions) of the start of instruction
388 * belonging to this BPF program within its containing main BPF
389 * program. For the entry-point (main) BPF program, this is always
390 * zero. For a sub-program, this gets reset before each of main BPF
391 * programs are processed and relocated and is used to determined
392 * whether sub-program was already appended to the main program, and
393 * if yes, at which instruction offset.
397 /* instructions that belong to BPF program; insns[0] is located at
398 * sec_insn_off instruction within its ELF section in ELF file, so
399 * when mapping ELF file instruction index to the local instruction,
400 * one needs to subtract sec_insn_off; and vice versa.
402 struct bpf_insn *insns;
403 /* actual number of instruction in this BPF program's image; for
404 * entry-point BPF programs this includes the size of main program
405 * itself plus all the used sub-programs, appended at the end
409 struct reloc_desc *reloc_desc;
412 /* BPF verifier log settings */
417 struct bpf_object *obj;
422 bool mark_btf_static;
423 enum bpf_prog_type type;
424 enum bpf_attach_type expected_attach_type;
427 __u32 attach_btf_obj_fd;
429 __u32 attach_prog_fd;
432 __u32 func_info_rec_size;
436 __u32 line_info_rec_size;
441 struct bpf_struct_ops {
443 const struct btf_type *type;
444 struct bpf_program **progs;
445 __u32 *kern_func_off;
446 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
448 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
449 * btf_vmlinux's format.
450 * struct bpf_struct_ops_tcp_congestion_ops {
451 * [... some other kernel fields ...]
452 * struct tcp_congestion_ops data;
454 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
455 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
462 #define DATA_SEC ".data"
463 #define BSS_SEC ".bss"
464 #define RODATA_SEC ".rodata"
465 #define KCONFIG_SEC ".kconfig"
466 #define KSYMS_SEC ".ksyms"
467 #define STRUCT_OPS_SEC ".struct_ops"
469 enum libbpf_map_type {
479 unsigned int key_size;
480 unsigned int value_size;
481 unsigned int max_entries;
482 unsigned int map_flags;
486 struct bpf_object *obj;
488 /* real_name is defined for special internal maps (.rodata*,
489 * .data*, .bss, .kconfig) and preserves their original ELF section
490 * name. This is important to be be able to find corresponding BTF
491 * DATASEC information.
499 struct bpf_map_def def;
502 __u32 btf_key_type_id;
503 __u32 btf_value_type_id;
504 __u32 btf_vmlinux_value_type_id;
505 enum libbpf_map_type libbpf_type;
507 struct bpf_struct_ops *st_ops;
508 struct bpf_map *inner_map;
534 enum extern_type type;
550 unsigned long long addr;
552 /* target btf_id of the corresponding kernel var. */
553 int kernel_btf_obj_fd;
556 /* local btf_id of the ksym extern's type. */
558 /* BTF fd index to be patched in for insn->off, this is
559 * 0 for vmlinux BTF, index in obj->fd_array for module
583 struct elf_sec_desc {
584 enum sec_type sec_type;
596 Elf_Data *st_ops_data;
597 size_t shstrndx; /* section index for section name strings */
599 struct elf_sec_desc *secs;
602 __u32 btf_maps_sec_btf_id;
611 char name[BPF_OBJ_NAME_LEN];
615 struct bpf_program *programs;
617 struct bpf_map *maps;
622 struct extern_desc *externs;
630 struct bpf_gen *gen_loader;
632 /* Information when doing ELF related work. Only valid if efile.elf is not NULL */
633 struct elf_state efile;
636 struct btf_ext *btf_ext;
638 /* Parse and load BTF vmlinux if any of the programs in the object need
641 struct btf *btf_vmlinux;
642 /* Path to the custom BTF to be used for BPF CO-RE relocations as an
643 * override for vmlinux BTF.
645 char *btf_custom_path;
646 /* vmlinux BTF override for CO-RE relocations */
647 struct btf *btf_vmlinux_override;
648 /* Lazily initialized kernel module BTFs */
649 struct module_btf *btf_modules;
650 bool btf_modules_loaded;
651 size_t btf_module_cnt;
652 size_t btf_module_cap;
654 /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */
663 struct usdt_manager *usdt_man;
668 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
669 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
670 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
671 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
672 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);
673 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
674 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
675 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);
676 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx);
678 void bpf_program__unload(struct bpf_program *prog)
685 zfree(&prog->func_info);
686 zfree(&prog->line_info);
689 static void bpf_program__exit(struct bpf_program *prog)
694 bpf_program__unload(prog);
696 zfree(&prog->sec_name);
698 zfree(&prog->reloc_desc);
705 static bool insn_is_subprog_call(const struct bpf_insn *insn)
707 return BPF_CLASS(insn->code) == BPF_JMP &&
708 BPF_OP(insn->code) == BPF_CALL &&
709 BPF_SRC(insn->code) == BPF_K &&
710 insn->src_reg == BPF_PSEUDO_CALL &&
711 insn->dst_reg == 0 &&
715 static bool is_call_insn(const struct bpf_insn *insn)
717 return insn->code == (BPF_JMP | BPF_CALL);
720 static bool insn_is_pseudo_func(struct bpf_insn *insn)
722 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
726 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
727 const char *name, size_t sec_idx, const char *sec_name,
728 size_t sec_off, void *insn_data, size_t insn_data_sz)
730 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
731 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
732 sec_name, name, sec_off, insn_data_sz);
736 memset(prog, 0, sizeof(*prog));
739 prog->sec_idx = sec_idx;
740 prog->sec_insn_off = sec_off / BPF_INSN_SZ;
741 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
742 /* insns_cnt can later be increased by appending used subprograms */
743 prog->insns_cnt = prog->sec_insn_cnt;
745 prog->type = BPF_PROG_TYPE_UNSPEC;
748 /* libbpf's convention for SEC("?abc...") is that it's just like
749 * SEC("abc...") but the corresponding bpf_program starts out with
750 * autoload set to false.
752 if (sec_name[0] == '?') {
753 prog->autoload = false;
754 /* from now on forget there was ? in section name */
757 prog->autoload = true;
760 prog->autoattach = true;
762 /* inherit object's log_level */
763 prog->log_level = obj->log_level;
765 prog->sec_name = strdup(sec_name);
769 prog->name = strdup(name);
773 prog->insns = malloc(insn_data_sz);
776 memcpy(prog->insns, insn_data, insn_data_sz);
780 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
781 bpf_program__exit(prog);
786 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
787 const char *sec_name, int sec_idx)
789 Elf_Data *symbols = obj->efile.symbols;
790 struct bpf_program *prog, *progs;
791 void *data = sec_data->d_buf;
792 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
793 int nr_progs, err, i;
797 progs = obj->programs;
798 nr_progs = obj->nr_programs;
799 nr_syms = symbols->d_size / sizeof(Elf64_Sym);
802 for (i = 0; i < nr_syms; i++) {
803 sym = elf_sym_by_idx(obj, i);
805 if (sym->st_shndx != sec_idx)
807 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
810 prog_sz = sym->st_size;
811 sec_off = sym->st_value;
813 name = elf_sym_str(obj, sym->st_name);
815 pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
817 return -LIBBPF_ERRNO__FORMAT;
820 if (sec_off + prog_sz > sec_sz) {
821 pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
823 return -LIBBPF_ERRNO__FORMAT;
826 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
827 pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);
831 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
832 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
834 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
837 * In this case the original obj->programs
838 * is still valid, so don't need special treat for
839 * bpf_close_object().
841 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
845 obj->programs = progs;
847 prog = &progs[nr_progs];
849 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
850 sec_off, data + sec_off, prog_sz);
854 /* if function is a global/weak symbol, but has restricted
855 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC
856 * as static to enable more permissive BPF verification mode
857 * with more outside context available to BPF verifier
859 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL
860 && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
861 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
862 prog->mark_btf_static = true;
865 obj->nr_programs = nr_progs;
871 __u32 get_kernel_version(void)
873 /* On Ubuntu LINUX_VERSION_CODE doesn't correspond to info.release,
874 * but Ubuntu provides /proc/version_signature file, as described at
875 * https://ubuntu.com/kernel, with an example contents below, which we
876 * can use to get a proper LINUX_VERSION_CODE.
878 * Ubuntu 5.4.0-12.15-generic 5.4.8
880 * In the above, 5.4.8 is what kernel is actually expecting, while
881 * uname() call will return 5.4.0 in info.release.
883 const char *ubuntu_kver_file = "/proc/version_signature";
884 __u32 major, minor, patch;
887 if (faccessat(AT_FDCWD, ubuntu_kver_file, R_OK, AT_EACCESS) == 0) {
890 f = fopen(ubuntu_kver_file, "r");
892 if (fscanf(f, "%*s %*s %d.%d.%d\n", &major, &minor, &patch) == 3) {
894 return KERNEL_VERSION(major, minor, patch);
898 /* something went wrong, fall back to uname() approach */
902 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
904 return KERNEL_VERSION(major, minor, patch);
907 static const struct btf_member *
908 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
910 struct btf_member *m;
913 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
914 if (btf_member_bit_offset(t, i) == bit_offset)
921 static const struct btf_member *
922 find_member_by_name(const struct btf *btf, const struct btf_type *t,
925 struct btf_member *m;
928 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
929 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
936 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
937 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
938 const char *name, __u32 kind);
941 find_struct_ops_kern_types(const struct btf *btf, const char *tname,
942 const struct btf_type **type, __u32 *type_id,
943 const struct btf_type **vtype, __u32 *vtype_id,
944 const struct btf_member **data_member)
946 const struct btf_type *kern_type, *kern_vtype;
947 const struct btf_member *kern_data_member;
948 __s32 kern_vtype_id, kern_type_id;
951 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
952 if (kern_type_id < 0) {
953 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
957 kern_type = btf__type_by_id(btf, kern_type_id);
959 /* Find the corresponding "map_value" type that will be used
960 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example,
961 * find "struct bpf_struct_ops_tcp_congestion_ops" from the
964 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
965 tname, BTF_KIND_STRUCT);
966 if (kern_vtype_id < 0) {
967 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
968 STRUCT_OPS_VALUE_PREFIX, tname);
969 return kern_vtype_id;
971 kern_vtype = btf__type_by_id(btf, kern_vtype_id);
973 /* Find "struct tcp_congestion_ops" from
974 * struct bpf_struct_ops_tcp_congestion_ops {
976 * struct tcp_congestion_ops data;
979 kern_data_member = btf_members(kern_vtype);
980 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
981 if (kern_data_member->type == kern_type_id)
984 if (i == btf_vlen(kern_vtype)) {
985 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
986 tname, STRUCT_OPS_VALUE_PREFIX, tname);
991 *type_id = kern_type_id;
993 *vtype_id = kern_vtype_id;
994 *data_member = kern_data_member;
999 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
1001 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
1004 /* Init the map's fields that depend on kern_btf */
1005 static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
1006 const struct btf *btf,
1007 const struct btf *kern_btf)
1009 const struct btf_member *member, *kern_member, *kern_data_member;
1010 const struct btf_type *type, *kern_type, *kern_vtype;
1011 __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
1012 struct bpf_struct_ops *st_ops;
1013 void *data, *kern_data;
1017 st_ops = map->st_ops;
1018 type = st_ops->type;
1019 tname = st_ops->tname;
1020 err = find_struct_ops_kern_types(kern_btf, tname,
1021 &kern_type, &kern_type_id,
1022 &kern_vtype, &kern_vtype_id,
1027 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
1028 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
1030 map->def.value_size = kern_vtype->size;
1031 map->btf_vmlinux_value_type_id = kern_vtype_id;
1033 st_ops->kern_vdata = calloc(1, kern_vtype->size);
1034 if (!st_ops->kern_vdata)
1037 data = st_ops->data;
1038 kern_data_off = kern_data_member->offset / 8;
1039 kern_data = st_ops->kern_vdata + kern_data_off;
1041 member = btf_members(type);
1042 for (i = 0; i < btf_vlen(type); i++, member++) {
1043 const struct btf_type *mtype, *kern_mtype;
1044 __u32 mtype_id, kern_mtype_id;
1045 void *mdata, *kern_mdata;
1046 __s64 msize, kern_msize;
1047 __u32 moff, kern_moff;
1048 __u32 kern_member_idx;
1051 mname = btf__name_by_offset(btf, member->name_off);
1052 kern_member = find_member_by_name(kern_btf, kern_type, mname);
1054 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
1059 kern_member_idx = kern_member - btf_members(kern_type);
1060 if (btf_member_bitfield_size(type, i) ||
1061 btf_member_bitfield_size(kern_type, kern_member_idx)) {
1062 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
1067 moff = member->offset / 8;
1068 kern_moff = kern_member->offset / 8;
1070 mdata = data + moff;
1071 kern_mdata = kern_data + kern_moff;
1073 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
1074 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
1076 if (BTF_INFO_KIND(mtype->info) !=
1077 BTF_INFO_KIND(kern_mtype->info)) {
1078 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
1079 map->name, mname, BTF_INFO_KIND(mtype->info),
1080 BTF_INFO_KIND(kern_mtype->info));
1084 if (btf_is_ptr(mtype)) {
1085 struct bpf_program *prog;
1087 prog = st_ops->progs[i];
1091 kern_mtype = skip_mods_and_typedefs(kern_btf,
1095 /* mtype->type must be a func_proto which was
1096 * guaranteed in bpf_object__collect_st_ops_relos(),
1097 * so only check kern_mtype for func_proto here.
1099 if (!btf_is_func_proto(kern_mtype)) {
1100 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
1105 prog->attach_btf_id = kern_type_id;
1106 prog->expected_attach_type = kern_member_idx;
1108 st_ops->kern_func_off[i] = kern_data_off + kern_moff;
1110 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1111 map->name, mname, prog->name, moff,
1117 msize = btf__resolve_size(btf, mtype_id);
1118 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
1119 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
1120 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
1121 map->name, mname, (ssize_t)msize,
1122 (ssize_t)kern_msize);
1126 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
1127 map->name, mname, (unsigned int)msize,
1129 memcpy(kern_mdata, mdata, msize);
1135 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
1137 struct bpf_map *map;
1141 for (i = 0; i < obj->nr_maps; i++) {
1142 map = &obj->maps[i];
1144 if (!bpf_map__is_struct_ops(map))
1147 err = bpf_map__init_kern_struct_ops(map, obj->btf,
1156 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
1158 const struct btf_type *type, *datasec;
1159 const struct btf_var_secinfo *vsi;
1160 struct bpf_struct_ops *st_ops;
1161 const char *tname, *var_name;
1162 __s32 type_id, datasec_id;
1163 const struct btf *btf;
1164 struct bpf_map *map;
1167 if (obj->efile.st_ops_shndx == -1)
1171 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
1173 if (datasec_id < 0) {
1174 pr_warn("struct_ops init: DATASEC %s not found\n",
1179 datasec = btf__type_by_id(btf, datasec_id);
1180 vsi = btf_var_secinfos(datasec);
1181 for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
1182 type = btf__type_by_id(obj->btf, vsi->type);
1183 var_name = btf__name_by_offset(obj->btf, type->name_off);
1185 type_id = btf__resolve_type(obj->btf, vsi->type);
1187 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
1188 vsi->type, STRUCT_OPS_SEC);
1192 type = btf__type_by_id(obj->btf, type_id);
1193 tname = btf__name_by_offset(obj->btf, type->name_off);
1195 pr_warn("struct_ops init: anonymous type is not supported\n");
1198 if (!btf_is_struct(type)) {
1199 pr_warn("struct_ops init: %s is not a struct\n", tname);
1203 map = bpf_object__add_map(obj);
1205 return PTR_ERR(map);
1207 map->sec_idx = obj->efile.st_ops_shndx;
1208 map->sec_offset = vsi->offset;
1209 map->name = strdup(var_name);
1213 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1214 map->def.key_size = sizeof(int);
1215 map->def.value_size = type->size;
1216 map->def.max_entries = 1;
1218 map->st_ops = calloc(1, sizeof(*map->st_ops));
1221 st_ops = map->st_ops;
1222 st_ops->data = malloc(type->size);
1223 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1224 st_ops->kern_func_off = malloc(btf_vlen(type) *
1225 sizeof(*st_ops->kern_func_off));
1226 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1229 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
1230 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1231 var_name, STRUCT_OPS_SEC);
1235 memcpy(st_ops->data,
1236 obj->efile.st_ops_data->d_buf + vsi->offset,
1238 st_ops->tname = tname;
1239 st_ops->type = type;
1240 st_ops->type_id = type_id;
1242 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1243 tname, type_id, var_name, vsi->offset);
1249 static struct bpf_object *bpf_object__new(const char *path,
1250 const void *obj_buf,
1252 const char *obj_name)
1254 struct bpf_object *obj;
1257 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1259 pr_warn("alloc memory failed for %s\n", path);
1260 return ERR_PTR(-ENOMEM);
1263 strcpy(obj->path, path);
1265 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1267 /* Using basename() GNU version which doesn't modify arg. */
1268 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1269 end = strchr(obj->name, '.');
1276 * Caller of this function should also call
1277 * bpf_object__elf_finish() after data collection to return
1278 * obj_buf to user. If not, we should duplicate the buffer to
1279 * avoid user freeing them before elf finish.
1281 obj->efile.obj_buf = obj_buf;
1282 obj->efile.obj_buf_sz = obj_buf_sz;
1283 obj->efile.btf_maps_shndx = -1;
1284 obj->efile.st_ops_shndx = -1;
1285 obj->kconfig_map_idx = -1;
1287 obj->kern_version = get_kernel_version();
1288 obj->loaded = false;
1293 static void bpf_object__elf_finish(struct bpf_object *obj)
1295 if (!obj->efile.elf)
1298 elf_end(obj->efile.elf);
1299 obj->efile.elf = NULL;
1300 obj->efile.symbols = NULL;
1301 obj->efile.st_ops_data = NULL;
1303 zfree(&obj->efile.secs);
1304 obj->efile.sec_cnt = 0;
1305 zclose(obj->efile.fd);
1306 obj->efile.obj_buf = NULL;
1307 obj->efile.obj_buf_sz = 0;
1310 static int bpf_object__elf_init(struct bpf_object *obj)
1316 if (obj->efile.elf) {
1317 pr_warn("elf: init internal error\n");
1318 return -LIBBPF_ERRNO__LIBELF;
1321 if (obj->efile.obj_buf_sz > 0) {
1322 /* obj_buf should have been validated by bpf_object__open_mem(). */
1323 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1325 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1326 if (obj->efile.fd < 0) {
1327 char errmsg[STRERR_BUFSIZE], *cp;
1330 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1331 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1335 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1339 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1340 err = -LIBBPF_ERRNO__LIBELF;
1344 obj->efile.elf = elf;
1346 if (elf_kind(elf) != ELF_K_ELF) {
1347 err = -LIBBPF_ERRNO__FORMAT;
1348 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1352 if (gelf_getclass(elf) != ELFCLASS64) {
1353 err = -LIBBPF_ERRNO__FORMAT;
1354 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1358 obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1359 if (!obj->efile.ehdr) {
1360 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1361 err = -LIBBPF_ERRNO__FORMAT;
1365 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1366 pr_warn("elf: failed to get section names section index for %s: %s\n",
1367 obj->path, elf_errmsg(-1));
1368 err = -LIBBPF_ERRNO__FORMAT;
1372 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1373 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1374 pr_warn("elf: failed to get section names strings from %s: %s\n",
1375 obj->path, elf_errmsg(-1));
1376 err = -LIBBPF_ERRNO__FORMAT;
1380 /* Old LLVM set e_machine to EM_NONE */
1381 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1382 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1383 err = -LIBBPF_ERRNO__FORMAT;
1389 bpf_object__elf_finish(obj);
1393 static int bpf_object__check_endianness(struct bpf_object *obj)
1395 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1396 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1398 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1399 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1402 # error "Unrecognized __BYTE_ORDER__"
1404 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1405 return -LIBBPF_ERRNO__ENDIAN;
1409 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1412 pr_warn("invalid license section in %s\n", obj->path);
1413 return -LIBBPF_ERRNO__FORMAT;
1415 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1416 * go over allowed ELF data section buffer
1418 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1419 pr_debug("license of %s is %s\n", obj->path, obj->license);
1424 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1428 if (!data || size != sizeof(kver)) {
1429 pr_warn("invalid kver section in %s\n", obj->path);
1430 return -LIBBPF_ERRNO__FORMAT;
1432 memcpy(&kver, data, sizeof(kver));
1433 obj->kern_version = kver;
1434 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1438 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1440 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1441 type == BPF_MAP_TYPE_HASH_OF_MAPS)
1446 static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size)
1454 scn = elf_sec_by_name(obj, name);
1455 data = elf_sec_data(obj, scn);
1457 *size = data->d_size;
1458 return 0; /* found it */
1464 static int find_elf_var_offset(const struct bpf_object *obj, const char *name, __u32 *off)
1466 Elf_Data *symbols = obj->efile.symbols;
1473 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1474 Elf64_Sym *sym = elf_sym_by_idx(obj, si);
1476 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1479 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1480 ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1483 sname = elf_sym_str(obj, sym->st_name);
1485 pr_warn("failed to get sym name string for var %s\n", name);
1488 if (strcmp(name, sname) == 0) {
1489 *off = sym->st_value;
1497 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1499 struct bpf_map *map;
1502 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1503 sizeof(*obj->maps), obj->nr_maps + 1);
1505 return ERR_PTR(err);
1507 map = &obj->maps[obj->nr_maps++];
1510 map->inner_map_fd = -1;
1511 map->autocreate = true;
1516 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1518 long page_sz = sysconf(_SC_PAGE_SIZE);
1521 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1522 map_sz = roundup(map_sz, page_sz);
1526 static char *internal_map_name(struct bpf_object *obj, const char *real_name)
1528 char map_name[BPF_OBJ_NAME_LEN], *p;
1529 int pfx_len, sfx_len = max((size_t)7, strlen(real_name));
1531 /* This is one of the more confusing parts of libbpf for various
1532 * reasons, some of which are historical. The original idea for naming
1533 * internal names was to include as much of BPF object name prefix as
1534 * possible, so that it can be distinguished from similar internal
1535 * maps of a different BPF object.
1536 * As an example, let's say we have bpf_object named 'my_object_name'
1537 * and internal map corresponding to '.rodata' ELF section. The final
1538 * map name advertised to user and to the kernel will be
1539 * 'my_objec.rodata', taking first 8 characters of object name and
1540 * entire 7 characters of '.rodata'.
1541 * Somewhat confusingly, if internal map ELF section name is shorter
1542 * than 7 characters, e.g., '.bss', we still reserve 7 characters
1543 * for the suffix, even though we only have 4 actual characters, and
1544 * resulting map will be called 'my_objec.bss', not even using all 15
1545 * characters allowed by the kernel. Oh well, at least the truncated
1546 * object name is somewhat consistent in this case. But if the map
1547 * name is '.kconfig', we'll still have entirety of '.kconfig' added
1548 * (8 chars) and thus will be left with only first 7 characters of the
1549 * object name ('my_obje'). Happy guessing, user, that the final map
1550 * name will be "my_obje.kconfig".
1551 * Now, with libbpf starting to support arbitrarily named .rodata.*
1552 * and .data.* data sections, it's possible that ELF section name is
1553 * longer than allowed 15 chars, so we now need to be careful to take
1554 * only up to 15 first characters of ELF name, taking no BPF object
1555 * name characters at all. So '.rodata.abracadabra' will result in
1556 * '.rodata.abracad' kernel and user-visible name.
1557 * We need to keep this convoluted logic intact for .data, .bss and
1558 * .rodata maps, but for new custom .data.custom and .rodata.custom
1559 * maps we use their ELF names as is, not prepending bpf_object name
1560 * in front. We still need to truncate them to 15 characters for the
1561 * kernel. Full name can be recovered for such maps by using DATASEC
1562 * BTF type associated with such map's value type, though.
1564 if (sfx_len >= BPF_OBJ_NAME_LEN)
1565 sfx_len = BPF_OBJ_NAME_LEN - 1;
1567 /* if there are two or more dots in map name, it's a custom dot map */
1568 if (strchr(real_name + 1, '.') != NULL)
1571 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1573 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1574 sfx_len, real_name);
1576 /* sanitise map name to characters allowed by kernel */
1577 for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1578 if (!isalnum(*p) && *p != '_' && *p != '.')
1581 return strdup(map_name);
1585 bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map);
1588 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1589 const char *real_name, int sec_idx, void *data, size_t data_sz)
1591 struct bpf_map_def *def;
1592 struct bpf_map *map;
1595 map = bpf_object__add_map(obj);
1597 return PTR_ERR(map);
1599 map->libbpf_type = type;
1600 map->sec_idx = sec_idx;
1601 map->sec_offset = 0;
1602 map->real_name = strdup(real_name);
1603 map->name = internal_map_name(obj, real_name);
1604 if (!map->real_name || !map->name) {
1605 zfree(&map->real_name);
1611 def->type = BPF_MAP_TYPE_ARRAY;
1612 def->key_size = sizeof(int);
1613 def->value_size = data_sz;
1614 def->max_entries = 1;
1615 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1616 ? BPF_F_RDONLY_PROG : 0;
1617 def->map_flags |= BPF_F_MMAPABLE;
1619 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1620 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1622 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1623 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1624 if (map->mmaped == MAP_FAILED) {
1627 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1629 zfree(&map->real_name);
1634 /* failures are fine because of maps like .rodata.str1.1 */
1635 (void) bpf_map_find_btf_info(obj, map);
1638 memcpy(map->mmaped, data, data_sz);
1640 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1644 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1646 struct elf_sec_desc *sec_desc;
1647 const char *sec_name;
1648 int err = 0, sec_idx;
1651 * Populate obj->maps with libbpf internal maps.
1653 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1654 sec_desc = &obj->efile.secs[sec_idx];
1656 /* Skip recognized sections with size 0. */
1657 if (!sec_desc->data || sec_desc->data->d_size == 0)
1660 switch (sec_desc->sec_type) {
1662 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1663 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1665 sec_desc->data->d_buf,
1666 sec_desc->data->d_size);
1669 obj->has_rodata = true;
1670 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1671 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1673 sec_desc->data->d_buf,
1674 sec_desc->data->d_size);
1677 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1678 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1681 sec_desc->data->d_size);
1694 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1699 for (i = 0; i < obj->nr_extern; i++) {
1700 if (strcmp(obj->externs[i].name, name) == 0)
1701 return &obj->externs[i];
1706 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1709 switch (ext->kcfg.type) {
1712 pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",
1716 *(bool *)ext_val = value == 'y' ? true : false;
1720 *(enum libbpf_tristate *)ext_val = TRI_YES;
1721 else if (value == 'm')
1722 *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1723 else /* value == 'n' */
1724 *(enum libbpf_tristate *)ext_val = TRI_NO;
1727 *(char *)ext_val = value;
1733 pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",
1741 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1746 if (ext->kcfg.type != KCFG_CHAR_ARR) {
1747 pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",
1752 len = strlen(value);
1753 if (value[len - 1] != '"') {
1754 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1761 if (len >= ext->kcfg.sz) {
1762 pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",
1763 ext->name, value, len, ext->kcfg.sz - 1);
1764 len = ext->kcfg.sz - 1;
1766 memcpy(ext_val, value + 1, len);
1767 ext_val[len] = '\0';
1772 static int parse_u64(const char *value, __u64 *res)
1778 *res = strtoull(value, &value_end, 0);
1781 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1785 pr_warn("failed to parse '%s' as integer completely\n", value);
1791 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1793 int bit_sz = ext->kcfg.sz * 8;
1795 if (ext->kcfg.sz == 8)
1798 /* Validate that value stored in u64 fits in integer of `ext->sz`
1799 * bytes size without any loss of information. If the target integer
1800 * is signed, we rely on the following limits of integer type of
1801 * Y bits and subsequent transformation:
1803 * -2^(Y-1) <= X <= 2^(Y-1) - 1
1804 * 0 <= X + 2^(Y-1) <= 2^Y - 1
1805 * 0 <= X + 2^(Y-1) < 2^Y
1807 * For unsigned target integer, check that all the (64 - Y) bits are
1810 if (ext->kcfg.is_signed)
1811 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1813 return (v >> bit_sz) == 0;
1816 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1819 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
1820 ext->kcfg.type != KCFG_BOOL) {
1821 pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",
1822 ext->name, (unsigned long long)value);
1825 if (ext->kcfg.type == KCFG_BOOL && value > 1) {
1826 pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",
1827 ext->name, (unsigned long long)value);
1831 if (!is_kcfg_value_in_range(ext, value)) {
1832 pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",
1833 ext->name, (unsigned long long)value, ext->kcfg.sz);
1836 switch (ext->kcfg.sz) {
1837 case 1: *(__u8 *)ext_val = value; break;
1838 case 2: *(__u16 *)ext_val = value; break;
1839 case 4: *(__u32 *)ext_val = value; break;
1840 case 8: *(__u64 *)ext_val = value; break;
1848 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1849 char *buf, void *data)
1851 struct extern_desc *ext;
1857 if (!str_has_pfx(buf, "CONFIG_"))
1860 sep = strchr(buf, '=');
1862 pr_warn("failed to parse '%s': no separator\n", buf);
1866 /* Trim ending '\n' */
1868 if (buf[len - 1] == '\n')
1869 buf[len - 1] = '\0';
1870 /* Split on '=' and ensure that a value is present. */
1874 pr_warn("failed to parse '%s': no value\n", buf);
1878 ext = find_extern_by_name(obj, buf);
1879 if (!ext || ext->is_set)
1882 ext_val = data + ext->kcfg.data_off;
1886 case 'y': case 'n': case 'm':
1887 err = set_kcfg_value_tri(ext, ext_val, *value);
1890 err = set_kcfg_value_str(ext, ext_val, value);
1893 /* assume integer */
1894 err = parse_u64(value, &num);
1896 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
1899 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1900 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
1903 err = set_kcfg_value_num(ext, ext_val, num);
1908 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
1912 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1920 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1923 else if (len >= PATH_MAX)
1924 return -ENAMETOOLONG;
1926 /* gzopen also accepts uncompressed files. */
1927 file = gzopen(buf, "r");
1929 file = gzopen("/proc/config.gz", "r");
1932 pr_warn("failed to open system Kconfig\n");
1936 while (gzgets(file, buf, sizeof(buf))) {
1937 err = bpf_object__process_kconfig_line(obj, buf, data);
1939 pr_warn("error parsing system Kconfig line '%s': %d\n",
1950 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1951 const char *config, void *data)
1957 file = fmemopen((void *)config, strlen(config), "r");
1960 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1964 while (fgets(buf, sizeof(buf), file)) {
1965 err = bpf_object__process_kconfig_line(obj, buf, data);
1967 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1977 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1979 struct extern_desc *last_ext = NULL, *ext;
1983 for (i = 0; i < obj->nr_extern; i++) {
1984 ext = &obj->externs[i];
1985 if (ext->type == EXT_KCFG)
1992 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
1993 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1994 ".kconfig", obj->efile.symbols_shndx,
1999 obj->kconfig_map_idx = obj->nr_maps - 1;
2004 const struct btf_type *
2005 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
2007 const struct btf_type *t = btf__type_by_id(btf, id);
2012 while (btf_is_mod(t) || btf_is_typedef(t)) {
2015 t = btf__type_by_id(btf, t->type);
2021 static const struct btf_type *
2022 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
2024 const struct btf_type *t;
2026 t = skip_mods_and_typedefs(btf, id, NULL);
2030 t = skip_mods_and_typedefs(btf, t->type, res_id);
2032 return btf_is_func_proto(t) ? t : NULL;
2035 static const char *__btf_kind_str(__u16 kind)
2038 case BTF_KIND_UNKN: return "void";
2039 case BTF_KIND_INT: return "int";
2040 case BTF_KIND_PTR: return "ptr";
2041 case BTF_KIND_ARRAY: return "array";
2042 case BTF_KIND_STRUCT: return "struct";
2043 case BTF_KIND_UNION: return "union";
2044 case BTF_KIND_ENUM: return "enum";
2045 case BTF_KIND_FWD: return "fwd";
2046 case BTF_KIND_TYPEDEF: return "typedef";
2047 case BTF_KIND_VOLATILE: return "volatile";
2048 case BTF_KIND_CONST: return "const";
2049 case BTF_KIND_RESTRICT: return "restrict";
2050 case BTF_KIND_FUNC: return "func";
2051 case BTF_KIND_FUNC_PROTO: return "func_proto";
2052 case BTF_KIND_VAR: return "var";
2053 case BTF_KIND_DATASEC: return "datasec";
2054 case BTF_KIND_FLOAT: return "float";
2055 case BTF_KIND_DECL_TAG: return "decl_tag";
2056 case BTF_KIND_TYPE_TAG: return "type_tag";
2057 case BTF_KIND_ENUM64: return "enum64";
2058 default: return "unknown";
2062 const char *btf_kind_str(const struct btf_type *t)
2064 return __btf_kind_str(btf_kind(t));
2068 * Fetch integer attribute of BTF map definition. Such attributes are
2069 * represented using a pointer to an array, in which dimensionality of array
2070 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
2071 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
2072 * type definition, while using only sizeof(void *) space in ELF data section.
2074 static bool get_map_field_int(const char *map_name, const struct btf *btf,
2075 const struct btf_member *m, __u32 *res)
2077 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2078 const char *name = btf__name_by_offset(btf, m->name_off);
2079 const struct btf_array *arr_info;
2080 const struct btf_type *arr_t;
2082 if (!btf_is_ptr(t)) {
2083 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
2084 map_name, name, btf_kind_str(t));
2088 arr_t = btf__type_by_id(btf, t->type);
2090 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
2091 map_name, name, t->type);
2094 if (!btf_is_array(arr_t)) {
2095 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
2096 map_name, name, btf_kind_str(arr_t));
2099 arr_info = btf_array(arr_t);
2100 *res = arr_info->nelems;
2104 static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name)
2108 len = snprintf(buf, buf_sz, "%s/%s", path, name);
2112 return -ENAMETOOLONG;
2117 static int build_map_pin_path(struct bpf_map *map, const char *path)
2123 path = "/sys/fs/bpf";
2125 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
2129 return bpf_map__set_pin_path(map, buf);
2132 /* should match definition in bpf_helpers.h */
2133 enum libbpf_pin_type {
2135 /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
2139 int parse_btf_map_def(const char *map_name, struct btf *btf,
2140 const struct btf_type *def_t, bool strict,
2141 struct btf_map_def *map_def, struct btf_map_def *inner_def)
2143 const struct btf_type *t;
2144 const struct btf_member *m;
2145 bool is_inner = inner_def == NULL;
2148 vlen = btf_vlen(def_t);
2149 m = btf_members(def_t);
2150 for (i = 0; i < vlen; i++, m++) {
2151 const char *name = btf__name_by_offset(btf, m->name_off);
2154 pr_warn("map '%s': invalid field #%d.\n", map_name, i);
2157 if (strcmp(name, "type") == 0) {
2158 if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2160 map_def->parts |= MAP_DEF_MAP_TYPE;
2161 } else if (strcmp(name, "max_entries") == 0) {
2162 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2164 map_def->parts |= MAP_DEF_MAX_ENTRIES;
2165 } else if (strcmp(name, "map_flags") == 0) {
2166 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2168 map_def->parts |= MAP_DEF_MAP_FLAGS;
2169 } else if (strcmp(name, "numa_node") == 0) {
2170 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2172 map_def->parts |= MAP_DEF_NUMA_NODE;
2173 } else if (strcmp(name, "key_size") == 0) {
2176 if (!get_map_field_int(map_name, btf, m, &sz))
2178 if (map_def->key_size && map_def->key_size != sz) {
2179 pr_warn("map '%s': conflicting key size %u != %u.\n",
2180 map_name, map_def->key_size, sz);
2183 map_def->key_size = sz;
2184 map_def->parts |= MAP_DEF_KEY_SIZE;
2185 } else if (strcmp(name, "key") == 0) {
2188 t = btf__type_by_id(btf, m->type);
2190 pr_warn("map '%s': key type [%d] not found.\n",
2194 if (!btf_is_ptr(t)) {
2195 pr_warn("map '%s': key spec is not PTR: %s.\n",
2196 map_name, btf_kind_str(t));
2199 sz = btf__resolve_size(btf, t->type);
2201 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2202 map_name, t->type, (ssize_t)sz);
2205 if (map_def->key_size && map_def->key_size != sz) {
2206 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2207 map_name, map_def->key_size, (ssize_t)sz);
2210 map_def->key_size = sz;
2211 map_def->key_type_id = t->type;
2212 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2213 } else if (strcmp(name, "value_size") == 0) {
2216 if (!get_map_field_int(map_name, btf, m, &sz))
2218 if (map_def->value_size && map_def->value_size != sz) {
2219 pr_warn("map '%s': conflicting value size %u != %u.\n",
2220 map_name, map_def->value_size, sz);
2223 map_def->value_size = sz;
2224 map_def->parts |= MAP_DEF_VALUE_SIZE;
2225 } else if (strcmp(name, "value") == 0) {
2228 t = btf__type_by_id(btf, m->type);
2230 pr_warn("map '%s': value type [%d] not found.\n",
2234 if (!btf_is_ptr(t)) {
2235 pr_warn("map '%s': value spec is not PTR: %s.\n",
2236 map_name, btf_kind_str(t));
2239 sz = btf__resolve_size(btf, t->type);
2241 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2242 map_name, t->type, (ssize_t)sz);
2245 if (map_def->value_size && map_def->value_size != sz) {
2246 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2247 map_name, map_def->value_size, (ssize_t)sz);
2250 map_def->value_size = sz;
2251 map_def->value_type_id = t->type;
2252 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2254 else if (strcmp(name, "values") == 0) {
2255 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2256 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2257 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2258 char inner_map_name[128];
2262 pr_warn("map '%s': multi-level inner maps not supported.\n",
2266 if (i != vlen - 1) {
2267 pr_warn("map '%s': '%s' member should be last.\n",
2271 if (!is_map_in_map && !is_prog_array) {
2272 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2276 if (map_def->value_size && map_def->value_size != 4) {
2277 pr_warn("map '%s': conflicting value size %u != 4.\n",
2278 map_name, map_def->value_size);
2281 map_def->value_size = 4;
2282 t = btf__type_by_id(btf, m->type);
2284 pr_warn("map '%s': %s type [%d] not found.\n",
2285 map_name, desc, m->type);
2288 if (!btf_is_array(t) || btf_array(t)->nelems) {
2289 pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2293 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2294 if (!btf_is_ptr(t)) {
2295 pr_warn("map '%s': %s def is of unexpected kind %s.\n",
2296 map_name, desc, btf_kind_str(t));
2299 t = skip_mods_and_typedefs(btf, t->type, NULL);
2300 if (is_prog_array) {
2301 if (!btf_is_func_proto(t)) {
2302 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2303 map_name, btf_kind_str(t));
2308 if (!btf_is_struct(t)) {
2309 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2310 map_name, btf_kind_str(t));
2314 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
2315 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
2319 map_def->parts |= MAP_DEF_INNER_MAP;
2320 } else if (strcmp(name, "pinning") == 0) {
2324 pr_warn("map '%s': inner def can't be pinned.\n", map_name);
2327 if (!get_map_field_int(map_name, btf, m, &val))
2329 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
2330 pr_warn("map '%s': invalid pinning value %u.\n",
2334 map_def->pinning = val;
2335 map_def->parts |= MAP_DEF_PINNING;
2336 } else if (strcmp(name, "map_extra") == 0) {
2339 if (!get_map_field_int(map_name, btf, m, &map_extra))
2341 map_def->map_extra = map_extra;
2342 map_def->parts |= MAP_DEF_MAP_EXTRA;
2345 pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2348 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2352 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2353 pr_warn("map '%s': map type isn't specified.\n", map_name);
2360 static size_t adjust_ringbuf_sz(size_t sz)
2362 __u32 page_sz = sysconf(_SC_PAGE_SIZE);
2365 /* if user forgot to set any size, make sure they see error */
2368 /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
2369 * a power-of-2 multiple of kernel's page size. If user diligently
2370 * satisified these conditions, pass the size through.
2372 if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
2375 /* Otherwise find closest (page_sz * power_of_2) product bigger than
2376 * user-set size to satisfy both user size request and kernel
2377 * requirements and substitute correct max_entries for map creation.
2379 for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
2380 if (mul * page_sz > sz)
2381 return mul * page_sz;
2384 /* if it's impossible to satisfy the conditions (i.e., user size is
2385 * very close to UINT_MAX but is not a power-of-2 multiple of
2386 * page_size) then just return original size and let kernel reject it
2391 static bool map_is_ringbuf(const struct bpf_map *map)
2393 return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2394 map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2397 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
2399 map->def.type = def->map_type;
2400 map->def.key_size = def->key_size;
2401 map->def.value_size = def->value_size;
2402 map->def.max_entries = def->max_entries;
2403 map->def.map_flags = def->map_flags;
2404 map->map_extra = def->map_extra;
2406 map->numa_node = def->numa_node;
2407 map->btf_key_type_id = def->key_type_id;
2408 map->btf_value_type_id = def->value_type_id;
2410 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2411 if (map_is_ringbuf(map))
2412 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2414 if (def->parts & MAP_DEF_MAP_TYPE)
2415 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2417 if (def->parts & MAP_DEF_KEY_TYPE)
2418 pr_debug("map '%s': found key [%u], sz = %u.\n",
2419 map->name, def->key_type_id, def->key_size);
2420 else if (def->parts & MAP_DEF_KEY_SIZE)
2421 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2423 if (def->parts & MAP_DEF_VALUE_TYPE)
2424 pr_debug("map '%s': found value [%u], sz = %u.\n",
2425 map->name, def->value_type_id, def->value_size);
2426 else if (def->parts & MAP_DEF_VALUE_SIZE)
2427 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2429 if (def->parts & MAP_DEF_MAX_ENTRIES)
2430 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2431 if (def->parts & MAP_DEF_MAP_FLAGS)
2432 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2433 if (def->parts & MAP_DEF_MAP_EXTRA)
2434 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2435 (unsigned long long)def->map_extra);
2436 if (def->parts & MAP_DEF_PINNING)
2437 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2438 if (def->parts & MAP_DEF_NUMA_NODE)
2439 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2441 if (def->parts & MAP_DEF_INNER_MAP)
2442 pr_debug("map '%s': found inner map definition.\n", map->name);
2445 static const char *btf_var_linkage_str(__u32 linkage)
2448 case BTF_VAR_STATIC: return "static";
2449 case BTF_VAR_GLOBAL_ALLOCATED: return "global";
2450 case BTF_VAR_GLOBAL_EXTERN: return "extern";
2451 default: return "unknown";
2455 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2456 const struct btf_type *sec,
2457 int var_idx, int sec_idx,
2458 const Elf_Data *data, bool strict,
2459 const char *pin_root_path)
2461 struct btf_map_def map_def = {}, inner_def = {};
2462 const struct btf_type *var, *def;
2463 const struct btf_var_secinfo *vi;
2464 const struct btf_var *var_extra;
2465 const char *map_name;
2466 struct bpf_map *map;
2469 vi = btf_var_secinfos(sec) + var_idx;
2470 var = btf__type_by_id(obj->btf, vi->type);
2471 var_extra = btf_var(var);
2472 map_name = btf__name_by_offset(obj->btf, var->name_off);
2474 if (map_name == NULL || map_name[0] == '\0') {
2475 pr_warn("map #%d: empty name.\n", var_idx);
2478 if ((__u64)vi->offset + vi->size > data->d_size) {
2479 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2482 if (!btf_is_var(var)) {
2483 pr_warn("map '%s': unexpected var kind %s.\n",
2484 map_name, btf_kind_str(var));
2487 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2488 pr_warn("map '%s': unsupported map linkage %s.\n",
2489 map_name, btf_var_linkage_str(var_extra->linkage));
2493 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2494 if (!btf_is_struct(def)) {
2495 pr_warn("map '%s': unexpected def kind %s.\n",
2496 map_name, btf_kind_str(var));
2499 if (def->size > vi->size) {
2500 pr_warn("map '%s': invalid def size.\n", map_name);
2504 map = bpf_object__add_map(obj);
2506 return PTR_ERR(map);
2507 map->name = strdup(map_name);
2509 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2512 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2513 map->def.type = BPF_MAP_TYPE_UNSPEC;
2514 map->sec_idx = sec_idx;
2515 map->sec_offset = vi->offset;
2516 map->btf_var_idx = var_idx;
2517 pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2518 map_name, map->sec_idx, map->sec_offset);
2520 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2524 fill_map_from_def(map, &map_def);
2526 if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
2527 err = build_map_pin_path(map, pin_root_path);
2529 pr_warn("map '%s': couldn't build pin path.\n", map->name);
2534 if (map_def.parts & MAP_DEF_INNER_MAP) {
2535 map->inner_map = calloc(1, sizeof(*map->inner_map));
2536 if (!map->inner_map)
2538 map->inner_map->fd = -1;
2539 map->inner_map->sec_idx = sec_idx;
2540 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2541 if (!map->inner_map->name)
2543 sprintf(map->inner_map->name, "%s.inner", map_name);
2545 fill_map_from_def(map->inner_map, &inner_def);
2548 err = bpf_map_find_btf_info(obj, map);
2555 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2556 const char *pin_root_path)
2558 const struct btf_type *sec = NULL;
2559 int nr_types, i, vlen, err;
2560 const struct btf_type *t;
2565 if (obj->efile.btf_maps_shndx < 0)
2568 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2569 data = elf_sec_data(obj, scn);
2570 if (!scn || !data) {
2571 pr_warn("elf: failed to get %s map definitions for %s\n",
2572 MAPS_ELF_SEC, obj->path);
2576 nr_types = btf__type_cnt(obj->btf);
2577 for (i = 1; i < nr_types; i++) {
2578 t = btf__type_by_id(obj->btf, i);
2579 if (!btf_is_datasec(t))
2581 name = btf__name_by_offset(obj->btf, t->name_off);
2582 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2584 obj->efile.btf_maps_sec_btf_id = i;
2590 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2594 vlen = btf_vlen(sec);
2595 for (i = 0; i < vlen; i++) {
2596 err = bpf_object__init_user_btf_map(obj, sec, i,
2597 obj->efile.btf_maps_shndx,
2607 static int bpf_object__init_maps(struct bpf_object *obj,
2608 const struct bpf_object_open_opts *opts)
2610 const char *pin_root_path;
2614 strict = !OPTS_GET(opts, relaxed_maps, false);
2615 pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2617 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2618 err = err ?: bpf_object__init_global_data_maps(obj);
2619 err = err ?: bpf_object__init_kconfig_map(obj);
2620 err = err ?: bpf_object__init_struct_ops_maps(obj);
2625 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2629 sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));
2633 return sh->sh_flags & SHF_EXECINSTR;
2636 static bool btf_needs_sanitization(struct bpf_object *obj)
2638 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2639 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2640 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2641 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2642 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2643 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2644 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2646 return !has_func || !has_datasec || !has_func_global || !has_float ||
2647 !has_decl_tag || !has_type_tag || !has_enum64;
2650 static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2652 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2653 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2654 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2655 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2656 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2657 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2658 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2659 int enum64_placeholder_id = 0;
2663 for (i = 1; i < btf__type_cnt(btf); i++) {
2664 t = (struct btf_type *)btf__type_by_id(btf, i);
2666 if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) {
2667 /* replace VAR/DECL_TAG with INT */
2668 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2670 * using size = 1 is the safest choice, 4 will be too
2671 * big and cause kernel BTF validation failure if
2672 * original variable took less than 4 bytes
2675 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2676 } else if (!has_datasec && btf_is_datasec(t)) {
2677 /* replace DATASEC with STRUCT */
2678 const struct btf_var_secinfo *v = btf_var_secinfos(t);
2679 struct btf_member *m = btf_members(t);
2680 struct btf_type *vt;
2683 name = (char *)btf__name_by_offset(btf, t->name_off);
2691 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2692 for (j = 0; j < vlen; j++, v++, m++) {
2693 /* order of field assignments is important */
2694 m->offset = v->offset * 8;
2696 /* preserve variable name as member name */
2697 vt = (void *)btf__type_by_id(btf, v->type);
2698 m->name_off = vt->name_off;
2700 } else if (!has_func && btf_is_func_proto(t)) {
2701 /* replace FUNC_PROTO with ENUM */
2703 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2704 t->size = sizeof(__u32); /* kernel enforced */
2705 } else if (!has_func && btf_is_func(t)) {
2706 /* replace FUNC with TYPEDEF */
2707 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2708 } else if (!has_func_global && btf_is_func(t)) {
2709 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2710 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2711 } else if (!has_float && btf_is_float(t)) {
2712 /* replace FLOAT with an equally-sized empty STRUCT;
2713 * since C compilers do not accept e.g. "float" as a
2714 * valid struct name, make it anonymous
2717 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2718 } else if (!has_type_tag && btf_is_type_tag(t)) {
2719 /* replace TYPE_TAG with a CONST */
2721 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
2722 } else if (!has_enum64 && btf_is_enum(t)) {
2723 /* clear the kflag */
2724 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
2725 } else if (!has_enum64 && btf_is_enum64(t)) {
2726 /* replace ENUM64 with a union */
2727 struct btf_member *m;
2729 if (enum64_placeholder_id == 0) {
2730 enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
2731 if (enum64_placeholder_id < 0)
2732 return enum64_placeholder_id;
2734 t = (struct btf_type *)btf__type_by_id(btf, i);
2739 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
2740 for (j = 0; j < vlen; j++, m++) {
2741 m->type = enum64_placeholder_id;
2750 static bool libbpf_needs_btf(const struct bpf_object *obj)
2752 return obj->efile.btf_maps_shndx >= 0 ||
2753 obj->efile.st_ops_shndx >= 0 ||
2757 static bool kernel_needs_btf(const struct bpf_object *obj)
2759 return obj->efile.st_ops_shndx >= 0;
2762 static int bpf_object__init_btf(struct bpf_object *obj,
2764 Elf_Data *btf_ext_data)
2769 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2770 err = libbpf_get_error(obj->btf);
2773 pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);
2776 /* enforce 8-byte pointers for BPF-targeted BTFs */
2777 btf__set_pointer_size(obj->btf, 8);
2780 struct btf_ext_info *ext_segs[3];
2781 int seg_num, sec_num;
2784 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2785 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2788 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2789 err = libbpf_get_error(obj->btf_ext);
2791 pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",
2792 BTF_EXT_ELF_SEC, err);
2793 obj->btf_ext = NULL;
2797 /* setup .BTF.ext to ELF section mapping */
2798 ext_segs[0] = &obj->btf_ext->func_info;
2799 ext_segs[1] = &obj->btf_ext->line_info;
2800 ext_segs[2] = &obj->btf_ext->core_relo_info;
2801 for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) {
2802 struct btf_ext_info *seg = ext_segs[seg_num];
2803 const struct btf_ext_info_sec *sec;
2804 const char *sec_name;
2807 if (seg->sec_cnt == 0)
2810 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
2811 if (!seg->sec_idxs) {
2817 for_each_btf_ext_sec(seg, sec) {
2818 /* preventively increment index to avoid doing
2819 * this before every continue below
2823 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
2824 if (str_is_empty(sec_name))
2826 scn = elf_sec_by_name(obj, sec_name);
2830 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
2835 if (err && libbpf_needs_btf(obj)) {
2836 pr_warn("BTF is required, but is missing or corrupted.\n");
2842 static int compare_vsi_off(const void *_a, const void *_b)
2844 const struct btf_var_secinfo *a = _a;
2845 const struct btf_var_secinfo *b = _b;
2847 return a->offset - b->offset;
2850 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
2853 __u32 size = 0, off = 0, i, vars = btf_vlen(t);
2854 const char *name = btf__name_by_offset(btf, t->name_off);
2855 const struct btf_type *t_var;
2856 struct btf_var_secinfo *vsi;
2857 const struct btf_var *var;
2861 pr_debug("No name found in string section for DATASEC kind.\n");
2865 /* .extern datasec size and var offsets were set correctly during
2866 * extern collection step, so just skip straight to sorting variables
2871 ret = find_elf_sec_sz(obj, name, &size);
2873 pr_debug("Invalid size for section %s: %u bytes\n", name, size);
2879 for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
2880 t_var = btf__type_by_id(btf, vsi->type);
2881 if (!t_var || !btf_is_var(t_var)) {
2882 pr_debug("Non-VAR type seen in section %s\n", name);
2886 var = btf_var(t_var);
2887 if (var->linkage == BTF_VAR_STATIC)
2890 name = btf__name_by_offset(btf, t_var->name_off);
2892 pr_debug("No name found in string section for VAR kind\n");
2896 ret = find_elf_var_offset(obj, name, &off);
2898 pr_debug("No offset found in symbol table for VAR %s\n",
2907 qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
2911 static int btf_finalize_data(struct bpf_object *obj, struct btf *btf)
2914 __u32 i, n = btf__type_cnt(btf);
2916 for (i = 1; i < n; i++) {
2917 struct btf_type *t = btf_type_by_id(btf, i);
2919 /* Loader needs to fix up some of the things compiler
2920 * couldn't get its hands on while emitting BTF. This
2921 * is section size and global variable offset. We use
2922 * the info from the ELF itself for this purpose.
2924 if (btf_is_datasec(t)) {
2925 err = btf_fixup_datasec(obj, btf, t);
2931 return libbpf_err(err);
2934 static int bpf_object__finalize_btf(struct bpf_object *obj)
2941 err = btf_finalize_data(obj, obj->btf);
2943 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2950 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
2952 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2953 prog->type == BPF_PROG_TYPE_LSM)
2956 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
2957 * also need vmlinux BTF
2959 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2965 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
2967 struct bpf_program *prog;
2970 /* CO-RE relocations need kernel BTF, only when btf_custom_path
2973 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
2976 /* Support for typed ksyms needs kernel BTF */
2977 for (i = 0; i < obj->nr_extern; i++) {
2978 const struct extern_desc *ext;
2980 ext = &obj->externs[i];
2981 if (ext->type == EXT_KSYM && ext->ksym.type_id)
2985 bpf_object__for_each_program(prog, obj) {
2986 if (!prog->autoload)
2988 if (prog_needs_vmlinux_btf(prog))
2995 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
2999 /* btf_vmlinux could be loaded earlier */
3000 if (obj->btf_vmlinux || obj->gen_loader)
3003 if (!force && !obj_needs_vmlinux_btf(obj))
3006 obj->btf_vmlinux = btf__load_vmlinux_btf();
3007 err = libbpf_get_error(obj->btf_vmlinux);
3009 pr_warn("Error loading vmlinux BTF: %d\n", err);
3010 obj->btf_vmlinux = NULL;
3016 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
3018 struct btf *kern_btf = obj->btf;
3019 bool btf_mandatory, sanitize;
3025 if (!kernel_supports(obj, FEAT_BTF)) {
3026 if (kernel_needs_btf(obj)) {
3030 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
3034 /* Even though some subprogs are global/weak, user might prefer more
3035 * permissive BPF verification process that BPF verifier performs for
3036 * static functions, taking into account more context from the caller
3037 * functions. In such case, they need to mark such subprogs with
3038 * __attribute__((visibility("hidden"))) and libbpf will adjust
3039 * corresponding FUNC BTF type to be marked as static and trigger more
3040 * involved BPF verification process.
3042 for (i = 0; i < obj->nr_programs; i++) {
3043 struct bpf_program *prog = &obj->programs[i];
3048 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3051 n = btf__type_cnt(obj->btf);
3052 for (j = 1; j < n; j++) {
3053 t = btf_type_by_id(obj->btf, j);
3054 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)
3057 name = btf__str_by_offset(obj->btf, t->name_off);
3058 if (strcmp(name, prog->name) != 0)
3061 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3066 sanitize = btf_needs_sanitization(obj);
3068 const void *raw_data;
3071 /* clone BTF to sanitize a copy and leave the original intact */
3072 raw_data = btf__raw_data(obj->btf, &sz);
3073 kern_btf = btf__new(raw_data, sz);
3074 err = libbpf_get_error(kern_btf);
3078 /* enforce 8-byte pointers for BPF-targeted BTFs */
3079 btf__set_pointer_size(obj->btf, 8);
3080 err = bpf_object__sanitize_btf(obj, kern_btf);
3085 if (obj->gen_loader) {
3087 const void *raw_data = btf__raw_data(kern_btf, &raw_size);
3091 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3092 /* Pretend to have valid FD to pass various fd >= 0 checks.
3093 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3095 btf__set_fd(kern_btf, 0);
3097 /* currently BPF_BTF_LOAD only supports log_level 1 */
3098 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3099 obj->log_level ? 1 : 0);
3103 /* move fd to libbpf's BTF */
3104 btf__set_fd(obj->btf, btf__fd(kern_btf));
3105 btf__set_fd(kern_btf, -1);
3107 btf__free(kern_btf);
3111 btf_mandatory = kernel_needs_btf(obj);
3112 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
3113 btf_mandatory ? "BTF is mandatory, can't proceed."
3114 : "BTF is optional, ignoring.");
3121 static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
3125 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3127 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3128 off, obj->path, elf_errmsg(-1));
3135 static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
3139 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3141 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3142 off, obj->path, elf_errmsg(-1));
3149 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
3153 scn = elf_getscn(obj->efile.elf, idx);
3155 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
3156 idx, obj->path, elf_errmsg(-1));
3162 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
3164 Elf_Scn *scn = NULL;
3165 Elf *elf = obj->efile.elf;
3166 const char *sec_name;
3168 while ((scn = elf_nextscn(elf, scn)) != NULL) {
3169 sec_name = elf_sec_name(obj, scn);
3173 if (strcmp(sec_name, name) != 0)
3181 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn)
3188 shdr = elf64_getshdr(scn);
3190 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
3191 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3198 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
3206 sh = elf_sec_hdr(obj, scn);
3210 name = elf_sec_str(obj, sh->sh_name);
3212 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
3213 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3220 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
3227 data = elf_getdata(scn, 0);
3229 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
3230 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
3231 obj->path, elf_errmsg(-1));
3238 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx)
3240 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3243 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3246 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx)
3248 if (idx >= data->d_size / sizeof(Elf64_Rel))
3251 return (Elf64_Rel *)data->d_buf + idx;
3254 static bool is_sec_name_dwarf(const char *name)
3256 /* approximation, but the actual list is too long */
3257 return str_has_pfx(name, ".debug_");
3260 static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name)
3262 /* no special handling of .strtab */
3263 if (hdr->sh_type == SHT_STRTAB)
3266 /* ignore .llvm_addrsig section as well */
3267 if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3270 /* no subprograms will lead to an empty .text section, ignore it */
3271 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3272 strcmp(name, ".text") == 0)
3275 /* DWARF sections */
3276 if (is_sec_name_dwarf(name))
3279 if (str_has_pfx(name, ".rel")) {
3280 name += sizeof(".rel") - 1;
3281 /* DWARF section relocations */
3282 if (is_sec_name_dwarf(name))
3285 /* .BTF and .BTF.ext don't need relocations */
3286 if (strcmp(name, BTF_ELF_SEC) == 0 ||
3287 strcmp(name, BTF_EXT_ELF_SEC) == 0)
3294 static int cmp_progs(const void *_a, const void *_b)
3296 const struct bpf_program *a = _a;
3297 const struct bpf_program *b = _b;
3299 if (a->sec_idx != b->sec_idx)
3300 return a->sec_idx < b->sec_idx ? -1 : 1;
3302 /* sec_insn_off can't be the same within the section */
3303 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3306 static int bpf_object__elf_collect(struct bpf_object *obj)
3308 struct elf_sec_desc *sec_desc;
3309 Elf *elf = obj->efile.elf;
3310 Elf_Data *btf_ext_data = NULL;
3311 Elf_Data *btf_data = NULL;
3312 int idx = 0, err = 0;
3318 /* ELF section indices are 0-based, but sec #0 is special "invalid"
3319 * section. Since section count retrieved by elf_getshdrnum() does
3320 * include sec #0, it is already the necessary size of an array to keep
3323 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3324 pr_warn("elf: failed to get the number of sections for %s: %s\n",
3325 obj->path, elf_errmsg(-1));
3326 return -LIBBPF_ERRNO__FORMAT;
3328 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3329 if (!obj->efile.secs)
3332 /* a bunch of ELF parsing functionality depends on processing symbols,
3333 * so do the first pass and find the symbol table
3336 while ((scn = elf_nextscn(elf, scn)) != NULL) {
3337 sh = elf_sec_hdr(obj, scn);
3339 return -LIBBPF_ERRNO__FORMAT;
3341 if (sh->sh_type == SHT_SYMTAB) {
3342 if (obj->efile.symbols) {
3343 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3344 return -LIBBPF_ERRNO__FORMAT;
3347 data = elf_sec_data(obj, scn);
3349 return -LIBBPF_ERRNO__FORMAT;
3351 idx = elf_ndxscn(scn);
3353 obj->efile.symbols = data;
3354 obj->efile.symbols_shndx = idx;
3355 obj->efile.strtabidx = sh->sh_link;
3359 if (!obj->efile.symbols) {
3360 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
3366 while ((scn = elf_nextscn(elf, scn)) != NULL) {
3367 idx = elf_ndxscn(scn);
3368 sec_desc = &obj->efile.secs[idx];
3370 sh = elf_sec_hdr(obj, scn);
3372 return -LIBBPF_ERRNO__FORMAT;
3374 name = elf_sec_str(obj, sh->sh_name);
3376 return -LIBBPF_ERRNO__FORMAT;
3378 if (ignore_elf_section(sh, name))
3381 data = elf_sec_data(obj, scn);
3383 return -LIBBPF_ERRNO__FORMAT;
3385 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
3386 idx, name, (unsigned long)data->d_size,
3387 (int)sh->sh_link, (unsigned long)sh->sh_flags,
3390 if (strcmp(name, "license") == 0) {
3391 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3394 } else if (strcmp(name, "version") == 0) {
3395 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3398 } else if (strcmp(name, "maps") == 0) {
3399 pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");
3401 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
3402 obj->efile.btf_maps_shndx = idx;
3403 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
3404 if (sh->sh_type != SHT_PROGBITS)
3405 return -LIBBPF_ERRNO__FORMAT;
3407 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
3408 if (sh->sh_type != SHT_PROGBITS)
3409 return -LIBBPF_ERRNO__FORMAT;
3410 btf_ext_data = data;
3411 } else if (sh->sh_type == SHT_SYMTAB) {
3412 /* already processed during the first pass above */
3413 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3414 if (sh->sh_flags & SHF_EXECINSTR) {
3415 if (strcmp(name, ".text") == 0)
3416 obj->efile.text_shndx = idx;
3417 err = bpf_object__add_programs(obj, data, name, idx);
3420 } else if (strcmp(name, DATA_SEC) == 0 ||
3421 str_has_pfx(name, DATA_SEC ".")) {
3422 sec_desc->sec_type = SEC_DATA;
3423 sec_desc->shdr = sh;
3424 sec_desc->data = data;
3425 } else if (strcmp(name, RODATA_SEC) == 0 ||
3426 str_has_pfx(name, RODATA_SEC ".")) {
3427 sec_desc->sec_type = SEC_RODATA;
3428 sec_desc->shdr = sh;
3429 sec_desc->data = data;
3430 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
3431 obj->efile.st_ops_data = data;
3432 obj->efile.st_ops_shndx = idx;
3434 pr_info("elf: skipping unrecognized data section(%d) %s\n",
3437 } else if (sh->sh_type == SHT_REL) {
3438 int targ_sec_idx = sh->sh_info; /* points to other section */
3440 if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3441 targ_sec_idx >= obj->efile.sec_cnt)
3442 return -LIBBPF_ERRNO__FORMAT;
3444 /* Only do relo for section with exec instructions */
3445 if (!section_have_execinstr(obj, targ_sec_idx) &&
3446 strcmp(name, ".rel" STRUCT_OPS_SEC) &&
3447 strcmp(name, ".rel" MAPS_ELF_SEC)) {
3448 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
3449 idx, name, targ_sec_idx,
3450 elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");
3454 sec_desc->sec_type = SEC_RELO;
3455 sec_desc->shdr = sh;
3456 sec_desc->data = data;
3457 } else if (sh->sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) {
3458 sec_desc->sec_type = SEC_BSS;
3459 sec_desc->shdr = sh;
3460 sec_desc->data = data;
3462 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
3463 (size_t)sh->sh_size);
3467 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3468 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3469 return -LIBBPF_ERRNO__FORMAT;
3472 /* sort BPF programs by section name and in-section instruction offset
3473 * for faster search */
3474 if (obj->nr_programs)
3475 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3477 return bpf_object__init_btf(obj, btf_data, btf_ext_data);
3480 static bool sym_is_extern(const Elf64_Sym *sym)
3482 int bind = ELF64_ST_BIND(sym->st_info);
3483 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
3484 return sym->st_shndx == SHN_UNDEF &&
3485 (bind == STB_GLOBAL || bind == STB_WEAK) &&
3486 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3489 static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx)
3491 int bind = ELF64_ST_BIND(sym->st_info);
3492 int type = ELF64_ST_TYPE(sym->st_info);
3494 /* in .text section */
3495 if (sym->st_shndx != text_shndx)
3498 /* local function */
3499 if (bind == STB_LOCAL && type == STT_SECTION)
3502 /* global function */
3503 return bind == STB_GLOBAL && type == STT_FUNC;
3506 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
3508 const struct btf_type *t;
3515 n = btf__type_cnt(btf);
3516 for (i = 1; i < n; i++) {
3517 t = btf__type_by_id(btf, i);
3519 if (!btf_is_var(t) && !btf_is_func(t))
3522 tname = btf__name_by_offset(btf, t->name_off);
3523 if (strcmp(tname, ext_name))
3526 if (btf_is_var(t) &&
3527 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3530 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
3539 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3540 const struct btf_var_secinfo *vs;
3541 const struct btf_type *t;
3547 n = btf__type_cnt(btf);
3548 for (i = 1; i < n; i++) {
3549 t = btf__type_by_id(btf, i);
3551 if (!btf_is_datasec(t))
3554 vs = btf_var_secinfos(t);
3555 for (j = 0; j < btf_vlen(t); j++, vs++) {
3556 if (vs->type == ext_btf_id)
3564 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3567 const struct btf_type *t;
3570 t = skip_mods_and_typedefs(btf, id, NULL);
3571 name = btf__name_by_offset(btf, t->name_off);
3575 switch (btf_kind(t)) {
3576 case BTF_KIND_INT: {
3577 int enc = btf_int_encoding(t);
3579 if (enc & BTF_INT_BOOL)
3580 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3582 *is_signed = enc & BTF_INT_SIGNED;
3585 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3586 return KCFG_UNKNOWN;
3591 return KCFG_UNKNOWN;
3592 if (strcmp(name, "libbpf_tristate"))
3593 return KCFG_UNKNOWN;
3594 return KCFG_TRISTATE;
3595 case BTF_KIND_ENUM64:
3596 if (strcmp(name, "libbpf_tristate"))
3597 return KCFG_UNKNOWN;
3598 return KCFG_TRISTATE;
3599 case BTF_KIND_ARRAY:
3600 if (btf_array(t)->nelems == 0)
3601 return KCFG_UNKNOWN;
3602 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3603 return KCFG_UNKNOWN;
3604 return KCFG_CHAR_ARR;
3606 return KCFG_UNKNOWN;
3610 static int cmp_externs(const void *_a, const void *_b)
3612 const struct extern_desc *a = _a;
3613 const struct extern_desc *b = _b;
3615 if (a->type != b->type)
3616 return a->type < b->type ? -1 : 1;
3618 if (a->type == EXT_KCFG) {
3619 /* descending order by alignment requirements */
3620 if (a->kcfg.align != b->kcfg.align)
3621 return a->kcfg.align > b->kcfg.align ? -1 : 1;
3622 /* ascending order by size, within same alignment class */
3623 if (a->kcfg.sz != b->kcfg.sz)
3624 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3627 /* resolve ties by name */
3628 return strcmp(a->name, b->name);
3631 static int find_int_btf_id(const struct btf *btf)
3633 const struct btf_type *t;
3636 n = btf__type_cnt(btf);
3637 for (i = 1; i < n; i++) {
3638 t = btf__type_by_id(btf, i);
3640 if (btf_is_int(t) && btf_int_bits(t) == 32)
3647 static int add_dummy_ksym_var(struct btf *btf)
3649 int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
3650 const struct btf_var_secinfo *vs;
3651 const struct btf_type *sec;
3656 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
3661 sec = btf__type_by_id(btf, sec_btf_id);
3662 vs = btf_var_secinfos(sec);
3663 for (i = 0; i < btf_vlen(sec); i++, vs++) {
3664 const struct btf_type *vt;
3666 vt = btf__type_by_id(btf, vs->type);
3667 if (btf_is_func(vt))
3671 /* No func in ksyms sec. No need to add dummy var. */
3672 if (i == btf_vlen(sec))
3675 int_btf_id = find_int_btf_id(btf);
3676 dummy_var_btf_id = btf__add_var(btf,
3678 BTF_VAR_GLOBAL_ALLOCATED,
3680 if (dummy_var_btf_id < 0)
3681 pr_warn("cannot create a dummy_ksym var\n");
3683 return dummy_var_btf_id;
3686 static int bpf_object__collect_externs(struct bpf_object *obj)
3688 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3689 const struct btf_type *t;
3690 struct extern_desc *ext;
3691 int i, n, off, dummy_var_btf_id;
3692 const char *ext_name, *sec_name;
3696 if (!obj->efile.symbols)
3699 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3700 sh = elf_sec_hdr(obj, scn);
3701 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
3702 return -LIBBPF_ERRNO__FORMAT;
3704 dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
3705 if (dummy_var_btf_id < 0)
3706 return dummy_var_btf_id;
3708 n = sh->sh_size / sh->sh_entsize;
3709 pr_debug("looking for externs among %d symbols...\n", n);
3711 for (i = 0; i < n; i++) {
3712 Elf64_Sym *sym = elf_sym_by_idx(obj, i);
3715 return -LIBBPF_ERRNO__FORMAT;
3716 if (!sym_is_extern(sym))
3718 ext_name = elf_sym_str(obj, sym->st_name);
3719 if (!ext_name || !ext_name[0])
3723 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3727 ext = &ext[obj->nr_extern];
3728 memset(ext, 0, sizeof(*ext));
3731 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3732 if (ext->btf_id <= 0) {
3733 pr_warn("failed to find BTF for extern '%s': %d\n",
3734 ext_name, ext->btf_id);
3737 t = btf__type_by_id(obj->btf, ext->btf_id);
3738 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3740 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
3742 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3743 if (ext->sec_btf_id <= 0) {
3744 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3745 ext_name, ext->btf_id, ext->sec_btf_id);
3746 return ext->sec_btf_id;
3748 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3749 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3751 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3752 if (btf_is_func(t)) {
3753 pr_warn("extern function %s is unsupported under %s section\n",
3754 ext->name, KCONFIG_SEC);
3758 ext->type = EXT_KCFG;
3759 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3760 if (ext->kcfg.sz <= 0) {
3761 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3762 ext_name, ext->kcfg.sz);
3763 return ext->kcfg.sz;
3765 ext->kcfg.align = btf__align_of(obj->btf, t->type);
3766 if (ext->kcfg.align <= 0) {
3767 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3768 ext_name, ext->kcfg.align);
3771 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3772 &ext->kcfg.is_signed);
3773 if (ext->kcfg.type == KCFG_UNKNOWN) {
3774 pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);
3777 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3779 ext->type = EXT_KSYM;
3780 skip_mods_and_typedefs(obj->btf, t->type,
3781 &ext->ksym.type_id);
3783 pr_warn("unrecognized extern section '%s'\n", sec_name);
3787 pr_debug("collected %d externs total\n", obj->nr_extern);
3789 if (!obj->nr_extern)
3792 /* sort externs by type, for kcfg ones also by (align, size, name) */
3793 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3795 /* for .ksyms section, we need to turn all externs into allocated
3796 * variables in BTF to pass kernel verification; we do this by
3797 * pretending that each extern is a 8-byte variable
3800 /* find existing 4-byte integer type in BTF to use for fake
3801 * extern variables in DATASEC
3803 int int_btf_id = find_int_btf_id(obj->btf);
3804 /* For extern function, a dummy_var added earlier
3805 * will be used to replace the vs->type and
3806 * its name string will be used to refill
3807 * the missing param's name.
3809 const struct btf_type *dummy_var;
3811 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
3812 for (i = 0; i < obj->nr_extern; i++) {
3813 ext = &obj->externs[i];
3814 if (ext->type != EXT_KSYM)
3816 pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
3817 i, ext->sym_idx, ext->name);
3822 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
3823 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3824 struct btf_type *vt;
3826 vt = (void *)btf__type_by_id(obj->btf, vs->type);
3827 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3828 ext = find_extern_by_name(obj, ext_name);
3830 pr_warn("failed to find extern definition for BTF %s '%s'\n",
3831 btf_kind_str(vt), ext_name);
3834 if (btf_is_func(vt)) {
3835 const struct btf_type *func_proto;
3836 struct btf_param *param;
3839 func_proto = btf__type_by_id(obj->btf,
3841 param = btf_params(func_proto);
3842 /* Reuse the dummy_var string if the
3843 * func proto does not have param name.
3845 for (j = 0; j < btf_vlen(func_proto); j++)
3846 if (param[j].type && !param[j].name_off)
3848 dummy_var->name_off;
3849 vs->type = dummy_var_btf_id;
3850 vt->info &= ~0xffff;
3851 vt->info |= BTF_FUNC_GLOBAL;
3853 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3854 vt->type = int_btf_id;
3857 vs->size = sizeof(int);
3864 /* for kcfg externs calculate their offsets within a .kconfig map */
3866 for (i = 0; i < obj->nr_extern; i++) {
3867 ext = &obj->externs[i];
3868 if (ext->type != EXT_KCFG)
3871 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3872 off = ext->kcfg.data_off + ext->kcfg.sz;
3873 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3874 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3878 for (i = 0; i < n; i++) {
3879 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3881 t = btf__type_by_id(obj->btf, vs->type);
3882 ext_name = btf__name_by_offset(obj->btf, t->name_off);
3883 ext = find_extern_by_name(obj, ext_name);
3885 pr_warn("failed to find extern definition for BTF var '%s'\n",
3889 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3890 vs->offset = ext->kcfg.data_off;
3896 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
3898 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3901 struct bpf_program *
3902 bpf_object__find_program_by_name(const struct bpf_object *obj,
3905 struct bpf_program *prog;
3907 bpf_object__for_each_program(prog, obj) {
3908 if (prog_is_subprog(obj, prog))
3910 if (!strcmp(prog->name, name))
3913 return errno = ENOENT, NULL;
3916 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3919 switch (obj->efile.secs[shndx].sec_type) {
3929 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3932 return shndx == obj->efile.btf_maps_shndx;
3935 static enum libbpf_map_type
3936 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3938 if (shndx == obj->efile.symbols_shndx)
3939 return LIBBPF_MAP_KCONFIG;
3941 switch (obj->efile.secs[shndx].sec_type) {
3943 return LIBBPF_MAP_BSS;
3945 return LIBBPF_MAP_DATA;
3947 return LIBBPF_MAP_RODATA;
3949 return LIBBPF_MAP_UNSPEC;
3953 static int bpf_program__record_reloc(struct bpf_program *prog,
3954 struct reloc_desc *reloc_desc,
3955 __u32 insn_idx, const char *sym_name,
3956 const Elf64_Sym *sym, const Elf64_Rel *rel)
3958 struct bpf_insn *insn = &prog->insns[insn_idx];
3959 size_t map_idx, nr_maps = prog->obj->nr_maps;
3960 struct bpf_object *obj = prog->obj;
3961 __u32 shdr_idx = sym->st_shndx;
3962 enum libbpf_map_type type;
3963 const char *sym_sec_name;
3964 struct bpf_map *map;
3966 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
3967 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
3968 prog->name, sym_name, insn_idx, insn->code);
3969 return -LIBBPF_ERRNO__RELOC;
3972 if (sym_is_extern(sym)) {
3973 int sym_idx = ELF64_R_SYM(rel->r_info);
3974 int i, n = obj->nr_extern;
3975 struct extern_desc *ext;
3977 for (i = 0; i < n; i++) {
3978 ext = &obj->externs[i];
3979 if (ext->sym_idx == sym_idx)
3983 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
3984 prog->name, sym_name, sym_idx);
3985 return -LIBBPF_ERRNO__RELOC;
3987 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
3988 prog->name, i, ext->name, ext->sym_idx, insn_idx);
3989 if (insn->code == (BPF_JMP | BPF_CALL))
3990 reloc_desc->type = RELO_EXTERN_FUNC;
3992 reloc_desc->type = RELO_EXTERN_VAR;
3993 reloc_desc->insn_idx = insn_idx;
3994 reloc_desc->sym_off = i; /* sym_off stores extern index */
3998 /* sub-program call relocation */
3999 if (is_call_insn(insn)) {
4000 if (insn->src_reg != BPF_PSEUDO_CALL) {
4001 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4002 return -LIBBPF_ERRNO__RELOC;
4004 /* text_shndx can be 0, if no default "main" program exists */
4005 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4006 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4007 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
4008 prog->name, sym_name, sym_sec_name);
4009 return -LIBBPF_ERRNO__RELOC;
4011 if (sym->st_value % BPF_INSN_SZ) {
4012 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
4013 prog->name, sym_name, (size_t)sym->st_value);
4014 return -LIBBPF_ERRNO__RELOC;
4016 reloc_desc->type = RELO_CALL;
4017 reloc_desc->insn_idx = insn_idx;
4018 reloc_desc->sym_off = sym->st_value;
4022 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
4023 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
4024 prog->name, sym_name, shdr_idx);
4025 return -LIBBPF_ERRNO__RELOC;
4028 /* loading subprog addresses */
4029 if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4030 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4031 * local_func: sym->st_value = 0, insn->imm = offset in the section.
4033 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4034 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4035 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4036 return -LIBBPF_ERRNO__RELOC;
4039 reloc_desc->type = RELO_SUBPROG_ADDR;
4040 reloc_desc->insn_idx = insn_idx;
4041 reloc_desc->sym_off = sym->st_value;
4045 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
4046 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4048 /* generic map reference relocation */
4049 if (type == LIBBPF_MAP_UNSPEC) {
4050 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
4051 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4052 prog->name, sym_name, sym_sec_name);
4053 return -LIBBPF_ERRNO__RELOC;
4055 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4056 map = &obj->maps[map_idx];
4057 if (map->libbpf_type != type ||
4058 map->sec_idx != sym->st_shndx ||
4059 map->sec_offset != sym->st_value)
4061 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
4062 prog->name, map_idx, map->name, map->sec_idx,
4063 map->sec_offset, insn_idx);
4066 if (map_idx >= nr_maps) {
4067 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4068 prog->name, sym_sec_name, (size_t)sym->st_value);
4069 return -LIBBPF_ERRNO__RELOC;
4071 reloc_desc->type = RELO_LD64;
4072 reloc_desc->insn_idx = insn_idx;
4073 reloc_desc->map_idx = map_idx;
4074 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4078 /* global data map relocation */
4079 if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
4080 pr_warn("prog '%s': bad data relo against section '%s'\n",
4081 prog->name, sym_sec_name);
4082 return -LIBBPF_ERRNO__RELOC;
4084 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4085 map = &obj->maps[map_idx];
4086 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4088 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
4089 prog->name, map_idx, map->name, map->sec_idx,
4090 map->sec_offset, insn_idx);
4093 if (map_idx >= nr_maps) {
4094 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4095 prog->name, sym_sec_name);
4096 return -LIBBPF_ERRNO__RELOC;
4099 reloc_desc->type = RELO_DATA;
4100 reloc_desc->insn_idx = insn_idx;
4101 reloc_desc->map_idx = map_idx;
4102 reloc_desc->sym_off = sym->st_value;
4106 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4108 return insn_idx >= prog->sec_insn_off &&
4109 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4112 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
4113 size_t sec_idx, size_t insn_idx)
4115 int l = 0, r = obj->nr_programs - 1, m;
4116 struct bpf_program *prog;
4119 m = l + (r - l + 1) / 2;
4120 prog = &obj->programs[m];
4122 if (prog->sec_idx < sec_idx ||
4123 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4128 /* matching program could be at index l, but it still might be the
4129 * wrong one, so we need to double check conditions for the last time
4131 prog = &obj->programs[l];
4132 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4138 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)
4140 const char *relo_sec_name, *sec_name;
4141 size_t sec_idx = shdr->sh_info, sym_idx;
4142 struct bpf_program *prog;
4143 struct reloc_desc *relos;
4145 const char *sym_name;
4152 if (sec_idx >= obj->efile.sec_cnt)
4155 scn = elf_sec_by_idx(obj, sec_idx);
4156 scn_data = elf_sec_data(obj, scn);
4158 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4159 sec_name = elf_sec_name(obj, scn);
4160 if (!relo_sec_name || !sec_name)
4163 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
4164 relo_sec_name, sec_idx, sec_name);
4165 nrels = shdr->sh_size / shdr->sh_entsize;
4167 for (i = 0; i < nrels; i++) {
4168 rel = elf_rel_by_idx(data, i);
4170 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
4171 return -LIBBPF_ERRNO__FORMAT;
4174 sym_idx = ELF64_R_SYM(rel->r_info);
4175 sym = elf_sym_by_idx(obj, sym_idx);
4177 pr_warn("sec '%s': symbol #%zu not found for relo #%d\n",
4178 relo_sec_name, sym_idx, i);
4179 return -LIBBPF_ERRNO__FORMAT;
4182 if (sym->st_shndx >= obj->efile.sec_cnt) {
4183 pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n",
4184 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4185 return -LIBBPF_ERRNO__FORMAT;
4188 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4189 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
4190 relo_sec_name, (size_t)rel->r_offset, i);
4191 return -LIBBPF_ERRNO__FORMAT;
4194 insn_idx = rel->r_offset / BPF_INSN_SZ;
4195 /* relocations against static functions are recorded as
4196 * relocations against the section that contains a function;
4197 * in such case, symbol will be STT_SECTION and sym.st_name
4198 * will point to empty string (0), so fetch section name
4201 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4202 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4204 sym_name = elf_sym_str(obj, sym->st_name);
4205 sym_name = sym_name ?: "<?";
4207 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
4208 relo_sec_name, i, insn_idx, sym_name);
4210 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4212 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
4213 relo_sec_name, i, sec_name, insn_idx);
4217 relos = libbpf_reallocarray(prog->reloc_desc,
4218 prog->nr_reloc + 1, sizeof(*relos));
4221 prog->reloc_desc = relos;
4223 /* adjust insn_idx to local BPF program frame of reference */
4224 insn_idx -= prog->sec_insn_off;
4225 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4226 insn_idx, sym_name, sym, rel);
4235 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
4242 /* if it's BTF-defined map, we don't need to search for type IDs.
4243 * For struct_ops map, it does not need btf_key_type_id and
4244 * btf_value_type_id.
4246 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4250 * LLVM annotates global data differently in BTF, that is,
4251 * only as '.data', '.bss' or '.rodata'.
4253 if (!bpf_map__is_internal(map))
4256 id = btf__find_by_name(obj->btf, map->real_name);
4260 map->btf_key_type_id = 0;
4261 map->btf_value_type_id = id;
4265 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
4267 char file[PATH_MAX], buff[4096];
4272 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
4273 memset(info, 0, sizeof(*info));
4275 fp = fopen(file, "r");
4278 pr_warn("failed to open %s: %d. No procfs support?\n", file,
4283 while (fgets(buff, sizeof(buff), fp)) {
4284 if (sscanf(buff, "map_type:\t%u", &val) == 1)
4286 else if (sscanf(buff, "key_size:\t%u", &val) == 1)
4287 info->key_size = val;
4288 else if (sscanf(buff, "value_size:\t%u", &val) == 1)
4289 info->value_size = val;
4290 else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
4291 info->max_entries = val;
4292 else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
4293 info->map_flags = val;
4301 bool bpf_map__autocreate(const struct bpf_map *map)
4303 return map->autocreate;
4306 int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
4308 if (map->obj->loaded)
4309 return libbpf_err(-EBUSY);
4311 map->autocreate = autocreate;
4315 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
4317 struct bpf_map_info info;
4318 __u32 len = sizeof(info), name_len;
4322 memset(&info, 0, len);
4323 err = bpf_obj_get_info_by_fd(fd, &info, &len);
4324 if (err && errno == EINVAL)
4325 err = bpf_get_map_info_from_fdinfo(fd, &info);
4327 return libbpf_err(err);
4329 name_len = strlen(info.name);
4330 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4331 new_name = strdup(map->name);
4333 new_name = strdup(info.name);
4336 return libbpf_err(-errno);
4338 new_fd = open("/", O_RDONLY | O_CLOEXEC);
4341 goto err_free_new_name;
4344 new_fd = dup3(fd, new_fd, O_CLOEXEC);
4347 goto err_close_new_fd;
4350 err = zclose(map->fd);
4353 goto err_close_new_fd;
4358 map->name = new_name;
4359 map->def.type = info.type;
4360 map->def.key_size = info.key_size;
4361 map->def.value_size = info.value_size;
4362 map->def.max_entries = info.max_entries;
4363 map->def.map_flags = info.map_flags;
4364 map->btf_key_type_id = info.btf_key_type_id;
4365 map->btf_value_type_id = info.btf_value_type_id;
4367 map->map_extra = info.map_extra;
4375 return libbpf_err(err);
4378 __u32 bpf_map__max_entries(const struct bpf_map *map)
4380 return map->def.max_entries;
4383 struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
4385 if (!bpf_map_type__is_map_in_map(map->def.type))
4386 return errno = EINVAL, NULL;
4388 return map->inner_map;
4391 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
4393 if (map->obj->loaded)
4394 return libbpf_err(-EBUSY);
4396 map->def.max_entries = max_entries;
4398 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4399 if (map_is_ringbuf(map))
4400 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4406 bpf_object__probe_loading(struct bpf_object *obj)
4408 char *cp, errmsg[STRERR_BUFSIZE];
4409 struct bpf_insn insns[] = {
4410 BPF_MOV64_IMM(BPF_REG_0, 0),
4413 int ret, insn_cnt = ARRAY_SIZE(insns);
4415 if (obj->gen_loader)
4418 ret = bump_rlimit_memlock();
4420 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);
4422 /* make sure basic loading works */
4423 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4425 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
4428 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4429 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
4430 "program. Make sure your kernel supports BPF "
4431 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
4432 "set to big enough value.\n", __func__, cp, ret);
4440 static int probe_fd(int fd)
4447 static int probe_kern_prog_name(void)
4449 const size_t attr_sz = offsetofend(union bpf_attr, prog_name);
4450 struct bpf_insn insns[] = {
4451 BPF_MOV64_IMM(BPF_REG_0, 0),
4454 union bpf_attr attr;
4457 memset(&attr, 0, attr_sz);
4458 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4459 attr.license = ptr_to_u64("GPL");
4460 attr.insns = ptr_to_u64(insns);
4461 attr.insn_cnt = (__u32)ARRAY_SIZE(insns);
4462 libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name));
4464 /* make sure loading with name works */
4465 ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS);
4466 return probe_fd(ret);
4469 static int probe_kern_global_data(void)
4471 char *cp, errmsg[STRERR_BUFSIZE];
4472 struct bpf_insn insns[] = {
4473 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
4474 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
4475 BPF_MOV64_IMM(BPF_REG_0, 0),
4478 int ret, map, insn_cnt = ARRAY_SIZE(insns);
4480 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL);
4483 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4484 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4485 __func__, cp, -ret);
4491 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4493 return probe_fd(ret);
4496 static int probe_kern_btf(void)
4498 static const char strs[] = "\0int";
4501 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4504 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4505 strs, sizeof(strs)));
4508 static int probe_kern_btf_func(void)
4510 static const char strs[] = "\0int\0x\0a";
4511 /* void x(int a) {} */
4514 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
4515 /* FUNC_PROTO */ /* [2] */
4516 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4517 BTF_PARAM_ENC(7, 1),
4518 /* FUNC x */ /* [3] */
4519 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
4522 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4523 strs, sizeof(strs)));
4526 static int probe_kern_btf_func_global(void)
4528 static const char strs[] = "\0int\0x\0a";
4529 /* static void x(int a) {} */
4532 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
4533 /* FUNC_PROTO */ /* [2] */
4534 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4535 BTF_PARAM_ENC(7, 1),
4536 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */
4537 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
4540 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4541 strs, sizeof(strs)));
4544 static int probe_kern_btf_datasec(void)
4546 static const char strs[] = "\0x\0.data";
4550 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
4551 /* VAR x */ /* [2] */
4552 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4554 /* DATASEC val */ /* [3] */
4555 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
4556 BTF_VAR_SECINFO_ENC(2, 0, 4),
4559 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4560 strs, sizeof(strs)));
4563 static int probe_kern_btf_float(void)
4565 static const char strs[] = "\0float";
4568 BTF_TYPE_FLOAT_ENC(1, 4),
4571 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4572 strs, sizeof(strs)));
4575 static int probe_kern_btf_decl_tag(void)
4577 static const char strs[] = "\0tag";
4580 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
4581 /* VAR x */ /* [2] */
4582 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4585 BTF_TYPE_DECL_TAG_ENC(1, 2, -1),
4588 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4589 strs, sizeof(strs)));
4592 static int probe_kern_btf_type_tag(void)
4594 static const char strs[] = "\0tag";
4597 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
4599 BTF_TYPE_TYPE_TAG_ENC(1, 1), /* [2] */
4601 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */
4604 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4605 strs, sizeof(strs)));
4608 static int probe_kern_array_mmap(void)
4610 LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE);
4613 fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts);
4614 return probe_fd(fd);
4617 static int probe_kern_exp_attach_type(void)
4619 LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE);
4620 struct bpf_insn insns[] = {
4621 BPF_MOV64_IMM(BPF_REG_0, 0),
4624 int fd, insn_cnt = ARRAY_SIZE(insns);
4626 /* use any valid combination of program type and (optional)
4627 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
4628 * to see if kernel supports expected_attach_type field for
4629 * BPF_PROG_LOAD command
4631 fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts);
4632 return probe_fd(fd);
4635 static int probe_kern_probe_read_kernel(void)
4637 struct bpf_insn insns[] = {
4638 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */
4639 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */
4640 BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */
4641 BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */
4642 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
4645 int fd, insn_cnt = ARRAY_SIZE(insns);
4647 fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
4648 return probe_fd(fd);
4651 static int probe_prog_bind_map(void)
4653 char *cp, errmsg[STRERR_BUFSIZE];
4654 struct bpf_insn insns[] = {
4655 BPF_MOV64_IMM(BPF_REG_0, 0),
4658 int ret, map, prog, insn_cnt = ARRAY_SIZE(insns);
4660 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL);
4663 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4664 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4665 __func__, cp, -ret);
4669 prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4675 ret = bpf_prog_bind_map(prog, map, NULL);
4683 static int probe_module_btf(void)
4685 static const char strs[] = "\0int";
4688 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4690 struct bpf_btf_info info;
4691 __u32 len = sizeof(info);
4695 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
4697 return 0; /* BTF not supported at all */
4699 memset(&info, 0, sizeof(info));
4700 info.name = ptr_to_u64(name);
4701 info.name_len = sizeof(name);
4703 /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
4704 * kernel's module BTF support coincides with support for
4705 * name/name_len fields in struct bpf_btf_info.
4707 err = bpf_obj_get_info_by_fd(fd, &info, &len);
4712 static int probe_perf_link(void)
4714 struct bpf_insn insns[] = {
4715 BPF_MOV64_IMM(BPF_REG_0, 0),
4718 int prog_fd, link_fd, err;
4720 prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL",
4721 insns, ARRAY_SIZE(insns), NULL);
4725 /* use invalid perf_event FD to get EBADF, if link is supported;
4726 * otherwise EINVAL should be returned
4728 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
4729 err = -errno; /* close() can clobber errno */
4735 return link_fd < 0 && err == -EBADF;
4738 static int probe_kern_bpf_cookie(void)
4740 struct bpf_insn insns[] = {
4741 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie),
4744 int ret, insn_cnt = ARRAY_SIZE(insns);
4746 ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL);
4747 return probe_fd(ret);
4750 static int probe_kern_btf_enum64(void)
4752 static const char strs[] = "\0enum64";
4754 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8),
4757 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4758 strs, sizeof(strs)));
4761 static int probe_kern_syscall_wrapper(void);
4763 enum kern_feature_result {
4769 typedef int (*feature_probe_fn)(void);
4771 static struct kern_feature_desc {
4773 feature_probe_fn probe;
4774 enum kern_feature_result res;
4775 } feature_probes[__FEAT_CNT] = {
4776 [FEAT_PROG_NAME] = {
4777 "BPF program name", probe_kern_prog_name,
4779 [FEAT_GLOBAL_DATA] = {
4780 "global variables", probe_kern_global_data,
4783 "minimal BTF", probe_kern_btf,
4786 "BTF functions", probe_kern_btf_func,
4788 [FEAT_BTF_GLOBAL_FUNC] = {
4789 "BTF global function", probe_kern_btf_func_global,
4791 [FEAT_BTF_DATASEC] = {
4792 "BTF data section and variable", probe_kern_btf_datasec,
4794 [FEAT_ARRAY_MMAP] = {
4795 "ARRAY map mmap()", probe_kern_array_mmap,
4797 [FEAT_EXP_ATTACH_TYPE] = {
4798 "BPF_PROG_LOAD expected_attach_type attribute",
4799 probe_kern_exp_attach_type,
4801 [FEAT_PROBE_READ_KERN] = {
4802 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
4804 [FEAT_PROG_BIND_MAP] = {
4805 "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
4807 [FEAT_MODULE_BTF] = {
4808 "module BTF support", probe_module_btf,
4810 [FEAT_BTF_FLOAT] = {
4811 "BTF_KIND_FLOAT support", probe_kern_btf_float,
4813 [FEAT_PERF_LINK] = {
4814 "BPF perf link support", probe_perf_link,
4816 [FEAT_BTF_DECL_TAG] = {
4817 "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag,
4819 [FEAT_BTF_TYPE_TAG] = {
4820 "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag,
4822 [FEAT_MEMCG_ACCOUNT] = {
4823 "memcg-based memory accounting", probe_memcg_account,
4825 [FEAT_BPF_COOKIE] = {
4826 "BPF cookie support", probe_kern_bpf_cookie,
4828 [FEAT_BTF_ENUM64] = {
4829 "BTF_KIND_ENUM64 support", probe_kern_btf_enum64,
4831 [FEAT_SYSCALL_WRAPPER] = {
4832 "Kernel using syscall wrapper", probe_kern_syscall_wrapper,
4836 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
4838 struct kern_feature_desc *feat = &feature_probes[feat_id];
4841 if (obj && obj->gen_loader)
4842 /* To generate loader program assume the latest kernel
4843 * to avoid doing extra prog_load, map_create syscalls.
4847 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
4848 ret = feat->probe();
4850 WRITE_ONCE(feat->res, FEAT_SUPPORTED);
4851 } else if (ret == 0) {
4852 WRITE_ONCE(feat->res, FEAT_MISSING);
4854 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
4855 WRITE_ONCE(feat->res, FEAT_MISSING);
4859 return READ_ONCE(feat->res) == FEAT_SUPPORTED;
4862 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4864 struct bpf_map_info map_info;
4865 char msg[STRERR_BUFSIZE];
4866 __u32 map_info_len = sizeof(map_info);
4869 memset(&map_info, 0, map_info_len);
4870 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len);
4871 if (err && errno == EINVAL)
4872 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
4874 pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
4875 libbpf_strerror_r(errno, msg, sizeof(msg)));
4879 return (map_info.type == map->def.type &&
4880 map_info.key_size == map->def.key_size &&
4881 map_info.value_size == map->def.value_size &&
4882 map_info.max_entries == map->def.max_entries &&
4883 map_info.map_flags == map->def.map_flags &&
4884 map_info.map_extra == map->map_extra);
4888 bpf_object__reuse_map(struct bpf_map *map)
4890 char *cp, errmsg[STRERR_BUFSIZE];
4893 pin_fd = bpf_obj_get(map->pin_path);
4896 if (err == -ENOENT) {
4897 pr_debug("found no pinned map to reuse at '%s'\n",
4902 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4903 pr_warn("couldn't retrieve pinned map '%s': %s\n",
4908 if (!map_is_reuse_compat(map, pin_fd)) {
4909 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
4915 err = bpf_map__reuse_fd(map, pin_fd);
4921 pr_debug("reused pinned map at '%s'\n", map->pin_path);
4927 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
4929 enum libbpf_map_type map_type = map->libbpf_type;
4930 char *cp, errmsg[STRERR_BUFSIZE];
4933 if (obj->gen_loader) {
4934 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
4935 map->mmaped, map->def.value_size);
4936 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG)
4937 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
4940 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
4943 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4944 pr_warn("Error setting initial map(%s) contents: %s\n",
4949 /* Freeze .rodata and .kconfig map as read-only from syscall side. */
4950 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
4951 err = bpf_map_freeze(map->fd);
4954 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4955 pr_warn("Error freezing map(%s) as read-only: %s\n",
4963 static void bpf_map__destroy(struct bpf_map *map);
4965 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
4967 LIBBPF_OPTS(bpf_map_create_opts, create_attr);
4968 struct bpf_map_def *def = &map->def;
4969 const char *map_name = NULL;
4972 if (kernel_supports(obj, FEAT_PROG_NAME))
4973 map_name = map->name;
4974 create_attr.map_ifindex = map->map_ifindex;
4975 create_attr.map_flags = def->map_flags;
4976 create_attr.numa_node = map->numa_node;
4977 create_attr.map_extra = map->map_extra;
4979 if (bpf_map__is_struct_ops(map))
4980 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
4982 if (obj->btf && btf__fd(obj->btf) >= 0) {
4983 create_attr.btf_fd = btf__fd(obj->btf);
4984 create_attr.btf_key_type_id = map->btf_key_type_id;
4985 create_attr.btf_value_type_id = map->btf_value_type_id;
4988 if (bpf_map_type__is_map_in_map(def->type)) {
4989 if (map->inner_map) {
4990 err = bpf_object__create_map(obj, map->inner_map, true);
4992 pr_warn("map '%s': failed to create inner map: %d\n",
4996 map->inner_map_fd = bpf_map__fd(map->inner_map);
4998 if (map->inner_map_fd >= 0)
4999 create_attr.inner_map_fd = map->inner_map_fd;
5002 switch (def->type) {
5003 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
5004 case BPF_MAP_TYPE_CGROUP_ARRAY:
5005 case BPF_MAP_TYPE_STACK_TRACE:
5006 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
5007 case BPF_MAP_TYPE_HASH_OF_MAPS:
5008 case BPF_MAP_TYPE_DEVMAP:
5009 case BPF_MAP_TYPE_DEVMAP_HASH:
5010 case BPF_MAP_TYPE_CPUMAP:
5011 case BPF_MAP_TYPE_XSKMAP:
5012 case BPF_MAP_TYPE_SOCKMAP:
5013 case BPF_MAP_TYPE_SOCKHASH:
5014 case BPF_MAP_TYPE_QUEUE:
5015 case BPF_MAP_TYPE_STACK:
5016 create_attr.btf_fd = 0;
5017 create_attr.btf_key_type_id = 0;
5018 create_attr.btf_value_type_id = 0;
5019 map->btf_key_type_id = 0;
5020 map->btf_value_type_id = 0;
5025 if (obj->gen_loader) {
5026 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5027 def->key_size, def->value_size, def->max_entries,
5028 &create_attr, is_inner ? -1 : map - obj->maps);
5029 /* Pretend to have valid FD to pass various fd >= 0 checks.
5030 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
5034 map->fd = bpf_map_create(def->type, map_name,
5035 def->key_size, def->value_size,
5036 def->max_entries, &create_attr);
5038 if (map->fd < 0 && (create_attr.btf_key_type_id ||
5039 create_attr.btf_value_type_id)) {
5040 char *cp, errmsg[STRERR_BUFSIZE];
5043 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5044 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
5045 map->name, cp, err);
5046 create_attr.btf_fd = 0;
5047 create_attr.btf_key_type_id = 0;
5048 create_attr.btf_value_type_id = 0;
5049 map->btf_key_type_id = 0;
5050 map->btf_value_type_id = 0;
5051 map->fd = bpf_map_create(def->type, map_name,
5052 def->key_size, def->value_size,
5053 def->max_entries, &create_attr);
5056 err = map->fd < 0 ? -errno : 0;
5058 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
5059 if (obj->gen_loader)
5060 map->inner_map->fd = -1;
5061 bpf_map__destroy(map->inner_map);
5062 zfree(&map->inner_map);
5068 static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map)
5070 const struct bpf_map *targ_map;
5074 for (i = 0; i < map->init_slots_sz; i++) {
5075 if (!map->init_slots[i])
5078 targ_map = map->init_slots[i];
5079 fd = bpf_map__fd(targ_map);
5081 if (obj->gen_loader) {
5082 bpf_gen__populate_outer_map(obj->gen_loader,
5084 targ_map - obj->maps);
5086 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5090 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
5091 map->name, i, targ_map->name, fd, err);
5094 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
5095 map->name, i, targ_map->name, fd);
5098 zfree(&map->init_slots);
5099 map->init_slots_sz = 0;
5104 static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map)
5106 const struct bpf_program *targ_prog;
5110 if (obj->gen_loader)
5113 for (i = 0; i < map->init_slots_sz; i++) {
5114 if (!map->init_slots[i])
5117 targ_prog = map->init_slots[i];
5118 fd = bpf_program__fd(targ_prog);
5120 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5123 pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",
5124 map->name, i, targ_prog->name, fd, err);
5127 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
5128 map->name, i, targ_prog->name, fd);
5131 zfree(&map->init_slots);
5132 map->init_slots_sz = 0;
5137 static int bpf_object_init_prog_arrays(struct bpf_object *obj)
5139 struct bpf_map *map;
5142 for (i = 0; i < obj->nr_maps; i++) {
5143 map = &obj->maps[i];
5145 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)
5148 err = init_prog_array_slots(obj, map);
5157 static int map_set_def_max_entries(struct bpf_map *map)
5159 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {
5162 nr_cpus = libbpf_num_possible_cpus();
5164 pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
5165 map->name, nr_cpus);
5168 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
5169 map->def.max_entries = nr_cpus;
5176 bpf_object__create_maps(struct bpf_object *obj)
5178 struct bpf_map *map;
5179 char *cp, errmsg[STRERR_BUFSIZE];
5184 for (i = 0; i < obj->nr_maps; i++) {
5185 map = &obj->maps[i];
5187 /* To support old kernels, we skip creating global data maps
5188 * (.rodata, .data, .kconfig, etc); later on, during program
5189 * loading, if we detect that at least one of the to-be-loaded
5190 * programs is referencing any global data map, we'll error
5191 * out with program name and relocation index logged.
5192 * This approach allows to accommodate Clang emitting
5193 * unnecessary .rodata.str1.1 sections for string literals,
5194 * but also it allows to have CO-RE applications that use
5195 * global variables in some of BPF programs, but not others.
5196 * If those global variable-using programs are not loaded at
5197 * runtime due to bpf_program__set_autoload(prog, false),
5198 * bpf_object loading will succeed just fine even on old
5201 if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA))
5202 map->autocreate = false;
5204 if (!map->autocreate) {
5205 pr_debug("map '%s': skipped auto-creating...\n", map->name);
5209 err = map_set_def_max_entries(map);
5215 if (map->pin_path) {
5216 err = bpf_object__reuse_map(map);
5218 pr_warn("map '%s': error reusing pinned map\n",
5222 if (retried && map->fd < 0) {
5223 pr_warn("map '%s': cannot find pinned map\n",
5231 pr_debug("map '%s': skipping creation (preset fd=%d)\n",
5232 map->name, map->fd);
5234 err = bpf_object__create_map(obj, map, false);
5238 pr_debug("map '%s': created successfully, fd=%d\n",
5239 map->name, map->fd);
5241 if (bpf_map__is_internal(map)) {
5242 err = bpf_object__populate_internal_map(obj, map);
5249 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {
5250 err = init_map_in_map_slots(obj, map);
5258 if (map->pin_path && !map->pinned) {
5259 err = bpf_map__pin(map, NULL);
5262 if (!retried && err == -EEXIST) {
5266 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
5267 map->name, map->pin_path, err);
5276 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5277 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5279 for (j = 0; j < i; j++)
5280 zclose(obj->maps[j].fd);
5284 static bool bpf_core_is_flavor_sep(const char *s)
5286 /* check X___Y name pattern, where X and Y are not underscores */
5287 return s[0] != '_' && /* X */
5288 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */
5289 s[4] != '_'; /* Y */
5292 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
5293 * before last triple underscore. Struct name part after last triple
5294 * underscore is ignored by BPF CO-RE relocation during relocation matching.
5296 size_t bpf_core_essential_name_len(const char *name)
5298 size_t n = strlen(name);
5301 for (i = n - 5; i >= 0; i--) {
5302 if (bpf_core_is_flavor_sep(name + i))
5308 void bpf_core_free_cands(struct bpf_core_cand_list *cands)
5317 int bpf_core_add_cands(struct bpf_core_cand *local_cand,
5318 size_t local_essent_len,
5319 const struct btf *targ_btf,
5320 const char *targ_btf_name,
5322 struct bpf_core_cand_list *cands)
5324 struct bpf_core_cand *new_cands, *cand;
5325 const struct btf_type *t, *local_t;
5326 const char *targ_name, *local_name;
5327 size_t targ_essent_len;
5330 local_t = btf__type_by_id(local_cand->btf, local_cand->id);
5331 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);
5333 n = btf__type_cnt(targ_btf);
5334 for (i = targ_start_id; i < n; i++) {
5335 t = btf__type_by_id(targ_btf, i);
5336 if (!btf_kind_core_compat(t, local_t))
5339 targ_name = btf__name_by_offset(targ_btf, t->name_off);
5340 if (str_is_empty(targ_name))
5343 targ_essent_len = bpf_core_essential_name_len(targ_name);
5344 if (targ_essent_len != local_essent_len)
5347 if (strncmp(local_name, targ_name, local_essent_len) != 0)
5350 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
5351 local_cand->id, btf_kind_str(local_t),
5352 local_name, i, btf_kind_str(t), targ_name,
5354 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
5355 sizeof(*cands->cands));
5359 cand = &new_cands[cands->len];
5360 cand->btf = targ_btf;
5363 cands->cands = new_cands;
5369 static int load_module_btfs(struct bpf_object *obj)
5371 struct bpf_btf_info info;
5372 struct module_btf *mod_btf;
5378 if (obj->btf_modules_loaded)
5381 if (obj->gen_loader)
5384 /* don't do this again, even if we find no module BTFs */
5385 obj->btf_modules_loaded = true;
5387 /* kernel too old to support module BTFs */
5388 if (!kernel_supports(obj, FEAT_MODULE_BTF))
5392 err = bpf_btf_get_next_id(id, &id);
5393 if (err && errno == ENOENT)
5397 pr_warn("failed to iterate BTF objects: %d\n", err);
5401 fd = bpf_btf_get_fd_by_id(id);
5403 if (errno == ENOENT)
5404 continue; /* expected race: BTF was unloaded */
5406 pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
5411 memset(&info, 0, sizeof(info));
5412 info.name = ptr_to_u64(name);
5413 info.name_len = sizeof(name);
5415 err = bpf_obj_get_info_by_fd(fd, &info, &len);
5418 pr_warn("failed to get BTF object #%d info: %d\n", id, err);
5422 /* ignore non-module BTFs */
5423 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
5428 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5429 err = libbpf_get_error(btf);
5431 pr_warn("failed to load module [%s]'s BTF object #%d: %d\n",
5436 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5437 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5441 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
5446 mod_btf->name = strdup(name);
5447 if (!mod_btf->name) {
5461 static struct bpf_core_cand_list *
5462 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
5464 struct bpf_core_cand local_cand = {};
5465 struct bpf_core_cand_list *cands;
5466 const struct btf *main_btf;
5467 const struct btf_type *local_t;
5468 const char *local_name;
5469 size_t local_essent_len;
5472 local_cand.btf = local_btf;
5473 local_cand.id = local_type_id;
5474 local_t = btf__type_by_id(local_btf, local_type_id);
5476 return ERR_PTR(-EINVAL);
5478 local_name = btf__name_by_offset(local_btf, local_t->name_off);
5479 if (str_is_empty(local_name))
5480 return ERR_PTR(-EINVAL);
5481 local_essent_len = bpf_core_essential_name_len(local_name);
5483 cands = calloc(1, sizeof(*cands));
5485 return ERR_PTR(-ENOMEM);
5487 /* Attempt to find target candidates in vmlinux BTF first */
5488 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5489 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
5493 /* if vmlinux BTF has any candidate, don't got for module BTFs */
5497 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */
5498 if (obj->btf_vmlinux_override)
5501 /* now look through module BTFs, trying to still find candidates */
5502 err = load_module_btfs(obj);
5506 for (i = 0; i < obj->btf_module_cnt; i++) {
5507 err = bpf_core_add_cands(&local_cand, local_essent_len,
5508 obj->btf_modules[i].btf,
5509 obj->btf_modules[i].name,
5510 btf__type_cnt(obj->btf_vmlinux),
5518 bpf_core_free_cands(cands);
5519 return ERR_PTR(err);
5522 /* Check local and target types for compatibility. This check is used for
5523 * type-based CO-RE relocations and follow slightly different rules than
5524 * field-based relocations. This function assumes that root types were already
5525 * checked for name match. Beyond that initial root-level name check, names
5526 * are completely ignored. Compatibility rules are as follows:
5527 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5528 * kind should match for local and target types (i.e., STRUCT is not
5529 * compatible with UNION);
5530 * - for ENUMs, the size is ignored;
5531 * - for INT, size and signedness are ignored;
5532 * - for ARRAY, dimensionality is ignored, element types are checked for
5533 * compatibility recursively;
5534 * - CONST/VOLATILE/RESTRICT modifiers are ignored;
5535 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5536 * - FUNC_PROTOs are compatible if they have compatible signature: same
5537 * number of input args and compatible return and argument types.
5538 * These rules are not set in stone and probably will be adjusted as we get
5539 * more experience with using BPF CO-RE relocations.
5541 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5542 const struct btf *targ_btf, __u32 targ_id)
5544 return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32);
5547 int bpf_core_types_match(const struct btf *local_btf, __u32 local_id,
5548 const struct btf *targ_btf, __u32 targ_id)
5550 return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32);
5553 static size_t bpf_core_hash_fn(const void *key, void *ctx)
5558 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
5563 static void *u32_as_hash_key(__u32 x)
5565 return (void *)(uintptr_t)x;
5568 static int record_relo_core(struct bpf_program *prog,
5569 const struct bpf_core_relo *core_relo, int insn_idx)
5571 struct reloc_desc *relos, *relo;
5573 relos = libbpf_reallocarray(prog->reloc_desc,
5574 prog->nr_reloc + 1, sizeof(*relos));
5577 relo = &relos[prog->nr_reloc];
5578 relo->type = RELO_CORE;
5579 relo->insn_idx = insn_idx;
5580 relo->core_relo = core_relo;
5581 prog->reloc_desc = relos;
5586 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx)
5588 struct reloc_desc *relo;
5591 for (i = 0; i < prog->nr_reloc; i++) {
5592 relo = &prog->reloc_desc[i];
5593 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)
5596 return relo->core_relo;
5602 static int bpf_core_resolve_relo(struct bpf_program *prog,
5603 const struct bpf_core_relo *relo,
5605 const struct btf *local_btf,
5606 struct hashmap *cand_cache,
5607 struct bpf_core_relo_res *targ_res)
5609 struct bpf_core_spec specs_scratch[3] = {};
5610 const void *type_key = u32_as_hash_key(relo->type_id);
5611 struct bpf_core_cand_list *cands = NULL;
5612 const char *prog_name = prog->name;
5613 const struct btf_type *local_type;
5614 const char *local_name;
5615 __u32 local_id = relo->type_id;
5618 local_type = btf__type_by_id(local_btf, local_id);
5622 local_name = btf__name_by_offset(local_btf, local_type->name_off);
5626 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&
5627 !hashmap__find(cand_cache, type_key, (void **)&cands)) {
5628 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5629 if (IS_ERR(cands)) {
5630 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5631 prog_name, relo_idx, local_id, btf_kind_str(local_type),
5632 local_name, PTR_ERR(cands));
5633 return PTR_ERR(cands);
5635 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
5637 bpf_core_free_cands(cands);
5642 return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch,
5647 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
5649 const struct btf_ext_info_sec *sec;
5650 struct bpf_core_relo_res targ_res;
5651 const struct bpf_core_relo *rec;
5652 const struct btf_ext_info *seg;
5653 struct hashmap_entry *entry;
5654 struct hashmap *cand_cache = NULL;
5655 struct bpf_program *prog;
5656 struct bpf_insn *insn;
5657 const char *sec_name;
5658 int i, err = 0, insn_idx, sec_idx, sec_num;
5660 if (obj->btf_ext->core_relo_info.len == 0)
5663 if (targ_btf_path) {
5664 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
5665 err = libbpf_get_error(obj->btf_vmlinux_override);
5667 pr_warn("failed to parse target BTF: %d\n", err);
5672 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
5673 if (IS_ERR(cand_cache)) {
5674 err = PTR_ERR(cand_cache);
5678 seg = &obj->btf_ext->core_relo_info;
5680 for_each_btf_ext_sec(seg, sec) {
5681 sec_idx = seg->sec_idxs[sec_num];
5684 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
5685 if (str_is_empty(sec_name)) {
5690 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
5692 for_each_btf_ext_rec(seg, sec, i, rec) {
5693 if (rec->insn_off % BPF_INSN_SZ)
5695 insn_idx = rec->insn_off / BPF_INSN_SZ;
5696 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
5698 /* When __weak subprog is "overridden" by another instance
5699 * of the subprog from a different object file, linker still
5700 * appends all the .BTF.ext info that used to belong to that
5701 * eliminated subprogram.
5702 * This is similar to what x86-64 linker does for relocations.
5703 * So just ignore such relocations just like we ignore
5704 * subprog instructions when discovering subprograms.
5706 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n",
5707 sec_name, i, insn_idx);
5710 /* no need to apply CO-RE relocation if the program is
5711 * not going to be loaded
5713 if (!prog->autoload)
5716 /* adjust insn_idx from section frame of reference to the local
5717 * program's frame of reference; (sub-)program code is not yet
5718 * relocated, so it's enough to just subtract in-section offset
5720 insn_idx = insn_idx - prog->sec_insn_off;
5721 if (insn_idx >= prog->insns_cnt)
5723 insn = &prog->insns[insn_idx];
5725 err = record_relo_core(prog, rec, insn_idx);
5727 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",
5728 prog->name, i, err);
5732 if (prog->obj->gen_loader)
5735 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
5737 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
5738 prog->name, i, err);
5742 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
5744 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",
5745 prog->name, i, insn_idx, err);
5752 /* obj->btf_vmlinux and module BTFs are freed after object load */
5753 btf__free(obj->btf_vmlinux_override);
5754 obj->btf_vmlinux_override = NULL;
5756 if (!IS_ERR_OR_NULL(cand_cache)) {
5757 hashmap__for_each_entry(cand_cache, entry, i) {
5758 bpf_core_free_cands(entry->value);
5760 hashmap__free(cand_cache);
5765 /* base map load ldimm64 special constant, used also for log fixup logic */
5766 #define MAP_LDIMM64_POISON_BASE 2001000000
5767 #define MAP_LDIMM64_POISON_PFX "200100"
5769 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,
5770 int insn_idx, struct bpf_insn *insn,
5771 int map_idx, const struct bpf_map *map)
5775 pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",
5776 prog->name, relo_idx, insn_idx, map_idx, map->name);
5778 /* we turn single ldimm64 into two identical invalid calls */
5779 for (i = 0; i < 2; i++) {
5780 insn->code = BPF_JMP | BPF_CALL;
5784 /* if this instruction is reachable (not a dead code),
5785 * verifier will complain with something like:
5786 * invalid func unknown#2001000123
5787 * where lower 123 is map index into obj->maps[] array
5789 insn->imm = MAP_LDIMM64_POISON_BASE + map_idx;
5795 /* Relocate data references within program code:
5797 * - global variable references;
5798 * - extern references.
5801 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
5805 for (i = 0; i < prog->nr_reloc; i++) {
5806 struct reloc_desc *relo = &prog->reloc_desc[i];
5807 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
5808 const struct bpf_map *map;
5809 struct extern_desc *ext;
5811 switch (relo->type) {
5813 map = &obj->maps[relo->map_idx];
5814 if (obj->gen_loader) {
5815 insn[0].src_reg = BPF_PSEUDO_MAP_IDX;
5816 insn[0].imm = relo->map_idx;
5817 } else if (map->autocreate) {
5818 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
5819 insn[0].imm = map->fd;
5821 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
5822 relo->map_idx, map);
5826 map = &obj->maps[relo->map_idx];
5827 insn[1].imm = insn[0].imm + relo->sym_off;
5828 if (obj->gen_loader) {
5829 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
5830 insn[0].imm = relo->map_idx;
5831 } else if (map->autocreate) {
5832 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5833 insn[0].imm = map->fd;
5835 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
5836 relo->map_idx, map);
5839 case RELO_EXTERN_VAR:
5840 ext = &obj->externs[relo->sym_off];
5841 if (ext->type == EXT_KCFG) {
5842 if (obj->gen_loader) {
5843 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
5844 insn[0].imm = obj->kconfig_map_idx;
5846 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5847 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
5849 insn[1].imm = ext->kcfg.data_off;
5850 } else /* EXT_KSYM */ {
5851 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */
5852 insn[0].src_reg = BPF_PSEUDO_BTF_ID;
5853 insn[0].imm = ext->ksym.kernel_btf_id;
5854 insn[1].imm = ext->ksym.kernel_btf_obj_fd;
5855 } else { /* typeless ksyms or unresolved typed ksyms */
5856 insn[0].imm = (__u32)ext->ksym.addr;
5857 insn[1].imm = ext->ksym.addr >> 32;
5861 case RELO_EXTERN_FUNC:
5862 ext = &obj->externs[relo->sym_off];
5863 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
5865 insn[0].imm = ext->ksym.kernel_btf_id;
5866 insn[0].off = ext->ksym.btf_fd_idx;
5867 } else { /* unresolved weak kfunc */
5872 case RELO_SUBPROG_ADDR:
5873 if (insn[0].src_reg != BPF_PSEUDO_FUNC) {
5874 pr_warn("prog '%s': relo #%d: bad insn\n",
5878 /* handled already */
5881 /* handled already */
5884 /* will be handled by bpf_program_record_relos() */
5887 pr_warn("prog '%s': relo #%d: bad relo type %d\n",
5888 prog->name, i, relo->type);
5896 static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
5897 const struct bpf_program *prog,
5898 const struct btf_ext_info *ext_info,
5899 void **prog_info, __u32 *prog_rec_cnt,
5902 void *copy_start = NULL, *copy_end = NULL;
5903 void *rec, *rec_end, *new_prog_info;
5904 const struct btf_ext_info_sec *sec;
5905 size_t old_sz, new_sz;
5906 int i, sec_num, sec_idx, off_adj;
5909 for_each_btf_ext_sec(ext_info, sec) {
5910 sec_idx = ext_info->sec_idxs[sec_num];
5912 if (prog->sec_idx != sec_idx)
5915 for_each_btf_ext_rec(ext_info, sec, i, rec) {
5916 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
5918 if (insn_off < prog->sec_insn_off)
5920 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
5925 copy_end = rec + ext_info->rec_size;
5931 /* append func/line info of a given (sub-)program to the main
5932 * program func/line info
5934 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
5935 new_sz = old_sz + (copy_end - copy_start);
5936 new_prog_info = realloc(*prog_info, new_sz);
5939 *prog_info = new_prog_info;
5940 *prog_rec_cnt = new_sz / ext_info->rec_size;
5941 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
5943 /* Kernel instruction offsets are in units of 8-byte
5944 * instructions, while .BTF.ext instruction offsets generated
5945 * by Clang are in units of bytes. So convert Clang offsets
5946 * into kernel offsets and adjust offset according to program
5947 * relocated position.
5949 off_adj = prog->sub_insn_off - prog->sec_insn_off;
5950 rec = new_prog_info + old_sz;
5951 rec_end = new_prog_info + new_sz;
5952 for (; rec < rec_end; rec += ext_info->rec_size) {
5953 __u32 *insn_off = rec;
5955 *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
5957 *prog_rec_sz = ext_info->rec_size;
5965 reloc_prog_func_and_line_info(const struct bpf_object *obj,
5966 struct bpf_program *main_prog,
5967 const struct bpf_program *prog)
5971 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
5972 * supprot func/line info
5974 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
5977 /* only attempt func info relocation if main program's func_info
5978 * relocation was successful
5980 if (main_prog != prog && !main_prog->func_info)
5983 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
5984 &main_prog->func_info,
5985 &main_prog->func_info_cnt,
5986 &main_prog->func_info_rec_size);
5988 if (err != -ENOENT) {
5989 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
5993 if (main_prog->func_info) {
5995 * Some info has already been found but has problem
5996 * in the last btf_ext reloc. Must have to error out.
5998 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6001 /* Have problem loading the very first info. Ignore the rest. */
6002 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6007 /* don't relocate line info if main program's relocation failed */
6008 if (main_prog != prog && !main_prog->line_info)
6011 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6012 &main_prog->line_info,
6013 &main_prog->line_info_cnt,
6014 &main_prog->line_info_rec_size);
6016 if (err != -ENOENT) {
6017 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6021 if (main_prog->line_info) {
6023 * Some info has already been found but has problem
6024 * in the last btf_ext reloc. Must have to error out.
6026 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6029 /* Have problem loading the very first info. Ignore the rest. */
6030 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6036 static int cmp_relo_by_insn_idx(const void *key, const void *elem)
6038 size_t insn_idx = *(const size_t *)key;
6039 const struct reloc_desc *relo = elem;
6041 if (insn_idx == relo->insn_idx)
6043 return insn_idx < relo->insn_idx ? -1 : 1;
6046 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6048 if (!prog->nr_reloc)
6050 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6051 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6054 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog)
6056 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
6057 struct reloc_desc *relos;
6060 if (main_prog == subprog)
6062 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
6065 if (subprog->nr_reloc)
6066 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
6067 sizeof(*relos) * subprog->nr_reloc);
6069 for (i = main_prog->nr_reloc; i < new_cnt; i++)
6070 relos[i].insn_idx += subprog->sub_insn_off;
6071 /* After insn_idx adjustment the 'relos' array is still sorted
6072 * by insn_idx and doesn't break bsearch.
6074 main_prog->reloc_desc = relos;
6075 main_prog->nr_reloc = new_cnt;
6080 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6081 struct bpf_program *prog)
6083 size_t sub_insn_idx, insn_idx, new_cnt;
6084 struct bpf_program *subprog;
6085 struct bpf_insn *insns, *insn;
6086 struct reloc_desc *relo;
6089 err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6093 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6094 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6095 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))
6098 relo = find_prog_insn_relo(prog, insn_idx);
6099 if (relo && relo->type == RELO_EXTERN_FUNC)
6100 /* kfunc relocations will be handled later
6101 * in bpf_object__relocate_data()
6104 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6105 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6106 prog->name, insn_idx, relo->type);
6107 return -LIBBPF_ERRNO__RELOC;
6110 /* sub-program instruction index is a combination of
6111 * an offset of a symbol pointed to by relocation and
6112 * call instruction's imm field; for global functions,
6113 * call always has imm = -1, but for static functions
6114 * relocation is against STT_SECTION and insn->imm
6115 * points to a start of a static function
6117 * for subprog addr relocation, the relo->sym_off + insn->imm is
6118 * the byte offset in the corresponding section.
6120 if (relo->type == RELO_CALL)
6121 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6123 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6124 } else if (insn_is_pseudo_func(insn)) {
6126 * RELO_SUBPROG_ADDR relo is always emitted even if both
6127 * functions are in the same section, so it shouldn't reach here.
6129 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
6130 prog->name, insn_idx);
6131 return -LIBBPF_ERRNO__RELOC;
6133 /* if subprogram call is to a static function within
6134 * the same ELF section, there won't be any relocation
6135 * emitted, but it also means there is no additional
6136 * offset necessary, insns->imm is relative to
6137 * instruction's original position within the section
6139 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6142 /* we enforce that sub-programs should be in .text section */
6143 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6145 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6147 return -LIBBPF_ERRNO__RELOC;
6150 /* if it's the first call instruction calling into this
6151 * subprogram (meaning this subprog hasn't been processed
6152 * yet) within the context of current main program:
6153 * - append it at the end of main program's instructions blog;
6154 * - process is recursively, while current program is put on hold;
6155 * - if that subprogram calls some other not yet processes
6156 * subprogram, same thing will happen recursively until
6157 * there are no more unprocesses subprograms left to append
6160 if (subprog->sub_insn_off == 0) {
6161 subprog->sub_insn_off = main_prog->insns_cnt;
6163 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6164 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6166 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6169 main_prog->insns = insns;
6170 main_prog->insns_cnt = new_cnt;
6172 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6173 subprog->insns_cnt * sizeof(*insns));
6175 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6176 main_prog->name, subprog->insns_cnt, subprog->name);
6178 /* The subprog insns are now appended. Append its relos too. */
6179 err = append_subprog_relos(main_prog, subprog);
6182 err = bpf_object__reloc_code(obj, main_prog, subprog);
6187 /* main_prog->insns memory could have been re-allocated, so
6188 * calculate pointer again
6190 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6191 /* calculate correct instruction position within current main
6192 * prog; each main prog can have a different set of
6193 * subprograms appended (potentially in different order as
6194 * well), so position of any subprog can be different for
6195 * different main programs */
6196 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6198 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6199 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6206 * Relocate sub-program calls.
6208 * Algorithm operates as follows. Each entry-point BPF program (referred to as
6209 * main prog) is processed separately. For each subprog (non-entry functions,
6210 * that can be called from either entry progs or other subprogs) gets their
6211 * sub_insn_off reset to zero. This serves as indicator that this subprogram
6212 * hasn't been yet appended and relocated within current main prog. Once its
6213 * relocated, sub_insn_off will point at the position within current main prog
6214 * where given subprog was appended. This will further be used to relocate all
6215 * the call instructions jumping into this subprog.
6217 * We start with main program and process all call instructions. If the call
6218 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6219 * is zero), subprog instructions are appended at the end of main program's
6220 * instruction array. Then main program is "put on hold" while we recursively
6221 * process newly appended subprogram. If that subprogram calls into another
6222 * subprogram that hasn't been appended, new subprogram is appended again to
6223 * the *main* prog's instructions (subprog's instructions are always left
6224 * untouched, as they need to be in unmodified state for subsequent main progs
6225 * and subprog instructions are always sent only as part of a main prog) and
6226 * the process continues recursively. Once all the subprogs called from a main
6227 * prog or any of its subprogs are appended (and relocated), all their
6228 * positions within finalized instructions array are known, so it's easy to
6229 * rewrite call instructions with correct relative offsets, corresponding to
6230 * desired target subprog.
6232 * Its important to realize that some subprogs might not be called from some
6233 * main prog and any of its called/used subprogs. Those will keep their
6234 * subprog->sub_insn_off as zero at all times and won't be appended to current
6235 * main prog and won't be relocated within the context of current main prog.
6236 * They might still be used from other main progs later.
6238 * Visually this process can be shown as below. Suppose we have two main
6239 * programs mainA and mainB and BPF object contains three subprogs: subA,
6240 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
6241 * subC both call subB:
6243 * +--------+ +-------+
6245 * +--+---+ +--+-+-+ +---+--+
6246 * | subA | | subB | | subC |
6247 * +--+---+ +------+ +---+--+
6250 * +---+-------+ +------+----+
6251 * | mainA | | mainB |
6252 * +-----------+ +-----------+
6254 * We'll start relocating mainA, will find subA, append it and start
6255 * processing sub A recursively:
6257 * +-----------+------+
6259 * +-----------+------+
6261 * At this point we notice that subB is used from subA, so we append it and
6262 * relocate (there are no further subcalls from subB):
6264 * +-----------+------+------+
6265 * | mainA | subA | subB |
6266 * +-----------+------+------+
6268 * At this point, we relocate subA calls, then go one level up and finish with
6269 * relocatin mainA calls. mainA is done.
6271 * For mainB process is similar but results in different order. We start with
6272 * mainB and skip subA and subB, as mainB never calls them (at least
6273 * directly), but we see subC is needed, so we append and start processing it:
6275 * +-----------+------+
6277 * +-----------+------+
6278 * Now we see subC needs subB, so we go back to it, append and relocate it:
6280 * +-----------+------+------+
6281 * | mainB | subC | subB |
6282 * +-----------+------+------+
6284 * At this point we unwind recursion, relocate calls in subC, then in mainB.
6287 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6289 struct bpf_program *subprog;
6292 /* mark all subprogs as not relocated (yet) within the context of
6293 * current main program
6295 for (i = 0; i < obj->nr_programs; i++) {
6296 subprog = &obj->programs[i];
6297 if (!prog_is_subprog(obj, subprog))
6300 subprog->sub_insn_off = 0;
6303 err = bpf_object__reloc_code(obj, prog, prog);
6311 bpf_object__free_relocs(struct bpf_object *obj)
6313 struct bpf_program *prog;
6316 /* free up relocation descriptors */
6317 for (i = 0; i < obj->nr_programs; i++) {
6318 prog = &obj->programs[i];
6319 zfree(&prog->reloc_desc);
6324 static int cmp_relocs(const void *_a, const void *_b)
6326 const struct reloc_desc *a = _a;
6327 const struct reloc_desc *b = _b;
6329 if (a->insn_idx != b->insn_idx)
6330 return a->insn_idx < b->insn_idx ? -1 : 1;
6332 /* no two relocations should have the same insn_idx, but ... */
6333 if (a->type != b->type)
6334 return a->type < b->type ? -1 : 1;
6339 static void bpf_object__sort_relos(struct bpf_object *obj)
6343 for (i = 0; i < obj->nr_programs; i++) {
6344 struct bpf_program *p = &obj->programs[i];
6349 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6354 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
6356 struct bpf_program *prog;
6361 err = bpf_object__relocate_core(obj, targ_btf_path);
6363 pr_warn("failed to perform CO-RE relocations: %d\n",
6367 bpf_object__sort_relos(obj);
6370 /* Before relocating calls pre-process relocations and mark
6371 * few ld_imm64 instructions that points to subprogs.
6372 * Otherwise bpf_object__reloc_code() later would have to consider
6373 * all ld_imm64 insns as relocation candidates. That would
6374 * reduce relocation speed, since amount of find_prog_insn_relo()
6375 * would increase and most of them will fail to find a relo.
6377 for (i = 0; i < obj->nr_programs; i++) {
6378 prog = &obj->programs[i];
6379 for (j = 0; j < prog->nr_reloc; j++) {
6380 struct reloc_desc *relo = &prog->reloc_desc[j];
6381 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6383 /* mark the insn, so it's recognized by insn_is_pseudo_func() */
6384 if (relo->type == RELO_SUBPROG_ADDR)
6385 insn[0].src_reg = BPF_PSEUDO_FUNC;
6389 /* relocate subprogram calls and append used subprograms to main
6390 * programs; each copy of subprogram code needs to be relocated
6391 * differently for each main program, because its code location might
6393 * Append subprog relos to main programs to allow data relos to be
6394 * processed after text is completely relocated.
6396 for (i = 0; i < obj->nr_programs; i++) {
6397 prog = &obj->programs[i];
6398 /* sub-program's sub-calls are relocated within the context of
6399 * its main program only
6401 if (prog_is_subprog(obj, prog))
6403 if (!prog->autoload)
6406 err = bpf_object__relocate_calls(obj, prog);
6408 pr_warn("prog '%s': failed to relocate calls: %d\n",
6413 /* Process data relos for main programs */
6414 for (i = 0; i < obj->nr_programs; i++) {
6415 prog = &obj->programs[i];
6416 if (prog_is_subprog(obj, prog))
6418 if (!prog->autoload)
6420 err = bpf_object__relocate_data(obj, prog);
6422 pr_warn("prog '%s': failed to relocate data references: %d\n",
6431 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
6432 Elf64_Shdr *shdr, Elf_Data *data);
6434 static int bpf_object__collect_map_relos(struct bpf_object *obj,
6435 Elf64_Shdr *shdr, Elf_Data *data)
6437 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
6438 int i, j, nrels, new_sz;
6439 const struct btf_var_secinfo *vi = NULL;
6440 const struct btf_type *sec, *var, *def;
6441 struct bpf_map *map = NULL, *targ_map = NULL;
6442 struct bpf_program *targ_prog = NULL;
6443 bool is_prog_array, is_map_in_map;
6444 const struct btf_member *member;
6445 const char *name, *mname, *type;
6451 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6453 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6457 nrels = shdr->sh_size / shdr->sh_entsize;
6458 for (i = 0; i < nrels; i++) {
6459 rel = elf_rel_by_idx(data, i);
6461 pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
6462 return -LIBBPF_ERRNO__FORMAT;
6465 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
6467 pr_warn(".maps relo #%d: symbol %zx not found\n",
6468 i, (size_t)ELF64_R_SYM(rel->r_info));
6469 return -LIBBPF_ERRNO__FORMAT;
6471 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
6473 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
6474 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
6475 (size_t)rel->r_offset, sym->st_name, name);
6477 for (j = 0; j < obj->nr_maps; j++) {
6478 map = &obj->maps[j];
6479 if (map->sec_idx != obj->efile.btf_maps_shndx)
6482 vi = btf_var_secinfos(sec) + map->btf_var_idx;
6483 if (vi->offset <= rel->r_offset &&
6484 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)
6487 if (j == obj->nr_maps) {
6488 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",
6489 i, name, (size_t)rel->r_offset);
6493 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);
6494 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;
6495 type = is_map_in_map ? "map" : "prog";
6496 if (is_map_in_map) {
6497 if (sym->st_shndx != obj->efile.btf_maps_shndx) {
6498 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
6500 return -LIBBPF_ERRNO__RELOC;
6502 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
6503 map->def.key_size != sizeof(int)) {
6504 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
6505 i, map->name, sizeof(int));
6508 targ_map = bpf_object__find_map_by_name(obj, name);
6510 pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n",
6514 } else if (is_prog_array) {
6515 targ_prog = bpf_object__find_program_by_name(obj, name);
6517 pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n",
6521 if (targ_prog->sec_idx != sym->st_shndx ||
6522 targ_prog->sec_insn_off * 8 != sym->st_value ||
6523 prog_is_subprog(obj, targ_prog)) {
6524 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",
6526 return -LIBBPF_ERRNO__RELOC;
6532 var = btf__type_by_id(obj->btf, vi->type);
6533 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6534 if (btf_vlen(def) == 0)
6536 member = btf_members(def) + btf_vlen(def) - 1;
6537 mname = btf__name_by_offset(obj->btf, member->name_off);
6538 if (strcmp(mname, "values"))
6541 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
6542 if (rel->r_offset - vi->offset < moff)
6545 moff = rel->r_offset - vi->offset - moff;
6546 /* here we use BPF pointer size, which is always 64 bit, as we
6547 * are parsing ELF that was built for BPF target
6549 if (moff % bpf_ptr_sz)
6552 if (moff >= map->init_slots_sz) {
6554 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
6557 map->init_slots = tmp;
6558 memset(map->init_slots + map->init_slots_sz, 0,
6559 (new_sz - map->init_slots_sz) * host_ptr_sz);
6560 map->init_slots_sz = new_sz;
6562 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
6564 pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n",
6565 i, map->name, moff, type, name);
6571 static int bpf_object__collect_relos(struct bpf_object *obj)
6575 for (i = 0; i < obj->efile.sec_cnt; i++) {
6576 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
6581 if (sec_desc->sec_type != SEC_RELO)
6584 shdr = sec_desc->shdr;
6585 data = sec_desc->data;
6586 idx = shdr->sh_info;
6588 if (shdr->sh_type != SHT_REL) {
6589 pr_warn("internal error at %d\n", __LINE__);
6590 return -LIBBPF_ERRNO__INTERNAL;
6593 if (idx == obj->efile.st_ops_shndx)
6594 err = bpf_object__collect_st_ops_relos(obj, shdr, data);
6595 else if (idx == obj->efile.btf_maps_shndx)
6596 err = bpf_object__collect_map_relos(obj, shdr, data);
6598 err = bpf_object__collect_prog_relos(obj, shdr, data);
6603 bpf_object__sort_relos(obj);
6607 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
6609 if (BPF_CLASS(insn->code) == BPF_JMP &&
6610 BPF_OP(insn->code) == BPF_CALL &&
6611 BPF_SRC(insn->code) == BPF_K &&
6612 insn->src_reg == 0 &&
6613 insn->dst_reg == 0) {
6614 *func_id = insn->imm;
6620 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
6622 struct bpf_insn *insn = prog->insns;
6623 enum bpf_func_id func_id;
6626 if (obj->gen_loader)
6629 for (i = 0; i < prog->insns_cnt; i++, insn++) {
6630 if (!insn_is_helper_call(insn, &func_id))
6633 /* on kernels that don't yet support
6634 * bpf_probe_read_{kernel,user}[_str] helpers, fall back
6635 * to bpf_probe_read() which works well for old kernels
6638 case BPF_FUNC_probe_read_kernel:
6639 case BPF_FUNC_probe_read_user:
6640 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6641 insn->imm = BPF_FUNC_probe_read;
6643 case BPF_FUNC_probe_read_kernel_str:
6644 case BPF_FUNC_probe_read_user_str:
6645 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6646 insn->imm = BPF_FUNC_probe_read_str;
6655 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
6656 int *btf_obj_fd, int *btf_type_id);
6658 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
6659 static int libbpf_prepare_prog_load(struct bpf_program *prog,
6660 struct bpf_prog_load_opts *opts, long cookie)
6662 enum sec_def_flags def = cookie;
6664 /* old kernels might not support specifying expected_attach_type */
6665 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
6666 opts->expected_attach_type = 0;
6668 if (def & SEC_SLEEPABLE)
6669 opts->prog_flags |= BPF_F_SLEEPABLE;
6671 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
6672 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
6674 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
6675 int btf_obj_fd = 0, btf_type_id = 0, err;
6676 const char *attach_name;
6678 attach_name = strchr(prog->sec_name, '/');
6680 /* if BPF program is annotated with just SEC("fentry")
6681 * (or similar) without declaratively specifying
6682 * target, then it is expected that target will be
6683 * specified with bpf_program__set_attach_target() at
6684 * runtime before BPF object load step. If not, then
6685 * there is nothing to load into the kernel as BPF
6686 * verifier won't be able to validate BPF program
6687 * correctness anyways.
6689 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
6693 attach_name++; /* skip over / */
6695 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);
6699 /* cache resolved BTF FD and BTF type ID in the prog */
6700 prog->attach_btf_obj_fd = btf_obj_fd;
6701 prog->attach_btf_id = btf_type_id;
6703 /* but by now libbpf common logic is not utilizing
6704 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
6705 * this callback is called after opts were populated by
6706 * libbpf, so this callback has to update opts explicitly here
6708 opts->attach_btf_obj_fd = btf_obj_fd;
6709 opts->attach_btf_id = btf_type_id;
6714 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
6716 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
6717 struct bpf_insn *insns, int insns_cnt,
6718 const char *license, __u32 kern_version, int *prog_fd)
6720 LIBBPF_OPTS(bpf_prog_load_opts, load_attr);
6721 const char *prog_name = NULL;
6722 char *cp, errmsg[STRERR_BUFSIZE];
6723 size_t log_buf_size = 0;
6724 char *log_buf = NULL, *tmp;
6725 int btf_fd, ret, err;
6726 bool own_log_buf = true;
6727 __u32 log_level = prog->log_level;
6729 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6731 * The program type must be set. Most likely we couldn't find a proper
6732 * section definition at load time, and thus we didn't infer the type.
6734 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
6735 prog->name, prog->sec_name);
6739 if (!insns || !insns_cnt)
6742 load_attr.expected_attach_type = prog->expected_attach_type;
6743 if (kernel_supports(obj, FEAT_PROG_NAME))
6744 prog_name = prog->name;
6745 load_attr.attach_prog_fd = prog->attach_prog_fd;
6746 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
6747 load_attr.attach_btf_id = prog->attach_btf_id;
6748 load_attr.kern_version = kern_version;
6749 load_attr.prog_ifindex = prog->prog_ifindex;
6751 /* specify func_info/line_info only if kernel supports them */
6752 btf_fd = bpf_object__btf_fd(obj);
6753 if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
6754 load_attr.prog_btf_fd = btf_fd;
6755 load_attr.func_info = prog->func_info;
6756 load_attr.func_info_rec_size = prog->func_info_rec_size;
6757 load_attr.func_info_cnt = prog->func_info_cnt;
6758 load_attr.line_info = prog->line_info;
6759 load_attr.line_info_rec_size = prog->line_info_rec_size;
6760 load_attr.line_info_cnt = prog->line_info_cnt;
6762 load_attr.log_level = log_level;
6763 load_attr.prog_flags = prog->prog_flags;
6764 load_attr.fd_array = obj->fd_array;
6766 /* adjust load_attr if sec_def provides custom preload callback */
6767 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
6768 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
6770 pr_warn("prog '%s': failed to prepare load attributes: %d\n",
6774 insns = prog->insns;
6775 insns_cnt = prog->insns_cnt;
6778 if (obj->gen_loader) {
6779 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
6780 license, insns, insns_cnt, &load_attr,
6781 prog - obj->programs);
6787 /* if log_level is zero, we don't request logs initially even if
6788 * custom log_buf is specified; if the program load fails, then we'll
6789 * bump log_level to 1 and use either custom log_buf or we'll allocate
6790 * our own and retry the load to get details on what failed
6793 if (prog->log_buf) {
6794 log_buf = prog->log_buf;
6795 log_buf_size = prog->log_size;
6796 own_log_buf = false;
6797 } else if (obj->log_buf) {
6798 log_buf = obj->log_buf;
6799 log_buf_size = obj->log_size;
6800 own_log_buf = false;
6802 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2);
6803 tmp = realloc(log_buf, log_buf_size);
6814 load_attr.log_buf = log_buf;
6815 load_attr.log_size = log_buf_size;
6816 load_attr.log_level = log_level;
6818 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
6820 if (log_level && own_log_buf) {
6821 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
6822 prog->name, log_buf);
6825 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
6826 struct bpf_map *map;
6829 for (i = 0; i < obj->nr_maps; i++) {
6830 map = &prog->obj->maps[i];
6831 if (map->libbpf_type != LIBBPF_MAP_RODATA)
6834 if (bpf_prog_bind_map(ret, bpf_map__fd(map), NULL)) {
6835 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6836 pr_warn("prog '%s': failed to bind map '%s': %s\n",
6837 prog->name, map->real_name, cp);
6838 /* Don't fail hard if can't bind rodata. */
6848 if (log_level == 0) {
6852 /* On ENOSPC, increase log buffer size and retry, unless custom
6853 * log_buf is specified.
6854 * Be careful to not overflow u32, though. Kernel's log buf size limit
6855 * isn't part of UAPI so it can always be bumped to full 4GB. So don't
6856 * multiply by 2 unless we are sure we'll fit within 32 bits.
6857 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).
6859 if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2)
6864 /* post-process verifier log to improve error descriptions */
6865 fixup_verifier_log(prog, log_buf, log_buf_size);
6867 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6868 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
6871 if (own_log_buf && log_buf && log_buf[0] != '\0') {
6872 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
6873 prog->name, log_buf);
6882 static char *find_prev_line(char *buf, char *cur)
6886 if (cur == buf) /* end of a log buf */
6890 while (p - 1 >= buf && *(p - 1) != '\n')
6896 static void patch_log(char *buf, size_t buf_sz, size_t log_sz,
6897 char *orig, size_t orig_sz, const char *patch)
6899 /* size of the remaining log content to the right from the to-be-replaced part */
6900 size_t rem_sz = (buf + log_sz) - (orig + orig_sz);
6901 size_t patch_sz = strlen(patch);
6903 if (patch_sz != orig_sz) {
6904 /* If patch line(s) are longer than original piece of verifier log,
6905 * shift log contents by (patch_sz - orig_sz) bytes to the right
6906 * starting from after to-be-replaced part of the log.
6908 * If patch line(s) are shorter than original piece of verifier log,
6909 * shift log contents by (orig_sz - patch_sz) bytes to the left
6910 * starting from after to-be-replaced part of the log
6912 * We need to be careful about not overflowing available
6913 * buf_sz capacity. If that's the case, we'll truncate the end
6914 * of the original log, as necessary.
6916 if (patch_sz > orig_sz) {
6917 if (orig + patch_sz >= buf + buf_sz) {
6918 /* patch is big enough to cover remaining space completely */
6919 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;
6921 } else if (patch_sz - orig_sz > buf_sz - log_sz) {
6922 /* patch causes part of remaining log to be truncated */
6923 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);
6926 /* shift remaining log to the right by calculated amount */
6927 memmove(orig + patch_sz, orig + orig_sz, rem_sz);
6930 memcpy(orig, patch, patch_sz);
6933 static void fixup_log_failed_core_relo(struct bpf_program *prog,
6934 char *buf, size_t buf_sz, size_t log_sz,
6935 char *line1, char *line2, char *line3)
6937 /* Expected log for failed and not properly guarded CO-RE relocation:
6938 * line1 -> 123: (85) call unknown#195896080
6939 * line2 -> invalid func unknown#195896080
6940 * line3 -> <anything else or end of buffer>
6942 * "123" is the index of the instruction that was poisoned. We extract
6943 * instruction index to find corresponding CO-RE relocation and
6944 * replace this part of the log with more relevant information about
6945 * failed CO-RE relocation.
6947 const struct bpf_core_relo *relo;
6948 struct bpf_core_spec spec;
6949 char patch[512], spec_buf[256];
6950 int insn_idx, err, spec_len;
6952 if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1)
6955 relo = find_relo_core(prog, insn_idx);
6959 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
6963 spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec);
6964 snprintf(patch, sizeof(patch),
6965 "%d: <invalid CO-RE relocation>\n"
6966 "failed to resolve CO-RE relocation %s%s\n",
6967 insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : "");
6969 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
6972 static void fixup_log_missing_map_load(struct bpf_program *prog,
6973 char *buf, size_t buf_sz, size_t log_sz,
6974 char *line1, char *line2, char *line3)
6976 /* Expected log for failed and not properly guarded CO-RE relocation:
6977 * line1 -> 123: (85) call unknown#2001000345
6978 * line2 -> invalid func unknown#2001000345
6979 * line3 -> <anything else or end of buffer>
6981 * "123" is the index of the instruction that was poisoned.
6982 * "345" in "2001000345" are map index in obj->maps to fetch map name.
6984 struct bpf_object *obj = prog->obj;
6985 const struct bpf_map *map;
6986 int insn_idx, map_idx;
6989 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2)
6992 map_idx -= MAP_LDIMM64_POISON_BASE;
6993 if (map_idx < 0 || map_idx >= obj->nr_maps)
6995 map = &obj->maps[map_idx];
6997 snprintf(patch, sizeof(patch),
6998 "%d: <invalid BPF map reference>\n"
6999 "BPF map '%s' is referenced but wasn't created\n",
7000 insn_idx, map->name);
7002 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);
7005 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz)
7007 /* look for familiar error patterns in last N lines of the log */
7008 const size_t max_last_line_cnt = 10;
7009 char *prev_line, *cur_line, *next_line;
7016 log_sz = strlen(buf) + 1;
7017 next_line = buf + log_sz - 1;
7019 for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) {
7020 cur_line = find_prev_line(buf, next_line);
7024 /* failed CO-RE relocation case */
7025 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) {
7026 prev_line = find_prev_line(buf, cur_line);
7030 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,
7031 prev_line, cur_line, next_line);
7033 } else if (str_has_pfx(cur_line, "invalid func unknown#"MAP_LDIMM64_POISON_PFX)) {
7034 prev_line = find_prev_line(buf, cur_line);
7038 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,
7039 prev_line, cur_line, next_line);
7045 static int bpf_program_record_relos(struct bpf_program *prog)
7047 struct bpf_object *obj = prog->obj;
7050 for (i = 0; i < prog->nr_reloc; i++) {
7051 struct reloc_desc *relo = &prog->reloc_desc[i];
7052 struct extern_desc *ext = &obj->externs[relo->sym_off];
7054 switch (relo->type) {
7055 case RELO_EXTERN_VAR:
7056 if (ext->type != EXT_KSYM)
7058 bpf_gen__record_extern(obj->gen_loader, ext->name,
7059 ext->is_weak, !ext->ksym.type_id,
7060 BTF_KIND_VAR, relo->insn_idx);
7062 case RELO_EXTERN_FUNC:
7063 bpf_gen__record_extern(obj->gen_loader, ext->name,
7064 ext->is_weak, false, BTF_KIND_FUNC,
7068 struct bpf_core_relo cr = {
7069 .insn_off = relo->insn_idx * 8,
7070 .type_id = relo->core_relo->type_id,
7071 .access_str_off = relo->core_relo->access_str_off,
7072 .kind = relo->core_relo->kind,
7075 bpf_gen__record_relo_core(obj->gen_loader, &cr);
7086 bpf_object__load_progs(struct bpf_object *obj, int log_level)
7088 struct bpf_program *prog;
7092 for (i = 0; i < obj->nr_programs; i++) {
7093 prog = &obj->programs[i];
7094 err = bpf_object__sanitize_prog(obj, prog);
7099 for (i = 0; i < obj->nr_programs; i++) {
7100 prog = &obj->programs[i];
7101 if (prog_is_subprog(obj, prog))
7103 if (!prog->autoload) {
7104 pr_debug("prog '%s': skipped loading\n", prog->name);
7107 prog->log_level |= log_level;
7109 if (obj->gen_loader)
7110 bpf_program_record_relos(prog);
7112 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7113 obj->license, obj->kern_version, &prog->fd);
7115 pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7120 bpf_object__free_relocs(obj);
7124 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
7126 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)
7128 struct bpf_program *prog;
7131 bpf_object__for_each_program(prog, obj) {
7132 prog->sec_def = find_sec_def(prog->sec_name);
7133 if (!prog->sec_def) {
7134 /* couldn't guess, but user might manually specify */
7135 pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7136 prog->name, prog->sec_name);
7140 prog->type = prog->sec_def->prog_type;
7141 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7143 /* sec_def can have custom callback which should be called
7144 * after bpf_program is initialized to adjust its properties
7146 if (prog->sec_def->prog_setup_fn) {
7147 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7149 pr_warn("prog '%s': failed to initialize: %d\n",
7159 static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz,
7160 const struct bpf_object_open_opts *opts)
7162 const char *obj_name, *kconfig, *btf_tmp_path;
7163 struct bpf_object *obj;
7170 if (elf_version(EV_CURRENT) == EV_NONE) {
7171 pr_warn("failed to init libelf for %s\n",
7172 path ? : "(mem buf)");
7173 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7176 if (!OPTS_VALID(opts, bpf_object_open_opts))
7177 return ERR_PTR(-EINVAL);
7179 obj_name = OPTS_GET(opts, object_name, NULL);
7182 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7183 (unsigned long)obj_buf,
7184 (unsigned long)obj_buf_sz);
7185 obj_name = tmp_name;
7188 pr_debug("loading object '%s' from buffer\n", obj_name);
7191 log_buf = OPTS_GET(opts, kernel_log_buf, NULL);
7192 log_size = OPTS_GET(opts, kernel_log_size, 0);
7193 log_level = OPTS_GET(opts, kernel_log_level, 0);
7194 if (log_size > UINT_MAX)
7195 return ERR_PTR(-EINVAL);
7196 if (log_size && !log_buf)
7197 return ERR_PTR(-EINVAL);
7199 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7203 obj->log_buf = log_buf;
7204 obj->log_size = log_size;
7205 obj->log_level = log_level;
7207 btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL);
7209 if (strlen(btf_tmp_path) >= PATH_MAX) {
7210 err = -ENAMETOOLONG;
7213 obj->btf_custom_path = strdup(btf_tmp_path);
7214 if (!obj->btf_custom_path) {
7220 kconfig = OPTS_GET(opts, kconfig, NULL);
7222 obj->kconfig = strdup(kconfig);
7223 if (!obj->kconfig) {
7229 err = bpf_object__elf_init(obj);
7230 err = err ? : bpf_object__check_endianness(obj);
7231 err = err ? : bpf_object__elf_collect(obj);
7232 err = err ? : bpf_object__collect_externs(obj);
7233 err = err ? : bpf_object__finalize_btf(obj);
7234 err = err ? : bpf_object__init_maps(obj, opts);
7235 err = err ? : bpf_object_init_progs(obj, opts);
7236 err = err ? : bpf_object__collect_relos(obj);
7240 bpf_object__elf_finish(obj);
7244 bpf_object__close(obj);
7245 return ERR_PTR(err);
7249 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
7252 return libbpf_err_ptr(-EINVAL);
7254 pr_debug("loading %s\n", path);
7256 return libbpf_ptr(bpf_object_open(path, NULL, 0, opts));
7259 struct bpf_object *bpf_object__open(const char *path)
7261 return bpf_object__open_file(path, NULL);
7265 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
7266 const struct bpf_object_open_opts *opts)
7268 if (!obj_buf || obj_buf_sz == 0)
7269 return libbpf_err_ptr(-EINVAL);
7271 return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts));
7274 static int bpf_object_unload(struct bpf_object *obj)
7279 return libbpf_err(-EINVAL);
7281 for (i = 0; i < obj->nr_maps; i++) {
7282 zclose(obj->maps[i].fd);
7283 if (obj->maps[i].st_ops)
7284 zfree(&obj->maps[i].st_ops->kern_vdata);
7287 for (i = 0; i < obj->nr_programs; i++)
7288 bpf_program__unload(&obj->programs[i]);
7293 static int bpf_object__sanitize_maps(struct bpf_object *obj)
7297 bpf_object__for_each_map(m, obj) {
7298 if (!bpf_map__is_internal(m))
7300 if (!kernel_supports(obj, FEAT_ARRAY_MMAP))
7301 m->def.map_flags ^= BPF_F_MMAPABLE;
7307 int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx)
7309 char sym_type, sym_name[500];
7310 unsigned long long sym_addr;
7314 f = fopen("/proc/kallsyms", "r");
7317 pr_warn("failed to open /proc/kallsyms: %d\n", err);
7322 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
7323 &sym_addr, &sym_type, sym_name);
7324 if (ret == EOF && feof(f))
7327 pr_warn("failed to read kallsyms entry: %d\n", ret);
7332 err = cb(sym_addr, sym_type, sym_name, ctx);
7341 static int kallsyms_cb(unsigned long long sym_addr, char sym_type,
7342 const char *sym_name, void *ctx)
7344 struct bpf_object *obj = ctx;
7345 const struct btf_type *t;
7346 struct extern_desc *ext;
7348 ext = find_extern_by_name(obj, sym_name);
7349 if (!ext || ext->type != EXT_KSYM)
7352 t = btf__type_by_id(obj->btf, ext->btf_id);
7356 if (ext->is_set && ext->ksym.addr != sym_addr) {
7357 pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n",
7358 sym_name, ext->ksym.addr, sym_addr);
7363 ext->ksym.addr = sym_addr;
7364 pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr);
7369 static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
7371 return libbpf_kallsyms_parse(kallsyms_cb, obj);
7374 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
7375 __u16 kind, struct btf **res_btf,
7376 struct module_btf **res_mod_btf)
7378 struct module_btf *mod_btf;
7382 btf = obj->btf_vmlinux;
7384 id = btf__find_by_name_kind(btf, ksym_name, kind);
7386 if (id == -ENOENT) {
7387 err = load_module_btfs(obj);
7391 for (i = 0; i < obj->btf_module_cnt; i++) {
7392 /* we assume module_btf's BTF FD is always >0 */
7393 mod_btf = &obj->btf_modules[i];
7395 id = btf__find_by_name_kind_own(btf, ksym_name, kind);
7404 *res_mod_btf = mod_btf;
7408 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
7409 struct extern_desc *ext)
7411 const struct btf_type *targ_var, *targ_type;
7412 __u32 targ_type_id, local_type_id;
7413 struct module_btf *mod_btf = NULL;
7414 const char *targ_var_name;
7415 struct btf *btf = NULL;
7418 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
7420 if (id == -ESRCH && ext->is_weak)
7422 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n",
7427 /* find local type_id */
7428 local_type_id = ext->ksym.type_id;
7430 /* find target type_id */
7431 targ_var = btf__type_by_id(btf, id);
7432 targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
7433 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
7435 err = bpf_core_types_are_compat(obj->btf, local_type_id,
7438 const struct btf_type *local_type;
7439 const char *targ_name, *local_name;
7441 local_type = btf__type_by_id(obj->btf, local_type_id);
7442 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
7443 targ_name = btf__name_by_offset(btf, targ_type->name_off);
7445 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7446 ext->name, local_type_id,
7447 btf_kind_str(local_type), local_name, targ_type_id,
7448 btf_kind_str(targ_type), targ_name);
7453 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
7454 ext->ksym.kernel_btf_id = id;
7455 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
7456 ext->name, id, btf_kind_str(targ_var), targ_var_name);
7461 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
7462 struct extern_desc *ext)
7464 int local_func_proto_id, kfunc_proto_id, kfunc_id;
7465 struct module_btf *mod_btf = NULL;
7466 const struct btf_type *kern_func;
7467 struct btf *kern_btf = NULL;
7470 local_func_proto_id = ext->ksym.type_id;
7472 kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, &kern_btf, &mod_btf);
7474 if (kfunc_id == -ESRCH && ext->is_weak)
7476 pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n",
7481 kern_func = btf__type_by_id(kern_btf, kfunc_id);
7482 kfunc_proto_id = kern_func->type;
7484 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
7485 kern_btf, kfunc_proto_id);
7487 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n",
7488 ext->name, local_func_proto_id, kfunc_proto_id);
7492 /* set index for module BTF fd in fd_array, if unset */
7493 if (mod_btf && !mod_btf->fd_array_idx) {
7494 /* insn->off is s16 */
7495 if (obj->fd_array_cnt == INT16_MAX) {
7496 pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n",
7497 ext->name, mod_btf->fd_array_idx);
7500 /* Cannot use index 0 for module BTF fd */
7501 if (!obj->fd_array_cnt)
7502 obj->fd_array_cnt = 1;
7504 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
7505 obj->fd_array_cnt + 1);
7508 mod_btf->fd_array_idx = obj->fd_array_cnt;
7509 /* we assume module BTF FD is always >0 */
7510 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
7514 ext->ksym.kernel_btf_id = kfunc_id;
7515 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
7516 pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n",
7517 ext->name, kfunc_id);
7522 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7524 const struct btf_type *t;
7525 struct extern_desc *ext;
7528 for (i = 0; i < obj->nr_extern; i++) {
7529 ext = &obj->externs[i];
7530 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7533 if (obj->gen_loader) {
7535 ext->ksym.kernel_btf_obj_fd = 0;
7536 ext->ksym.kernel_btf_id = 0;
7539 t = btf__type_by_id(obj->btf, ext->btf_id);
7541 err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
7543 err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
7550 static int bpf_object__resolve_externs(struct bpf_object *obj,
7551 const char *extra_kconfig)
7553 bool need_config = false, need_kallsyms = false;
7554 bool need_vmlinux_btf = false;
7555 struct extern_desc *ext;
7556 void *kcfg_data = NULL;
7559 if (obj->nr_extern == 0)
7562 if (obj->kconfig_map_idx >= 0)
7563 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7565 for (i = 0; i < obj->nr_extern; i++) {
7566 ext = &obj->externs[i];
7568 if (ext->type == EXT_KSYM) {
7569 if (ext->ksym.type_id)
7570 need_vmlinux_btf = true;
7572 need_kallsyms = true;
7574 } else if (ext->type == EXT_KCFG) {
7575 void *ext_ptr = kcfg_data + ext->kcfg.data_off;
7578 /* Kconfig externs need actual /proc/config.gz */
7579 if (str_has_pfx(ext->name, "CONFIG_")) {
7584 /* Virtual kcfg externs are customly handled by libbpf */
7585 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
7586 value = get_kernel_version();
7588 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
7591 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
7592 value = kernel_supports(obj, FEAT_BPF_COOKIE);
7593 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
7594 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER);
7595 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
7596 /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed
7597 * __kconfig externs, where LINUX_ ones are virtual and filled out
7598 * customly by libbpf (their values don't come from Kconfig).
7599 * If LINUX_xxx variable is not recognized by libbpf, but is marked
7600 * __weak, it defaults to zero value, just like for CONFIG_xxx
7603 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
7607 err = set_kcfg_value_num(ext, ext_ptr, value);
7610 pr_debug("extern (kcfg) '%s': set to 0x%llx\n",
7611 ext->name, (long long)value);
7613 pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
7617 if (need_config && extra_kconfig) {
7618 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
7621 need_config = false;
7622 for (i = 0; i < obj->nr_extern; i++) {
7623 ext = &obj->externs[i];
7624 if (ext->type == EXT_KCFG && !ext->is_set) {
7631 err = bpf_object__read_kconfig_file(obj, kcfg_data);
7635 if (need_kallsyms) {
7636 err = bpf_object__read_kallsyms_file(obj);
7640 if (need_vmlinux_btf) {
7641 err = bpf_object__resolve_ksyms_btf_id(obj);
7645 for (i = 0; i < obj->nr_extern; i++) {
7646 ext = &obj->externs[i];
7648 if (!ext->is_set && !ext->is_weak) {
7649 pr_warn("extern '%s' (strong): not resolved\n", ext->name);
7651 } else if (!ext->is_set) {
7652 pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n",
7660 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path)
7665 return libbpf_err(-EINVAL);
7668 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
7669 return libbpf_err(-EINVAL);
7672 if (obj->gen_loader)
7673 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
7675 err = bpf_object__probe_loading(obj);
7676 err = err ? : bpf_object__load_vmlinux_btf(obj, false);
7677 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7678 err = err ? : bpf_object__sanitize_and_load_btf(obj);
7679 err = err ? : bpf_object__sanitize_maps(obj);
7680 err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7681 err = err ? : bpf_object__create_maps(obj);
7682 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
7683 err = err ? : bpf_object__load_progs(obj, extra_log_level);
7684 err = err ? : bpf_object_init_prog_arrays(obj);
7686 if (obj->gen_loader) {
7689 btf__set_fd(obj->btf, -1);
7690 for (i = 0; i < obj->nr_maps; i++)
7691 obj->maps[i].fd = -1;
7693 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
7696 /* clean up fd_array */
7697 zfree(&obj->fd_array);
7699 /* clean up module BTFs */
7700 for (i = 0; i < obj->btf_module_cnt; i++) {
7701 close(obj->btf_modules[i].fd);
7702 btf__free(obj->btf_modules[i].btf);
7703 free(obj->btf_modules[i].name);
7705 free(obj->btf_modules);
7707 /* clean up vmlinux BTF */
7708 btf__free(obj->btf_vmlinux);
7709 obj->btf_vmlinux = NULL;
7711 obj->loaded = true; /* doesn't matter if successfully or not */
7718 /* unpin any maps that were auto-pinned during load */
7719 for (i = 0; i < obj->nr_maps; i++)
7720 if (obj->maps[i].pinned && !obj->maps[i].reused)
7721 bpf_map__unpin(&obj->maps[i], NULL);
7723 bpf_object_unload(obj);
7724 pr_warn("failed to load object '%s'\n", obj->path);
7725 return libbpf_err(err);
7728 int bpf_object__load(struct bpf_object *obj)
7730 return bpf_object_load(obj, 0, NULL);
7733 static int make_parent_dir(const char *path)
7735 char *cp, errmsg[STRERR_BUFSIZE];
7739 dname = strdup(path);
7743 dir = dirname(dname);
7744 if (mkdir(dir, 0700) && errno != EEXIST)
7749 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7750 pr_warn("failed to mkdir %s: %s\n", path, cp);
7755 static int check_path(const char *path)
7757 char *cp, errmsg[STRERR_BUFSIZE];
7758 struct statfs st_fs;
7765 dname = strdup(path);
7769 dir = dirname(dname);
7770 if (statfs(dir, &st_fs)) {
7771 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7772 pr_warn("failed to statfs %s: %s\n", dir, cp);
7777 if (!err && st_fs.f_type != BPF_FS_MAGIC) {
7778 pr_warn("specified path %s is not on BPF FS\n", path);
7785 int bpf_program__pin(struct bpf_program *prog, const char *path)
7787 char *cp, errmsg[STRERR_BUFSIZE];
7791 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
7792 return libbpf_err(-EINVAL);
7795 err = make_parent_dir(path);
7797 return libbpf_err(err);
7799 err = check_path(path);
7801 return libbpf_err(err);
7803 if (bpf_obj_pin(prog->fd, path)) {
7805 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
7806 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
7807 return libbpf_err(err);
7810 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
7814 int bpf_program__unpin(struct bpf_program *prog, const char *path)
7819 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
7820 return libbpf_err(-EINVAL);
7823 err = check_path(path);
7825 return libbpf_err(err);
7829 return libbpf_err(-errno);
7831 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
7835 int bpf_map__pin(struct bpf_map *map, const char *path)
7837 char *cp, errmsg[STRERR_BUFSIZE];
7841 pr_warn("invalid map pointer\n");
7842 return libbpf_err(-EINVAL);
7845 if (map->pin_path) {
7846 if (path && strcmp(path, map->pin_path)) {
7847 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7848 bpf_map__name(map), map->pin_path, path);
7849 return libbpf_err(-EINVAL);
7850 } else if (map->pinned) {
7851 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
7852 bpf_map__name(map), map->pin_path);
7857 pr_warn("missing a path to pin map '%s' at\n",
7858 bpf_map__name(map));
7859 return libbpf_err(-EINVAL);
7860 } else if (map->pinned) {
7861 pr_warn("map '%s' already pinned\n", bpf_map__name(map));
7862 return libbpf_err(-EEXIST);
7865 map->pin_path = strdup(path);
7866 if (!map->pin_path) {
7872 err = make_parent_dir(map->pin_path);
7874 return libbpf_err(err);
7876 err = check_path(map->pin_path);
7878 return libbpf_err(err);
7880 if (bpf_obj_pin(map->fd, map->pin_path)) {
7886 pr_debug("pinned map '%s'\n", map->pin_path);
7891 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7892 pr_warn("failed to pin map: %s\n", cp);
7893 return libbpf_err(err);
7896 int bpf_map__unpin(struct bpf_map *map, const char *path)
7901 pr_warn("invalid map pointer\n");
7902 return libbpf_err(-EINVAL);
7905 if (map->pin_path) {
7906 if (path && strcmp(path, map->pin_path)) {
7907 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7908 bpf_map__name(map), map->pin_path, path);
7909 return libbpf_err(-EINVAL);
7911 path = map->pin_path;
7913 pr_warn("no path to unpin map '%s' from\n",
7914 bpf_map__name(map));
7915 return libbpf_err(-EINVAL);
7918 err = check_path(path);
7920 return libbpf_err(err);
7924 return libbpf_err(-errno);
7926 map->pinned = false;
7927 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
7932 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
7939 return libbpf_err(-errno);
7942 free(map->pin_path);
7943 map->pin_path = new;
7947 __alias(bpf_map__pin_path)
7948 const char *bpf_map__get_pin_path(const struct bpf_map *map);
7950 const char *bpf_map__pin_path(const struct bpf_map *map)
7952 return map->pin_path;
7955 bool bpf_map__is_pinned(const struct bpf_map *map)
7960 static void sanitize_pin_path(char *s)
7962 /* bpffs disallows periods in path names */
7970 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
7972 struct bpf_map *map;
7976 return libbpf_err(-ENOENT);
7979 pr_warn("object not yet loaded; load it first\n");
7980 return libbpf_err(-ENOENT);
7983 bpf_object__for_each_map(map, obj) {
7984 char *pin_path = NULL;
7987 if (!map->autocreate)
7991 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
7993 goto err_unpin_maps;
7994 sanitize_pin_path(buf);
7996 } else if (!map->pin_path) {
8000 err = bpf_map__pin(map, pin_path);
8002 goto err_unpin_maps;
8008 while ((map = bpf_object__prev_map(obj, map))) {
8012 bpf_map__unpin(map, NULL);
8015 return libbpf_err(err);
8018 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
8020 struct bpf_map *map;
8024 return libbpf_err(-ENOENT);
8026 bpf_object__for_each_map(map, obj) {
8027 char *pin_path = NULL;
8031 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
8033 return libbpf_err(err);
8034 sanitize_pin_path(buf);
8036 } else if (!map->pin_path) {
8040 err = bpf_map__unpin(map, pin_path);
8042 return libbpf_err(err);
8048 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
8050 struct bpf_program *prog;
8055 return libbpf_err(-ENOENT);
8058 pr_warn("object not yet loaded; load it first\n");
8059 return libbpf_err(-ENOENT);
8062 bpf_object__for_each_program(prog, obj) {
8063 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8065 goto err_unpin_programs;
8067 err = bpf_program__pin(prog, buf);
8069 goto err_unpin_programs;
8075 while ((prog = bpf_object__prev_program(obj, prog))) {
8076 if (pathname_concat(buf, sizeof(buf), path, prog->name))
8079 bpf_program__unpin(prog, buf);
8082 return libbpf_err(err);
8085 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8087 struct bpf_program *prog;
8091 return libbpf_err(-ENOENT);
8093 bpf_object__for_each_program(prog, obj) {
8096 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8098 return libbpf_err(err);
8100 err = bpf_program__unpin(prog, buf);
8102 return libbpf_err(err);
8108 int bpf_object__pin(struct bpf_object *obj, const char *path)
8112 err = bpf_object__pin_maps(obj, path);
8114 return libbpf_err(err);
8116 err = bpf_object__pin_programs(obj, path);
8118 bpf_object__unpin_maps(obj, path);
8119 return libbpf_err(err);
8125 static void bpf_map__destroy(struct bpf_map *map)
8127 if (map->inner_map) {
8128 bpf_map__destroy(map->inner_map);
8129 zfree(&map->inner_map);
8132 zfree(&map->init_slots);
8133 map->init_slots_sz = 0;
8136 munmap(map->mmaped, bpf_map_mmap_sz(map));
8141 zfree(&map->st_ops->data);
8142 zfree(&map->st_ops->progs);
8143 zfree(&map->st_ops->kern_func_off);
8144 zfree(&map->st_ops);
8148 zfree(&map->real_name);
8149 zfree(&map->pin_path);
8155 void bpf_object__close(struct bpf_object *obj)
8159 if (IS_ERR_OR_NULL(obj))
8162 usdt_manager_free(obj->usdt_man);
8163 obj->usdt_man = NULL;
8165 bpf_gen__free(obj->gen_loader);
8166 bpf_object__elf_finish(obj);
8167 bpf_object_unload(obj);
8168 btf__free(obj->btf);
8169 btf_ext__free(obj->btf_ext);
8171 for (i = 0; i < obj->nr_maps; i++)
8172 bpf_map__destroy(&obj->maps[i]);
8174 zfree(&obj->btf_custom_path);
8175 zfree(&obj->kconfig);
8176 zfree(&obj->externs);
8182 if (obj->programs && obj->nr_programs) {
8183 for (i = 0; i < obj->nr_programs; i++)
8184 bpf_program__exit(&obj->programs[i]);
8186 zfree(&obj->programs);
8191 const char *bpf_object__name(const struct bpf_object *obj)
8193 return obj ? obj->name : libbpf_err_ptr(-EINVAL);
8196 unsigned int bpf_object__kversion(const struct bpf_object *obj)
8198 return obj ? obj->kern_version : 0;
8201 struct btf *bpf_object__btf(const struct bpf_object *obj)
8203 return obj ? obj->btf : NULL;
8206 int bpf_object__btf_fd(const struct bpf_object *obj)
8208 return obj->btf ? btf__fd(obj->btf) : -1;
8211 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
8214 return libbpf_err(-EINVAL);
8216 obj->kern_version = kern_version;
8221 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
8223 struct bpf_gen *gen;
8227 if (!OPTS_VALID(opts, gen_loader_opts))
8229 gen = calloc(sizeof(*gen), 1);
8233 obj->gen_loader = gen;
8237 static struct bpf_program *
8238 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
8241 size_t nr_programs = obj->nr_programs;
8248 /* Iter from the beginning */
8249 return forward ? &obj->programs[0] :
8250 &obj->programs[nr_programs - 1];
8252 if (p->obj != obj) {
8253 pr_warn("error: program handler doesn't match object\n");
8254 return errno = EINVAL, NULL;
8257 idx = (p - obj->programs) + (forward ? 1 : -1);
8258 if (idx >= obj->nr_programs || idx < 0)
8260 return &obj->programs[idx];
8263 struct bpf_program *
8264 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
8266 struct bpf_program *prog = prev;
8269 prog = __bpf_program__iter(prog, obj, true);
8270 } while (prog && prog_is_subprog(obj, prog));
8275 struct bpf_program *
8276 bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)
8278 struct bpf_program *prog = next;
8281 prog = __bpf_program__iter(prog, obj, false);
8282 } while (prog && prog_is_subprog(obj, prog));
8287 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
8289 prog->prog_ifindex = ifindex;
8292 const char *bpf_program__name(const struct bpf_program *prog)
8297 const char *bpf_program__section_name(const struct bpf_program *prog)
8299 return prog->sec_name;
8302 bool bpf_program__autoload(const struct bpf_program *prog)
8304 return prog->autoload;
8307 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
8309 if (prog->obj->loaded)
8310 return libbpf_err(-EINVAL);
8312 prog->autoload = autoload;
8316 bool bpf_program__autoattach(const struct bpf_program *prog)
8318 return prog->autoattach;
8321 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)
8323 prog->autoattach = autoattach;
8326 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
8331 size_t bpf_program__insn_cnt(const struct bpf_program *prog)
8333 return prog->insns_cnt;
8336 int bpf_program__set_insns(struct bpf_program *prog,
8337 struct bpf_insn *new_insns, size_t new_insn_cnt)
8339 struct bpf_insn *insns;
8341 if (prog->obj->loaded)
8344 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
8346 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
8349 memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));
8351 prog->insns = insns;
8352 prog->insns_cnt = new_insn_cnt;
8356 int bpf_program__fd(const struct bpf_program *prog)
8359 return libbpf_err(-EINVAL);
8362 return libbpf_err(-ENOENT);
8367 __alias(bpf_program__type)
8368 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
8370 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
8375 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
8377 if (prog->obj->loaded)
8378 return libbpf_err(-EBUSY);
8384 __alias(bpf_program__expected_attach_type)
8385 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
8387 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)
8389 return prog->expected_attach_type;
8392 int bpf_program__set_expected_attach_type(struct bpf_program *prog,
8393 enum bpf_attach_type type)
8395 if (prog->obj->loaded)
8396 return libbpf_err(-EBUSY);
8398 prog->expected_attach_type = type;
8402 __u32 bpf_program__flags(const struct bpf_program *prog)
8404 return prog->prog_flags;
8407 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
8409 if (prog->obj->loaded)
8410 return libbpf_err(-EBUSY);
8412 prog->prog_flags = flags;
8416 __u32 bpf_program__log_level(const struct bpf_program *prog)
8418 return prog->log_level;
8421 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
8423 if (prog->obj->loaded)
8424 return libbpf_err(-EBUSY);
8426 prog->log_level = log_level;
8430 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size)
8432 *log_size = prog->log_size;
8433 return prog->log_buf;
8436 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
8438 if (log_size && !log_buf)
8440 if (prog->log_size > UINT_MAX)
8442 if (prog->obj->loaded)
8445 prog->log_buf = log_buf;
8446 prog->log_size = log_size;
8450 #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \
8451 .sec = (char *)sec_pfx, \
8452 .prog_type = BPF_PROG_TYPE_##ptype, \
8453 .expected_attach_type = atype, \
8454 .cookie = (long)(flags), \
8455 .prog_prepare_load_fn = libbpf_prepare_prog_load, \
8459 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8460 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8461 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8462 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8463 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8464 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8465 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8466 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8467 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8468 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8470 static const struct bpf_sec_def section_defs[] = {
8471 SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE),
8472 SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE),
8473 SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE),
8474 SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),
8475 SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),
8476 SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
8477 SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),
8478 SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),
8479 SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
8480 SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
8481 SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
8482 SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
8483 SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
8484 SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt),
8485 SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE),
8486 SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE),
8487 SEC_DEF("action", SCHED_ACT, 0, SEC_NONE),
8488 SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp),
8489 SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp),
8490 SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
8491 SEC_DEF("raw_tp+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
8492 SEC_DEF("raw_tracepoint.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
8493 SEC_DEF("raw_tp.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
8494 SEC_DEF("tp_btf+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace),
8495 SEC_DEF("fentry+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace),
8496 SEC_DEF("fmod_ret+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace),
8497 SEC_DEF("fexit+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace),
8498 SEC_DEF("fentry.s+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8499 SEC_DEF("fmod_ret.s+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8500 SEC_DEF("fexit.s+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace),
8501 SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace),
8502 SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
8503 SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
8504 SEC_DEF("lsm_cgroup+", LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF),
8505 SEC_DEF("iter+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
8506 SEC_DEF("iter.s+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
8507 SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE),
8508 SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS),
8509 SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
8510 SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS),
8511 SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
8512 SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS),
8513 SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT),
8514 SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE),
8515 SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE),
8516 SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE),
8517 SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE),
8518 SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE),
8519 SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
8520 SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
8521 SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
8522 SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE),
8523 SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
8524 SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
8525 SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT),
8526 SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT),
8527 SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT),
8528 SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE),
8529 SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE),
8530 SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE),
8531 SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT),
8532 SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE),
8533 SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE),
8534 SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE),
8535 SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE),
8536 SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE),
8537 SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE),
8538 SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE),
8539 SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE),
8540 SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE),
8541 SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE),
8542 SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE),
8543 SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE),
8544 SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE),
8545 SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE),
8546 SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE),
8547 SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE),
8548 SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE),
8549 SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),
8550 SEC_DEF("struct_ops+", STRUCT_OPS, 0, SEC_NONE),
8551 SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE),
8554 static size_t custom_sec_def_cnt;
8555 static struct bpf_sec_def *custom_sec_defs;
8556 static struct bpf_sec_def custom_fallback_def;
8557 static bool has_custom_fallback_def;
8559 static int last_custom_sec_def_handler_id;
8561 int libbpf_register_prog_handler(const char *sec,
8562 enum bpf_prog_type prog_type,
8563 enum bpf_attach_type exp_attach_type,
8564 const struct libbpf_prog_handler_opts *opts)
8566 struct bpf_sec_def *sec_def;
8568 if (!OPTS_VALID(opts, libbpf_prog_handler_opts))
8569 return libbpf_err(-EINVAL);
8571 if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */
8572 return libbpf_err(-E2BIG);
8575 sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1,
8578 return libbpf_err(-ENOMEM);
8580 custom_sec_defs = sec_def;
8581 sec_def = &custom_sec_defs[custom_sec_def_cnt];
8583 if (has_custom_fallback_def)
8584 return libbpf_err(-EBUSY);
8586 sec_def = &custom_fallback_def;
8589 sec_def->sec = sec ? strdup(sec) : NULL;
8590 if (sec && !sec_def->sec)
8591 return libbpf_err(-ENOMEM);
8593 sec_def->prog_type = prog_type;
8594 sec_def->expected_attach_type = exp_attach_type;
8595 sec_def->cookie = OPTS_GET(opts, cookie, 0);
8597 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);
8598 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);
8599 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);
8601 sec_def->handler_id = ++last_custom_sec_def_handler_id;
8604 custom_sec_def_cnt++;
8606 has_custom_fallback_def = true;
8608 return sec_def->handler_id;
8611 int libbpf_unregister_prog_handler(int handler_id)
8613 struct bpf_sec_def *sec_defs;
8616 if (handler_id <= 0)
8617 return libbpf_err(-EINVAL);
8619 if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) {
8620 memset(&custom_fallback_def, 0, sizeof(custom_fallback_def));
8621 has_custom_fallback_def = false;
8625 for (i = 0; i < custom_sec_def_cnt; i++) {
8626 if (custom_sec_defs[i].handler_id == handler_id)
8630 if (i == custom_sec_def_cnt)
8631 return libbpf_err(-ENOENT);
8633 free(custom_sec_defs[i].sec);
8634 for (i = i + 1; i < custom_sec_def_cnt; i++)
8635 custom_sec_defs[i - 1] = custom_sec_defs[i];
8636 custom_sec_def_cnt--;
8638 /* try to shrink the array, but it's ok if we couldn't */
8639 sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs));
8641 custom_sec_defs = sec_defs;
8646 static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name)
8648 size_t len = strlen(sec_def->sec);
8650 /* "type/" always has to have proper SEC("type/extras") form */
8651 if (sec_def->sec[len - 1] == '/') {
8652 if (str_has_pfx(sec_name, sec_def->sec))
8657 /* "type+" means it can be either exact SEC("type") or
8658 * well-formed SEC("type/extras") with proper '/' separator
8660 if (sec_def->sec[len - 1] == '+') {
8662 /* not even a prefix */
8663 if (strncmp(sec_name, sec_def->sec, len) != 0)
8665 /* exact match or has '/' separator */
8666 if (sec_name[len] == '\0' || sec_name[len] == '/')
8671 return strcmp(sec_name, sec_def->sec) == 0;
8674 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
8676 const struct bpf_sec_def *sec_def;
8679 n = custom_sec_def_cnt;
8680 for (i = 0; i < n; i++) {
8681 sec_def = &custom_sec_defs[i];
8682 if (sec_def_matches(sec_def, sec_name))
8686 n = ARRAY_SIZE(section_defs);
8687 for (i = 0; i < n; i++) {
8688 sec_def = §ion_defs[i];
8689 if (sec_def_matches(sec_def, sec_name))
8693 if (has_custom_fallback_def)
8694 return &custom_fallback_def;
8699 #define MAX_TYPE_NAME_SIZE 32
8701 static char *libbpf_get_type_names(bool attach_type)
8703 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
8711 /* Forge string buf with all available names */
8712 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8713 const struct bpf_sec_def *sec_def = §ion_defs[i];
8716 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
8719 if (!(sec_def->cookie & SEC_ATTACHABLE))
8723 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
8728 strcat(buf, section_defs[i].sec);
8734 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
8735 enum bpf_attach_type *expected_attach_type)
8737 const struct bpf_sec_def *sec_def;
8741 return libbpf_err(-EINVAL);
8743 sec_def = find_sec_def(name);
8745 *prog_type = sec_def->prog_type;
8746 *expected_attach_type = sec_def->expected_attach_type;
8750 pr_debug("failed to guess program type from ELF section '%s'\n", name);
8751 type_names = libbpf_get_type_names(false);
8752 if (type_names != NULL) {
8753 pr_debug("supported section(type) names are:%s\n", type_names);
8757 return libbpf_err(-ESRCH);
8760 const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t)
8762 if (t < 0 || t >= ARRAY_SIZE(attach_type_name))
8765 return attach_type_name[t];
8768 const char *libbpf_bpf_link_type_str(enum bpf_link_type t)
8770 if (t < 0 || t >= ARRAY_SIZE(link_type_name))
8773 return link_type_name[t];
8776 const char *libbpf_bpf_map_type_str(enum bpf_map_type t)
8778 if (t < 0 || t >= ARRAY_SIZE(map_type_name))
8781 return map_type_name[t];
8784 const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t)
8786 if (t < 0 || t >= ARRAY_SIZE(prog_type_name))
8789 return prog_type_name[t];
8792 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
8795 struct bpf_map *map;
8798 for (i = 0; i < obj->nr_maps; i++) {
8799 map = &obj->maps[i];
8800 if (!bpf_map__is_struct_ops(map))
8802 if (map->sec_offset <= offset &&
8803 offset - map->sec_offset < map->def.value_size)
8810 /* Collect the reloc from ELF and populate the st_ops->progs[] */
8811 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
8812 Elf64_Shdr *shdr, Elf_Data *data)
8814 const struct btf_member *member;
8815 struct bpf_struct_ops *st_ops;
8816 struct bpf_program *prog;
8817 unsigned int shdr_idx;
8818 const struct btf *btf;
8819 struct bpf_map *map;
8820 unsigned int moff, insn_idx;
8828 nrels = shdr->sh_size / shdr->sh_entsize;
8829 for (i = 0; i < nrels; i++) {
8830 rel = elf_rel_by_idx(data, i);
8832 pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
8833 return -LIBBPF_ERRNO__FORMAT;
8836 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
8838 pr_warn("struct_ops reloc: symbol %zx not found\n",
8839 (size_t)ELF64_R_SYM(rel->r_info));
8840 return -LIBBPF_ERRNO__FORMAT;
8843 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
8844 map = find_struct_ops_map_by_offset(obj, rel->r_offset);
8846 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",
8847 (size_t)rel->r_offset);
8851 moff = rel->r_offset - map->sec_offset;
8852 shdr_idx = sym->st_shndx;
8853 st_ops = map->st_ops;
8854 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
8856 (long long)(rel->r_info >> 32),
8857 (long long)sym->st_value,
8858 shdr_idx, (size_t)rel->r_offset,
8859 map->sec_offset, sym->st_name, name);
8861 if (shdr_idx >= SHN_LORESERVE) {
8862 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",
8863 map->name, (size_t)rel->r_offset, shdr_idx);
8864 return -LIBBPF_ERRNO__RELOC;
8866 if (sym->st_value % BPF_INSN_SZ) {
8867 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
8868 map->name, (unsigned long long)sym->st_value);
8869 return -LIBBPF_ERRNO__FORMAT;
8871 insn_idx = sym->st_value / BPF_INSN_SZ;
8873 member = find_member_by_offset(st_ops->type, moff * 8);
8875 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
8879 member_idx = member - btf_members(st_ops->type);
8880 name = btf__name_by_offset(btf, member->name_off);
8882 if (!resolve_func_ptr(btf, member->type, NULL)) {
8883 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
8888 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
8890 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
8891 map->name, shdr_idx, name);
8895 /* prevent the use of BPF prog with invalid type */
8896 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
8897 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
8898 map->name, prog->name);
8902 /* if we haven't yet processed this BPF program, record proper
8903 * attach_btf_id and member_idx
8905 if (!prog->attach_btf_id) {
8906 prog->attach_btf_id = st_ops->type_id;
8907 prog->expected_attach_type = member_idx;
8910 /* struct_ops BPF prog can be re-used between multiple
8911 * .struct_ops as long as it's the same struct_ops struct
8912 * definition and the same function pointer field
8914 if (prog->attach_btf_id != st_ops->type_id ||
8915 prog->expected_attach_type != member_idx) {
8916 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8917 map->name, prog->name, prog->sec_name, prog->type,
8918 prog->attach_btf_id, prog->expected_attach_type, name);
8922 st_ops->progs[member_idx] = prog;
8928 #define BTF_TRACE_PREFIX "btf_trace_"
8929 #define BTF_LSM_PREFIX "bpf_lsm_"
8930 #define BTF_ITER_PREFIX "bpf_iter_"
8931 #define BTF_MAX_NAME_SIZE 128
8933 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
8934 const char **prefix, int *kind)
8936 switch (attach_type) {
8937 case BPF_TRACE_RAW_TP:
8938 *prefix = BTF_TRACE_PREFIX;
8939 *kind = BTF_KIND_TYPEDEF;
8942 case BPF_LSM_CGROUP:
8943 *prefix = BTF_LSM_PREFIX;
8944 *kind = BTF_KIND_FUNC;
8946 case BPF_TRACE_ITER:
8947 *prefix = BTF_ITER_PREFIX;
8948 *kind = BTF_KIND_FUNC;
8952 *kind = BTF_KIND_FUNC;
8956 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
8957 const char *name, __u32 kind)
8959 char btf_type_name[BTF_MAX_NAME_SIZE];
8962 ret = snprintf(btf_type_name, sizeof(btf_type_name),
8963 "%s%s", prefix, name);
8964 /* snprintf returns the number of characters written excluding the
8965 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
8966 * indicates truncation.
8968 if (ret < 0 || ret >= sizeof(btf_type_name))
8969 return -ENAMETOOLONG;
8970 return btf__find_by_name_kind(btf, btf_type_name, kind);
8973 static inline int find_attach_btf_id(struct btf *btf, const char *name,
8974 enum bpf_attach_type attach_type)
8979 btf_get_kernel_prefix_kind(attach_type, &prefix, &kind);
8980 return find_btf_by_prefix_kind(btf, prefix, name, kind);
8983 int libbpf_find_vmlinux_btf_id(const char *name,
8984 enum bpf_attach_type attach_type)
8989 btf = btf__load_vmlinux_btf();
8990 err = libbpf_get_error(btf);
8992 pr_warn("vmlinux BTF is not found\n");
8993 return libbpf_err(err);
8996 err = find_attach_btf_id(btf, name, attach_type);
8998 pr_warn("%s is not found in vmlinux BTF\n", name);
9001 return libbpf_err(err);
9004 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
9006 struct bpf_prog_info info;
9007 __u32 info_len = sizeof(info);
9011 memset(&info, 0, info_len);
9012 err = bpf_obj_get_info_by_fd(attach_prog_fd, &info, &info_len);
9014 pr_warn("failed bpf_obj_get_info_by_fd for FD %d: %d\n",
9015 attach_prog_fd, err);
9021 pr_warn("The target program doesn't have BTF\n");
9024 btf = btf__load_from_kernel_by_id(info.btf_id);
9025 err = libbpf_get_error(btf);
9027 pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err);
9030 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
9033 pr_warn("%s is not found in prog's BTF\n", name);
9040 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
9041 enum bpf_attach_type attach_type,
9042 int *btf_obj_fd, int *btf_type_id)
9046 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
9048 *btf_obj_fd = 0; /* vmlinux BTF */
9055 ret = load_module_btfs(obj);
9059 for (i = 0; i < obj->btf_module_cnt; i++) {
9060 const struct module_btf *mod = &obj->btf_modules[i];
9062 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
9064 *btf_obj_fd = mod->fd;
9077 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
9078 int *btf_obj_fd, int *btf_type_id)
9080 enum bpf_attach_type attach_type = prog->expected_attach_type;
9081 __u32 attach_prog_fd = prog->attach_prog_fd;
9084 /* BPF program's BTF ID */
9085 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
9086 if (!attach_prog_fd) {
9087 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
9090 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
9092 pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
9093 prog->name, attach_prog_fd, attach_name, err);
9101 /* kernel/module BTF ID */
9102 if (prog->obj->gen_loader) {
9103 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9107 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9110 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",
9111 prog->name, attach_name, err);
9117 int libbpf_attach_type_by_name(const char *name,
9118 enum bpf_attach_type *attach_type)
9121 const struct bpf_sec_def *sec_def;
9124 return libbpf_err(-EINVAL);
9126 sec_def = find_sec_def(name);
9128 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
9129 type_names = libbpf_get_type_names(true);
9130 if (type_names != NULL) {
9131 pr_debug("attachable section(type) names are:%s\n", type_names);
9135 return libbpf_err(-EINVAL);
9138 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)
9139 return libbpf_err(-EINVAL);
9140 if (!(sec_def->cookie & SEC_ATTACHABLE))
9141 return libbpf_err(-EINVAL);
9143 *attach_type = sec_def->expected_attach_type;
9147 int bpf_map__fd(const struct bpf_map *map)
9149 return map ? map->fd : libbpf_err(-EINVAL);
9152 static bool map_uses_real_name(const struct bpf_map *map)
9154 /* Since libbpf started to support custom .data.* and .rodata.* maps,
9155 * their user-visible name differs from kernel-visible name. Users see
9156 * such map's corresponding ELF section name as a map name.
9157 * This check distinguishes .data/.rodata from .data.* and .rodata.*
9158 * maps to know which name has to be returned to the user.
9160 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)
9162 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)
9167 const char *bpf_map__name(const struct bpf_map *map)
9172 if (map_uses_real_name(map))
9173 return map->real_name;
9178 enum bpf_map_type bpf_map__type(const struct bpf_map *map)
9180 return map->def.type;
9183 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
9186 return libbpf_err(-EBUSY);
9187 map->def.type = type;
9191 __u32 bpf_map__map_flags(const struct bpf_map *map)
9193 return map->def.map_flags;
9196 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
9199 return libbpf_err(-EBUSY);
9200 map->def.map_flags = flags;
9204 __u64 bpf_map__map_extra(const struct bpf_map *map)
9206 return map->map_extra;
9209 int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra)
9212 return libbpf_err(-EBUSY);
9213 map->map_extra = map_extra;
9217 __u32 bpf_map__numa_node(const struct bpf_map *map)
9219 return map->numa_node;
9222 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
9225 return libbpf_err(-EBUSY);
9226 map->numa_node = numa_node;
9230 __u32 bpf_map__key_size(const struct bpf_map *map)
9232 return map->def.key_size;
9235 int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
9238 return libbpf_err(-EBUSY);
9239 map->def.key_size = size;
9243 __u32 bpf_map__value_size(const struct bpf_map *map)
9245 return map->def.value_size;
9248 int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
9251 return libbpf_err(-EBUSY);
9252 map->def.value_size = size;
9256 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
9258 return map ? map->btf_key_type_id : 0;
9261 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
9263 return map ? map->btf_value_type_id : 0;
9266 int bpf_map__set_initial_value(struct bpf_map *map,
9267 const void *data, size_t size)
9269 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
9270 size != map->def.value_size || map->fd >= 0)
9271 return libbpf_err(-EINVAL);
9273 memcpy(map->mmaped, data, size);
9277 const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize)
9281 *psize = map->def.value_size;
9285 bool bpf_map__is_internal(const struct bpf_map *map)
9287 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
9290 __u32 bpf_map__ifindex(const struct bpf_map *map)
9292 return map->map_ifindex;
9295 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
9298 return libbpf_err(-EBUSY);
9299 map->map_ifindex = ifindex;
9303 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
9305 if (!bpf_map_type__is_map_in_map(map->def.type)) {
9306 pr_warn("error: unsupported map type\n");
9307 return libbpf_err(-EINVAL);
9309 if (map->inner_map_fd != -1) {
9310 pr_warn("error: inner_map_fd already specified\n");
9311 return libbpf_err(-EINVAL);
9313 if (map->inner_map) {
9314 bpf_map__destroy(map->inner_map);
9315 zfree(&map->inner_map);
9317 map->inner_map_fd = fd;
9321 static struct bpf_map *
9322 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
9325 struct bpf_map *s, *e;
9327 if (!obj || !obj->maps)
9328 return errno = EINVAL, NULL;
9331 e = obj->maps + obj->nr_maps;
9333 if ((m < s) || (m >= e)) {
9334 pr_warn("error in %s: map handler doesn't belong to object\n",
9336 return errno = EINVAL, NULL;
9339 idx = (m - obj->maps) + i;
9340 if (idx >= obj->nr_maps || idx < 0)
9342 return &obj->maps[idx];
9346 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
9351 return __bpf_map__iter(prev, obj, 1);
9355 bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next)
9360 return obj->maps + obj->nr_maps - 1;
9363 return __bpf_map__iter(next, obj, -1);
9367 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
9369 struct bpf_map *pos;
9371 bpf_object__for_each_map(pos, obj) {
9372 /* if it's a special internal map name (which always starts
9373 * with dot) then check if that special name matches the
9374 * real map name (ELF section name)
9376 if (name[0] == '.') {
9377 if (pos->real_name && strcmp(pos->real_name, name) == 0)
9381 /* otherwise map name has to be an exact match */
9382 if (map_uses_real_name(pos)) {
9383 if (strcmp(pos->real_name, name) == 0)
9387 if (strcmp(pos->name, name) == 0)
9390 return errno = ENOENT, NULL;
9394 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
9396 return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
9399 static int validate_map_op(const struct bpf_map *map, size_t key_sz,
9400 size_t value_sz, bool check_value_sz)
9405 if (map->def.key_size != key_sz) {
9406 pr_warn("map '%s': unexpected key size %zu provided, expected %u\n",
9407 map->name, key_sz, map->def.key_size);
9411 if (!check_value_sz)
9414 switch (map->def.type) {
9415 case BPF_MAP_TYPE_PERCPU_ARRAY:
9416 case BPF_MAP_TYPE_PERCPU_HASH:
9417 case BPF_MAP_TYPE_LRU_PERCPU_HASH:
9418 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: {
9419 int num_cpu = libbpf_num_possible_cpus();
9420 size_t elem_sz = roundup(map->def.value_size, 8);
9422 if (value_sz != num_cpu * elem_sz) {
9423 pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
9424 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
9430 if (map->def.value_size != value_sz) {
9431 pr_warn("map '%s': unexpected value size %zu provided, expected %u\n",
9432 map->name, value_sz, map->def.value_size);
9440 int bpf_map__lookup_elem(const struct bpf_map *map,
9441 const void *key, size_t key_sz,
9442 void *value, size_t value_sz, __u64 flags)
9446 err = validate_map_op(map, key_sz, value_sz, true);
9448 return libbpf_err(err);
9450 return bpf_map_lookup_elem_flags(map->fd, key, value, flags);
9453 int bpf_map__update_elem(const struct bpf_map *map,
9454 const void *key, size_t key_sz,
9455 const void *value, size_t value_sz, __u64 flags)
9459 err = validate_map_op(map, key_sz, value_sz, true);
9461 return libbpf_err(err);
9463 return bpf_map_update_elem(map->fd, key, value, flags);
9466 int bpf_map__delete_elem(const struct bpf_map *map,
9467 const void *key, size_t key_sz, __u64 flags)
9471 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
9473 return libbpf_err(err);
9475 return bpf_map_delete_elem_flags(map->fd, key, flags);
9478 int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,
9479 const void *key, size_t key_sz,
9480 void *value, size_t value_sz, __u64 flags)
9484 err = validate_map_op(map, key_sz, value_sz, true);
9486 return libbpf_err(err);
9488 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);
9491 int bpf_map__get_next_key(const struct bpf_map *map,
9492 const void *cur_key, void *next_key, size_t key_sz)
9496 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */);
9498 return libbpf_err(err);
9500 return bpf_map_get_next_key(map->fd, cur_key, next_key);
9503 long libbpf_get_error(const void *ptr)
9505 if (!IS_ERR_OR_NULL(ptr))
9509 errno = -PTR_ERR(ptr);
9511 /* If ptr == NULL, then errno should be already set by the failing
9512 * API, because libbpf never returns NULL on success and it now always
9513 * sets errno on error. So no extra errno handling for ptr == NULL
9519 /* Replace link's underlying BPF program with the new one */
9520 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
9524 ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
9525 return libbpf_err_errno(ret);
9528 /* Release "ownership" of underlying BPF resource (typically, BPF program
9529 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
9530 * link, when destructed through bpf_link__destroy() call won't attempt to
9531 * detach/unregisted that BPF resource. This is useful in situations where,
9532 * say, attached BPF program has to outlive userspace program that attached it
9533 * in the system. Depending on type of BPF program, though, there might be
9534 * additional steps (like pinning BPF program in BPF FS) necessary to ensure
9535 * exit of userspace program doesn't trigger automatic detachment and clean up
9536 * inside the kernel.
9538 void bpf_link__disconnect(struct bpf_link *link)
9540 link->disconnected = true;
9543 int bpf_link__destroy(struct bpf_link *link)
9547 if (IS_ERR_OR_NULL(link))
9550 if (!link->disconnected && link->detach)
9551 err = link->detach(link);
9553 free(link->pin_path);
9555 link->dealloc(link);
9559 return libbpf_err(err);
9562 int bpf_link__fd(const struct bpf_link *link)
9567 const char *bpf_link__pin_path(const struct bpf_link *link)
9569 return link->pin_path;
9572 static int bpf_link__detach_fd(struct bpf_link *link)
9574 return libbpf_err_errno(close(link->fd));
9577 struct bpf_link *bpf_link__open(const char *path)
9579 struct bpf_link *link;
9582 fd = bpf_obj_get(path);
9585 pr_warn("failed to open link at %s: %d\n", path, fd);
9586 return libbpf_err_ptr(fd);
9589 link = calloc(1, sizeof(*link));
9592 return libbpf_err_ptr(-ENOMEM);
9594 link->detach = &bpf_link__detach_fd;
9597 link->pin_path = strdup(path);
9598 if (!link->pin_path) {
9599 bpf_link__destroy(link);
9600 return libbpf_err_ptr(-ENOMEM);
9606 int bpf_link__detach(struct bpf_link *link)
9608 return bpf_link_detach(link->fd) ? -errno : 0;
9611 int bpf_link__pin(struct bpf_link *link, const char *path)
9616 return libbpf_err(-EBUSY);
9617 err = make_parent_dir(path);
9619 return libbpf_err(err);
9620 err = check_path(path);
9622 return libbpf_err(err);
9624 link->pin_path = strdup(path);
9625 if (!link->pin_path)
9626 return libbpf_err(-ENOMEM);
9628 if (bpf_obj_pin(link->fd, link->pin_path)) {
9630 zfree(&link->pin_path);
9631 return libbpf_err(err);
9634 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
9638 int bpf_link__unpin(struct bpf_link *link)
9642 if (!link->pin_path)
9643 return libbpf_err(-EINVAL);
9645 err = unlink(link->pin_path);
9649 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
9650 zfree(&link->pin_path);
9654 struct bpf_link_perf {
9655 struct bpf_link link;
9657 /* legacy kprobe support: keep track of probe identifier and type */
9658 char *legacy_probe_name;
9659 bool legacy_is_kprobe;
9660 bool legacy_is_retprobe;
9663 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe);
9664 static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe);
9666 static int bpf_link_perf_detach(struct bpf_link *link)
9668 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9671 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
9674 if (perf_link->perf_event_fd != link->fd)
9675 close(perf_link->perf_event_fd);
9678 /* legacy uprobe/kprobe needs to be removed after perf event fd closure */
9679 if (perf_link->legacy_probe_name) {
9680 if (perf_link->legacy_is_kprobe) {
9681 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
9682 perf_link->legacy_is_retprobe);
9684 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,
9685 perf_link->legacy_is_retprobe);
9692 static void bpf_link_perf_dealloc(struct bpf_link *link)
9694 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9696 free(perf_link->legacy_probe_name);
9700 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
9701 const struct bpf_perf_event_opts *opts)
9703 char errmsg[STRERR_BUFSIZE];
9704 struct bpf_link_perf *link;
9705 int prog_fd, link_fd = -1, err;
9707 if (!OPTS_VALID(opts, bpf_perf_event_opts))
9708 return libbpf_err_ptr(-EINVAL);
9711 pr_warn("prog '%s': invalid perf event FD %d\n",
9713 return libbpf_err_ptr(-EINVAL);
9715 prog_fd = bpf_program__fd(prog);
9717 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
9719 return libbpf_err_ptr(-EINVAL);
9722 link = calloc(1, sizeof(*link));
9724 return libbpf_err_ptr(-ENOMEM);
9725 link->link.detach = &bpf_link_perf_detach;
9726 link->link.dealloc = &bpf_link_perf_dealloc;
9727 link->perf_event_fd = pfd;
9729 if (kernel_supports(prog->obj, FEAT_PERF_LINK)) {
9730 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts,
9731 .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0));
9733 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);
9736 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",
9738 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9741 link->link.fd = link_fd;
9743 if (OPTS_GET(opts, bpf_cookie, 0)) {
9744 pr_warn("prog '%s': user context value is not supported\n", prog->name);
9749 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
9751 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
9752 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9754 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
9758 link->link.fd = pfd;
9760 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9762 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
9763 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9772 return libbpf_err_ptr(err);
9775 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)
9777 return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
9781 * this function is expected to parse integer in the range of [0, 2^31-1] from
9782 * given file using scanf format string fmt. If actual parsed value is
9783 * negative, the result might be indistinguishable from error
9785 static int parse_uint_from_file(const char *file, const char *fmt)
9787 char buf[STRERR_BUFSIZE];
9791 f = fopen(file, "r");
9794 pr_debug("failed to open '%s': %s\n", file,
9795 libbpf_strerror_r(err, buf, sizeof(buf)));
9798 err = fscanf(f, fmt, &ret);
9800 err = err == EOF ? -EIO : -errno;
9801 pr_debug("failed to parse '%s': %s\n", file,
9802 libbpf_strerror_r(err, buf, sizeof(buf)));
9810 static int determine_kprobe_perf_type(void)
9812 const char *file = "/sys/bus/event_source/devices/kprobe/type";
9814 return parse_uint_from_file(file, "%d\n");
9817 static int determine_uprobe_perf_type(void)
9819 const char *file = "/sys/bus/event_source/devices/uprobe/type";
9821 return parse_uint_from_file(file, "%d\n");
9824 static int determine_kprobe_retprobe_bit(void)
9826 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
9828 return parse_uint_from_file(file, "config:%d\n");
9831 static int determine_uprobe_retprobe_bit(void)
9833 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
9835 return parse_uint_from_file(file, "config:%d\n");
9838 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32
9839 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32
9841 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
9842 uint64_t offset, int pid, size_t ref_ctr_off)
9844 const size_t attr_sz = sizeof(struct perf_event_attr);
9845 struct perf_event_attr attr;
9846 char errmsg[STRERR_BUFSIZE];
9849 if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
9852 memset(&attr, 0, attr_sz);
9854 type = uprobe ? determine_uprobe_perf_type()
9855 : determine_kprobe_perf_type();
9857 pr_warn("failed to determine %s perf type: %s\n",
9858 uprobe ? "uprobe" : "kprobe",
9859 libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9863 int bit = uprobe ? determine_uprobe_retprobe_bit()
9864 : determine_kprobe_retprobe_bit();
9867 pr_warn("failed to determine %s retprobe bit: %s\n",
9868 uprobe ? "uprobe" : "kprobe",
9869 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
9872 attr.config |= 1 << bit;
9874 attr.size = attr_sz;
9876 attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
9877 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
9878 attr.config2 = offset; /* kprobe_addr or probe_offset */
9880 /* pid filter is meaningful only for uprobes */
9881 pfd = syscall(__NR_perf_event_open, &attr,
9882 pid < 0 ? -1 : pid /* pid */,
9883 pid == -1 ? 0 : -1 /* cpu */,
9884 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9885 return pfd >= 0 ? pfd : -errno;
9888 static int append_to_file(const char *file, const char *fmt, ...)
9893 fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0);
9898 n = vdprintf(fd, fmt, ap);
9908 #define DEBUGFS "/sys/kernel/debug/tracing"
9909 #define TRACEFS "/sys/kernel/tracing"
9911 static bool use_debugfs(void)
9913 static int has_debugfs = -1;
9915 if (has_debugfs < 0)
9916 has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0;
9918 return has_debugfs == 1;
9921 static const char *tracefs_path(void)
9923 return use_debugfs() ? DEBUGFS : TRACEFS;
9926 static const char *tracefs_kprobe_events(void)
9928 return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events";
9931 static const char *tracefs_uprobe_events(void)
9933 return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
9936 static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
9937 const char *kfunc_name, size_t offset)
9939 static int index = 0;
9941 snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset,
9942 __sync_fetch_and_add(&index, 1));
9945 static int add_kprobe_event_legacy(const char *probe_name, bool retprobe,
9946 const char *kfunc_name, size_t offset)
9948 return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx",
9949 retprobe ? 'r' : 'p',
9950 retprobe ? "kretprobes" : "kprobes",
9951 probe_name, kfunc_name, offset);
9954 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe)
9956 return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
9957 retprobe ? "kretprobes" : "kprobes", probe_name);
9960 static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe)
9964 snprintf(file, sizeof(file), "%s/events/%s/%s/id",
9965 tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name);
9967 return parse_uint_from_file(file, "%d\n");
9970 static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
9971 const char *kfunc_name, size_t offset, int pid)
9973 const size_t attr_sz = sizeof(struct perf_event_attr);
9974 struct perf_event_attr attr;
9975 char errmsg[STRERR_BUFSIZE];
9978 err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);
9980 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",
9982 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9985 type = determine_kprobe_perf_type_legacy(probe_name, retprobe);
9988 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",
9990 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9991 goto err_clean_legacy;
9994 memset(&attr, 0, attr_sz);
9995 attr.size = attr_sz;
9997 attr.type = PERF_TYPE_TRACEPOINT;
9999 pfd = syscall(__NR_perf_event_open, &attr,
10000 pid < 0 ? -1 : pid, /* pid */
10001 pid == -1 ? 0 : -1, /* cpu */
10002 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10005 pr_warn("legacy kprobe perf_event_open() failed: %s\n",
10006 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10007 goto err_clean_legacy;
10012 /* Clear the newly added legacy kprobe_event */
10013 remove_kprobe_event_legacy(probe_name, retprobe);
10017 static const char *arch_specific_syscall_pfx(void)
10019 #if defined(__x86_64__)
10021 #elif defined(__i386__)
10023 #elif defined(__s390x__)
10025 #elif defined(__s390__)
10027 #elif defined(__arm__)
10029 #elif defined(__aarch64__)
10031 #elif defined(__mips__)
10033 #elif defined(__riscv)
10035 #elif defined(__powerpc__)
10037 #elif defined(__powerpc64__)
10038 return "powerpc64";
10044 static int probe_kern_syscall_wrapper(void)
10046 char syscall_name[64];
10047 const char *ksys_pfx;
10049 ksys_pfx = arch_specific_syscall_pfx();
10053 snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx);
10055 if (determine_kprobe_perf_type() >= 0) {
10058 pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0);
10062 return pfd >= 0 ? 1 : 0;
10063 } else { /* legacy mode */
10064 char probe_name[128];
10066 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);
10067 if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0)
10070 (void)remove_kprobe_event_legacy(probe_name, false);
10076 bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
10077 const char *func_name,
10078 const struct bpf_kprobe_opts *opts)
10080 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
10081 char errmsg[STRERR_BUFSIZE];
10082 char *legacy_probe = NULL;
10083 struct bpf_link *link;
10085 bool retprobe, legacy;
10088 if (!OPTS_VALID(opts, bpf_kprobe_opts))
10089 return libbpf_err_ptr(-EINVAL);
10091 retprobe = OPTS_GET(opts, retprobe, false);
10092 offset = OPTS_GET(opts, offset, 0);
10093 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10095 legacy = determine_kprobe_perf_type() < 0;
10097 pfd = perf_event_open_probe(false /* uprobe */, retprobe,
10099 -1 /* pid */, 0 /* ref_ctr_off */);
10101 char probe_name[256];
10103 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name),
10104 func_name, offset);
10106 legacy_probe = strdup(probe_name);
10108 return libbpf_err_ptr(-ENOMEM);
10110 pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name,
10111 offset, -1 /* pid */);
10115 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
10116 prog->name, retprobe ? "kretprobe" : "kprobe",
10118 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10121 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10122 err = libbpf_get_error(link);
10125 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
10126 prog->name, retprobe ? "kretprobe" : "kprobe",
10128 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10129 goto err_clean_legacy;
10132 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10134 perf_link->legacy_probe_name = legacy_probe;
10135 perf_link->legacy_is_kprobe = true;
10136 perf_link->legacy_is_retprobe = retprobe;
10143 remove_kprobe_event_legacy(legacy_probe, retprobe);
10145 free(legacy_probe);
10146 return libbpf_err_ptr(err);
10149 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
10151 const char *func_name)
10153 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts,
10154 .retprobe = retprobe,
10157 return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
10160 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
10161 const char *syscall_name,
10162 const struct bpf_ksyscall_opts *opts)
10164 LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
10165 char func_name[128];
10167 if (!OPTS_VALID(opts, bpf_ksyscall_opts))
10168 return libbpf_err_ptr(-EINVAL);
10170 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
10171 /* arch_specific_syscall_pfx() should never return NULL here
10172 * because it is guarded by kernel_supports(). However, since
10173 * compiler does not know that we have an explicit conditional
10176 snprintf(func_name, sizeof(func_name), "__%s_sys_%s",
10177 arch_specific_syscall_pfx() ? : "", syscall_name);
10179 snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name);
10182 kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false);
10183 kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10185 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
10188 /* Adapted from perf/util/string.c */
10189 static bool glob_match(const char *str, const char *pat)
10191 while (*str && *pat && *pat != '*') {
10192 if (*pat == '?') { /* Matches any single character */
10202 /* Check wild card */
10204 while (*pat == '*')
10206 if (!*pat) /* Tail wild card matches all */
10209 if (glob_match(str++, pat))
10212 return !*str && !*pat;
10215 struct kprobe_multi_resolve {
10216 const char *pattern;
10217 unsigned long *addrs;
10223 resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
10224 const char *sym_name, void *ctx)
10226 struct kprobe_multi_resolve *res = ctx;
10229 if (!glob_match(sym_name, res->pattern))
10232 err = libbpf_ensure_mem((void **) &res->addrs, &res->cap, sizeof(unsigned long),
10237 res->addrs[res->cnt++] = (unsigned long) sym_addr;
10242 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
10243 const char *pattern,
10244 const struct bpf_kprobe_multi_opts *opts)
10246 LIBBPF_OPTS(bpf_link_create_opts, lopts);
10247 struct kprobe_multi_resolve res = {
10248 .pattern = pattern,
10250 struct bpf_link *link = NULL;
10251 char errmsg[STRERR_BUFSIZE];
10252 const unsigned long *addrs;
10253 int err, link_fd, prog_fd;
10254 const __u64 *cookies;
10259 if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))
10260 return libbpf_err_ptr(-EINVAL);
10262 syms = OPTS_GET(opts, syms, false);
10263 addrs = OPTS_GET(opts, addrs, false);
10264 cnt = OPTS_GET(opts, cnt, false);
10265 cookies = OPTS_GET(opts, cookies, false);
10267 if (!pattern && !addrs && !syms)
10268 return libbpf_err_ptr(-EINVAL);
10269 if (pattern && (addrs || syms || cookies || cnt))
10270 return libbpf_err_ptr(-EINVAL);
10271 if (!pattern && !cnt)
10272 return libbpf_err_ptr(-EINVAL);
10274 return libbpf_err_ptr(-EINVAL);
10277 err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
10288 retprobe = OPTS_GET(opts, retprobe, false);
10290 lopts.kprobe_multi.syms = syms;
10291 lopts.kprobe_multi.addrs = addrs;
10292 lopts.kprobe_multi.cookies = cookies;
10293 lopts.kprobe_multi.cnt = cnt;
10294 lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0;
10296 link = calloc(1, sizeof(*link));
10301 link->detach = &bpf_link__detach_fd;
10303 prog_fd = bpf_program__fd(prog);
10304 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts);
10307 pr_warn("prog '%s': failed to attach: %s\n",
10308 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10311 link->fd = link_fd;
10318 return libbpf_err_ptr(err);
10321 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10323 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);
10324 unsigned long offset = 0;
10325 const char *func_name;
10331 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */
10332 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
10335 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
10337 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
10339 func_name = prog->sec_name + sizeof("kprobe/") - 1;
10341 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
10343 pr_warn("kprobe name is invalid: %s\n", func_name);
10346 if (opts.retprobe && offset != 0) {
10348 pr_warn("kretprobes do not support offset specification\n");
10352 opts.offset = offset;
10353 *link = bpf_program__attach_kprobe_opts(prog, func, &opts);
10355 return libbpf_get_error(*link);
10358 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10360 LIBBPF_OPTS(bpf_ksyscall_opts, opts);
10361 const char *syscall_name;
10365 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
10366 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
10369 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
10371 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
10373 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
10375 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
10376 return *link ? 0 : -errno;
10379 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10381 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
10388 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */
10389 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
10390 strcmp(prog->sec_name, "kretprobe.multi") == 0)
10393 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
10395 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
10397 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
10399 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);
10401 pr_warn("kprobe multi pattern is invalid: %s\n", pattern);
10405 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
10407 return libbpf_get_error(*link);
10410 static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz,
10411 const char *binary_path, uint64_t offset)
10415 snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset);
10417 /* sanitize binary_path in the probe name */
10418 for (i = 0; buf[i]; i++) {
10419 if (!isalnum(buf[i]))
10424 static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,
10425 const char *binary_path, size_t offset)
10427 return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx",
10428 retprobe ? 'r' : 'p',
10429 retprobe ? "uretprobes" : "uprobes",
10430 probe_name, binary_path, offset);
10433 static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe)
10435 return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
10436 retprobe ? "uretprobes" : "uprobes", probe_name);
10439 static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe)
10443 snprintf(file, sizeof(file), "%s/events/%s/%s/id",
10444 tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name);
10446 return parse_uint_from_file(file, "%d\n");
10449 static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
10450 const char *binary_path, size_t offset, int pid)
10452 const size_t attr_sz = sizeof(struct perf_event_attr);
10453 struct perf_event_attr attr;
10454 int type, pfd, err;
10456 err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);
10458 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n",
10459 binary_path, (size_t)offset, err);
10462 type = determine_uprobe_perf_type_legacy(probe_name, retprobe);
10465 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n",
10466 binary_path, offset, err);
10467 goto err_clean_legacy;
10470 memset(&attr, 0, attr_sz);
10471 attr.size = attr_sz;
10472 attr.config = type;
10473 attr.type = PERF_TYPE_TRACEPOINT;
10475 pfd = syscall(__NR_perf_event_open, &attr,
10476 pid < 0 ? -1 : pid, /* pid */
10477 pid == -1 ? 0 : -1, /* cpu */
10478 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10481 pr_warn("legacy uprobe perf_event_open() failed: %d\n", err);
10482 goto err_clean_legacy;
10487 /* Clear the newly added legacy uprobe_event */
10488 remove_uprobe_event_legacy(probe_name, retprobe);
10492 /* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */
10493 static Elf_Scn *elf_find_next_scn_by_type(Elf *elf, int sh_type, Elf_Scn *scn)
10495 while ((scn = elf_nextscn(elf, scn)) != NULL) {
10498 if (!gelf_getshdr(scn, &sh))
10500 if (sh.sh_type == sh_type)
10506 /* Find offset of function name in object specified by path. "name" matches
10507 * symbol name or name@@LIB for library functions.
10509 static long elf_find_func_offset(const char *binary_path, const char *name)
10511 int fd, i, sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB };
10512 bool is_shared_lib, is_name_qualified;
10513 char errmsg[STRERR_BUFSIZE];
10514 long ret = -ENOENT;
10519 fd = open(binary_path, O_RDONLY | O_CLOEXEC);
10522 pr_warn("failed to open %s: %s\n", binary_path,
10523 libbpf_strerror_r(ret, errmsg, sizeof(errmsg)));
10526 elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
10528 pr_warn("elf: could not read elf from %s: %s\n", binary_path, elf_errmsg(-1));
10530 return -LIBBPF_ERRNO__FORMAT;
10532 if (!gelf_getehdr(elf, &ehdr)) {
10533 pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1));
10534 ret = -LIBBPF_ERRNO__FORMAT;
10537 /* for shared lib case, we do not need to calculate relative offset */
10538 is_shared_lib = ehdr.e_type == ET_DYN;
10540 name_len = strlen(name);
10541 /* Does name specify "@@LIB"? */
10542 is_name_qualified = strstr(name, "@@") != NULL;
10544 /* Search SHT_DYNSYM, SHT_SYMTAB for symbol. This search order is used because if
10545 * a binary is stripped, it may only have SHT_DYNSYM, and a fully-statically
10546 * linked binary may not have SHT_DYMSYM, so absence of a section should not be
10547 * reported as a warning/error.
10549 for (i = 0; i < ARRAY_SIZE(sh_types); i++) {
10550 size_t nr_syms, strtabidx, idx;
10551 Elf_Data *symbols = NULL;
10552 Elf_Scn *scn = NULL;
10553 int last_bind = -1;
10557 scn = elf_find_next_scn_by_type(elf, sh_types[i], NULL);
10559 pr_debug("elf: failed to find symbol table ELF sections in '%s'\n",
10563 if (!gelf_getshdr(scn, &sh))
10565 strtabidx = sh.sh_link;
10566 symbols = elf_getdata(scn, 0);
10568 pr_warn("elf: failed to get symbols for symtab section in '%s': %s\n",
10569 binary_path, elf_errmsg(-1));
10570 ret = -LIBBPF_ERRNO__FORMAT;
10573 nr_syms = symbols->d_size / sh.sh_entsize;
10575 for (idx = 0; idx < nr_syms; idx++) {
10581 if (!gelf_getsym(symbols, idx, &sym))
10584 if (GELF_ST_TYPE(sym.st_info) != STT_FUNC)
10587 sname = elf_strptr(elf, strtabidx, sym.st_name);
10591 curr_bind = GELF_ST_BIND(sym.st_info);
10593 /* User can specify func, func@@LIB or func@@LIB_VERSION. */
10594 if (strncmp(sname, name, name_len) != 0)
10596 /* ...but we don't want a search for "foo" to match 'foo2" also, so any
10597 * additional characters in sname should be of the form "@@LIB".
10599 if (!is_name_qualified && sname[name_len] != '\0' && sname[name_len] != '@')
10603 /* handle multiple matches */
10604 if (last_bind != STB_WEAK && curr_bind != STB_WEAK) {
10605 /* Only accept one non-weak bind. */
10606 pr_warn("elf: ambiguous match for '%s', '%s' in '%s'\n",
10607 sname, name, binary_path);
10608 ret = -LIBBPF_ERRNO__FORMAT;
10610 } else if (curr_bind == STB_WEAK) {
10611 /* already have a non-weak bind, and
10612 * this is a weak bind, so ignore.
10618 /* Transform symbol's virtual address (absolute for
10619 * binaries and relative for shared libs) into file
10620 * offset, which is what kernel is expecting for
10621 * uprobe/uretprobe attachment.
10622 * See Documentation/trace/uprobetracer.rst for more
10624 * This is done by looking up symbol's containing
10625 * section's header and using it's virtual address
10626 * (sh_addr) and corresponding file offset (sh_offset)
10627 * to transform sym.st_value (virtual address) into
10628 * desired final file offset.
10630 sym_scn = elf_getscn(elf, sym.st_shndx);
10633 if (!gelf_getshdr(sym_scn, &sym_sh))
10636 ret = sym.st_value - sym_sh.sh_addr + sym_sh.sh_offset;
10637 last_bind = curr_bind;
10644 pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path,
10648 pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path,
10649 is_shared_lib ? "should not be 0 in a shared library" :
10650 "try using shared library path instead");
10653 pr_warn("elf: failed to find symbol '%s' in '%s'\n", name, binary_path);
10662 static const char *arch_specific_lib_paths(void)
10665 * Based on https://packages.debian.org/sid/libc6.
10667 * Assume that the traced program is built for the same architecture
10668 * as libbpf, which should cover the vast majority of cases.
10670 #if defined(__x86_64__)
10671 return "/lib/x86_64-linux-gnu";
10672 #elif defined(__i386__)
10673 return "/lib/i386-linux-gnu";
10674 #elif defined(__s390x__)
10675 return "/lib/s390x-linux-gnu";
10676 #elif defined(__s390__)
10677 return "/lib/s390-linux-gnu";
10678 #elif defined(__arm__) && defined(__SOFTFP__)
10679 return "/lib/arm-linux-gnueabi";
10680 #elif defined(__arm__) && !defined(__SOFTFP__)
10681 return "/lib/arm-linux-gnueabihf";
10682 #elif defined(__aarch64__)
10683 return "/lib/aarch64-linux-gnu";
10684 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64
10685 return "/lib/mips64el-linux-gnuabi64";
10686 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32
10687 return "/lib/mipsel-linux-gnu";
10688 #elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
10689 return "/lib/powerpc64le-linux-gnu";
10690 #elif defined(__sparc__) && defined(__arch64__)
10691 return "/lib/sparc64-linux-gnu";
10692 #elif defined(__riscv) && __riscv_xlen == 64
10693 return "/lib/riscv64-linux-gnu";
10699 /* Get full path to program/shared library. */
10700 static int resolve_full_path(const char *file, char *result, size_t result_sz)
10702 const char *search_paths[3] = {};
10705 if (str_has_sfx(file, ".so") || strstr(file, ".so.")) {
10706 search_paths[0] = getenv("LD_LIBRARY_PATH");
10707 search_paths[1] = "/usr/lib64:/usr/lib";
10708 search_paths[2] = arch_specific_lib_paths();
10711 search_paths[0] = getenv("PATH");
10712 search_paths[1] = "/usr/bin:/usr/sbin";
10713 perm = R_OK | X_OK;
10716 for (i = 0; i < ARRAY_SIZE(search_paths); i++) {
10719 if (!search_paths[i])
10721 for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) {
10727 next_path = strchr(s, ':');
10728 seg_len = next_path ? next_path - s : strlen(s);
10731 snprintf(result, result_sz, "%.*s/%s", seg_len, s, file);
10732 /* ensure it has required permissions */
10733 if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0)
10735 pr_debug("resolved '%s' to '%s'\n", file, result);
10742 LIBBPF_API struct bpf_link *
10743 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
10744 const char *binary_path, size_t func_offset,
10745 const struct bpf_uprobe_opts *opts)
10747 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
10748 char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL;
10749 char full_binary_path[PATH_MAX];
10750 struct bpf_link *link;
10751 size_t ref_ctr_off;
10753 bool retprobe, legacy;
10754 const char *func_name;
10756 if (!OPTS_VALID(opts, bpf_uprobe_opts))
10757 return libbpf_err_ptr(-EINVAL);
10759 retprobe = OPTS_GET(opts, retprobe, false);
10760 ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0);
10761 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
10764 return libbpf_err_ptr(-EINVAL);
10766 if (!strchr(binary_path, '/')) {
10767 err = resolve_full_path(binary_path, full_binary_path,
10768 sizeof(full_binary_path));
10770 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
10771 prog->name, binary_path, err);
10772 return libbpf_err_ptr(err);
10774 binary_path = full_binary_path;
10776 func_name = OPTS_GET(opts, func_name, NULL);
10780 sym_off = elf_find_func_offset(binary_path, func_name);
10782 return libbpf_err_ptr(sym_off);
10783 func_offset += sym_off;
10786 legacy = determine_uprobe_perf_type() < 0;
10788 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path,
10789 func_offset, pid, ref_ctr_off);
10791 char probe_name[PATH_MAX + 64];
10794 return libbpf_err_ptr(-EINVAL);
10796 gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name),
10797 binary_path, func_offset);
10799 legacy_probe = strdup(probe_name);
10801 return libbpf_err_ptr(-ENOMEM);
10803 pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe,
10804 binary_path, func_offset, pid);
10808 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
10809 prog->name, retprobe ? "uretprobe" : "uprobe",
10810 binary_path, func_offset,
10811 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10815 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10816 err = libbpf_get_error(link);
10819 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
10820 prog->name, retprobe ? "uretprobe" : "uprobe",
10821 binary_path, func_offset,
10822 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10823 goto err_clean_legacy;
10826 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
10828 perf_link->legacy_probe_name = legacy_probe;
10829 perf_link->legacy_is_kprobe = false;
10830 perf_link->legacy_is_retprobe = retprobe;
10836 remove_uprobe_event_legacy(legacy_probe, retprobe);
10838 free(legacy_probe);
10839 return libbpf_err_ptr(err);
10842 /* Format of u[ret]probe section definition supporting auto-attach:
10843 * u[ret]probe/binary:function[+offset]
10845 * binary can be an absolute/relative path or a filename; the latter is resolved to a
10846 * full binary path via bpf_program__attach_uprobe_opts.
10848 * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be
10849 * specified (and auto-attach is not possible) or the above format is specified for
10852 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10854 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts);
10855 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL;
10856 int n, ret = -EINVAL;
10861 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[a-zA-Z0-9_.]+%li",
10862 &probe_type, &binary_path, &func_name, &offset);
10865 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */
10869 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",
10870 prog->name, prog->sec_name);
10874 opts.retprobe = strcmp(probe_type, "uretprobe") == 0 ||
10875 strcmp(probe_type, "uretprobe.s") == 0;
10876 if (opts.retprobe && offset != 0) {
10877 pr_warn("prog '%s': uretprobes do not support offset specification\n",
10881 opts.func_name = func_name;
10882 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
10883 ret = libbpf_get_error(*link);
10886 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
10897 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,
10898 bool retprobe, pid_t pid,
10899 const char *binary_path,
10900 size_t func_offset)
10902 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe);
10904 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
10907 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
10908 pid_t pid, const char *binary_path,
10909 const char *usdt_provider, const char *usdt_name,
10910 const struct bpf_usdt_opts *opts)
10912 char resolved_path[512];
10913 struct bpf_object *obj = prog->obj;
10914 struct bpf_link *link;
10918 if (!OPTS_VALID(opts, bpf_uprobe_opts))
10919 return libbpf_err_ptr(-EINVAL);
10921 if (bpf_program__fd(prog) < 0) {
10922 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
10924 return libbpf_err_ptr(-EINVAL);
10928 return libbpf_err_ptr(-EINVAL);
10930 if (!strchr(binary_path, '/')) {
10931 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));
10933 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
10934 prog->name, binary_path, err);
10935 return libbpf_err_ptr(err);
10937 binary_path = resolved_path;
10940 /* USDT manager is instantiated lazily on first USDT attach. It will
10941 * be destroyed together with BPF object in bpf_object__close().
10943 if (IS_ERR(obj->usdt_man))
10944 return libbpf_ptr(obj->usdt_man);
10945 if (!obj->usdt_man) {
10946 obj->usdt_man = usdt_manager_new(obj);
10947 if (IS_ERR(obj->usdt_man))
10948 return libbpf_ptr(obj->usdt_man);
10951 usdt_cookie = OPTS_GET(opts, usdt_cookie, 0);
10952 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
10953 usdt_provider, usdt_name, usdt_cookie);
10954 err = libbpf_get_error(link);
10956 return libbpf_err_ptr(err);
10960 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10962 char *path = NULL, *provider = NULL, *name = NULL;
10963 const char *sec_name;
10966 sec_name = bpf_program__section_name(prog);
10967 if (strcmp(sec_name, "usdt") == 0) {
10968 /* no auto-attach for just SEC("usdt") */
10973 n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name);
10975 pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n",
10979 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
10980 provider, name, NULL);
10981 err = libbpf_get_error(*link);
10989 static int determine_tracepoint_id(const char *tp_category,
10990 const char *tp_name)
10992 char file[PATH_MAX];
10995 ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id",
10996 tracefs_path(), tp_category, tp_name);
10999 if (ret >= sizeof(file)) {
11000 pr_debug("tracepoint %s/%s path is too long\n",
11001 tp_category, tp_name);
11004 return parse_uint_from_file(file, "%d\n");
11007 static int perf_event_open_tracepoint(const char *tp_category,
11008 const char *tp_name)
11010 const size_t attr_sz = sizeof(struct perf_event_attr);
11011 struct perf_event_attr attr;
11012 char errmsg[STRERR_BUFSIZE];
11013 int tp_id, pfd, err;
11015 tp_id = determine_tracepoint_id(tp_category, tp_name);
11017 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
11018 tp_category, tp_name,
11019 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
11023 memset(&attr, 0, attr_sz);
11024 attr.type = PERF_TYPE_TRACEPOINT;
11025 attr.size = attr_sz;
11026 attr.config = tp_id;
11028 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
11029 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11032 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
11033 tp_category, tp_name,
11034 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11040 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
11041 const char *tp_category,
11042 const char *tp_name,
11043 const struct bpf_tracepoint_opts *opts)
11045 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
11046 char errmsg[STRERR_BUFSIZE];
11047 struct bpf_link *link;
11050 if (!OPTS_VALID(opts, bpf_tracepoint_opts))
11051 return libbpf_err_ptr(-EINVAL);
11053 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
11055 pfd = perf_event_open_tracepoint(tp_category, tp_name);
11057 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
11058 prog->name, tp_category, tp_name,
11059 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11060 return libbpf_err_ptr(pfd);
11062 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11063 err = libbpf_get_error(link);
11066 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
11067 prog->name, tp_category, tp_name,
11068 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11069 return libbpf_err_ptr(err);
11074 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,
11075 const char *tp_category,
11076 const char *tp_name)
11078 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
11081 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11083 char *sec_name, *tp_cat, *tp_name;
11087 /* no auto-attach for SEC("tp") or SEC("tracepoint") */
11088 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
11091 sec_name = strdup(prog->sec_name);
11095 /* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */
11096 if (str_has_pfx(prog->sec_name, "tp/"))
11097 tp_cat = sec_name + sizeof("tp/") - 1;
11099 tp_cat = sec_name + sizeof("tracepoint/") - 1;
11100 tp_name = strchr(tp_cat, '/');
11108 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
11110 return libbpf_get_error(*link);
11113 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
11114 const char *tp_name)
11116 char errmsg[STRERR_BUFSIZE];
11117 struct bpf_link *link;
11120 prog_fd = bpf_program__fd(prog);
11122 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11123 return libbpf_err_ptr(-EINVAL);
11126 link = calloc(1, sizeof(*link));
11128 return libbpf_err_ptr(-ENOMEM);
11129 link->detach = &bpf_link__detach_fd;
11131 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
11135 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
11136 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11137 return libbpf_err_ptr(pfd);
11143 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11145 static const char *const prefixes[] = {
11149 "raw_tracepoint.w",
11152 const char *tp_name = NULL;
11156 for (i = 0; i < ARRAY_SIZE(prefixes); i++) {
11159 if (!str_has_pfx(prog->sec_name, prefixes[i]))
11162 pfx_len = strlen(prefixes[i]);
11163 /* no auto-attach case of, e.g., SEC("raw_tp") */
11164 if (prog->sec_name[pfx_len] == '\0')
11167 if (prog->sec_name[pfx_len] != '/')
11170 tp_name = prog->sec_name + pfx_len + 1;
11175 pr_warn("prog '%s': invalid section name '%s'\n",
11176 prog->name, prog->sec_name);
11180 *link = bpf_program__attach_raw_tracepoint(prog, tp_name);
11181 return libbpf_get_error(link);
11184 /* Common logic for all BPF program types that attach to a btf_id */
11185 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog,
11186 const struct bpf_trace_opts *opts)
11188 LIBBPF_OPTS(bpf_link_create_opts, link_opts);
11189 char errmsg[STRERR_BUFSIZE];
11190 struct bpf_link *link;
11193 if (!OPTS_VALID(opts, bpf_trace_opts))
11194 return libbpf_err_ptr(-EINVAL);
11196 prog_fd = bpf_program__fd(prog);
11198 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11199 return libbpf_err_ptr(-EINVAL);
11202 link = calloc(1, sizeof(*link));
11204 return libbpf_err_ptr(-ENOMEM);
11205 link->detach = &bpf_link__detach_fd;
11207 /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */
11208 link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0);
11209 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);
11213 pr_warn("prog '%s': failed to attach: %s\n",
11214 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11215 return libbpf_err_ptr(pfd);
11221 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
11223 return bpf_program__attach_btf_id(prog, NULL);
11226 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
11227 const struct bpf_trace_opts *opts)
11229 return bpf_program__attach_btf_id(prog, opts);
11232 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
11234 return bpf_program__attach_btf_id(prog, NULL);
11237 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11239 *link = bpf_program__attach_trace(prog);
11240 return libbpf_get_error(*link);
11243 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11245 *link = bpf_program__attach_lsm(prog);
11246 return libbpf_get_error(*link);
11249 static struct bpf_link *
11250 bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id,
11251 const char *target_name)
11253 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
11254 .target_btf_id = btf_id);
11255 enum bpf_attach_type attach_type;
11256 char errmsg[STRERR_BUFSIZE];
11257 struct bpf_link *link;
11258 int prog_fd, link_fd;
11260 prog_fd = bpf_program__fd(prog);
11262 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11263 return libbpf_err_ptr(-EINVAL);
11266 link = calloc(1, sizeof(*link));
11268 return libbpf_err_ptr(-ENOMEM);
11269 link->detach = &bpf_link__detach_fd;
11271 attach_type = bpf_program__expected_attach_type(prog);
11272 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
11276 pr_warn("prog '%s': failed to attach to %s: %s\n",
11277 prog->name, target_name,
11278 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11279 return libbpf_err_ptr(link_fd);
11281 link->fd = link_fd;
11286 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
11288 return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
11292 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
11294 return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
11297 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
11299 /* target_fd/target_ifindex use the same field in LINK_CREATE */
11300 return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
11303 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
11305 const char *attach_func_name)
11309 if (!!target_fd != !!attach_func_name) {
11310 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
11312 return libbpf_err_ptr(-EINVAL);
11315 if (prog->type != BPF_PROG_TYPE_EXT) {
11316 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
11318 return libbpf_err_ptr(-EINVAL);
11322 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
11324 return libbpf_err_ptr(btf_id);
11326 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
11328 /* no target, so use raw_tracepoint_open for compatibility
11331 return bpf_program__attach_trace(prog);
11336 bpf_program__attach_iter(const struct bpf_program *prog,
11337 const struct bpf_iter_attach_opts *opts)
11339 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
11340 char errmsg[STRERR_BUFSIZE];
11341 struct bpf_link *link;
11342 int prog_fd, link_fd;
11343 __u32 target_fd = 0;
11345 if (!OPTS_VALID(opts, bpf_iter_attach_opts))
11346 return libbpf_err_ptr(-EINVAL);
11348 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
11349 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
11351 prog_fd = bpf_program__fd(prog);
11353 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11354 return libbpf_err_ptr(-EINVAL);
11357 link = calloc(1, sizeof(*link));
11359 return libbpf_err_ptr(-ENOMEM);
11360 link->detach = &bpf_link__detach_fd;
11362 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
11363 &link_create_opts);
11367 pr_warn("prog '%s': failed to attach to iterator: %s\n",
11368 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
11369 return libbpf_err_ptr(link_fd);
11371 link->fd = link_fd;
11375 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11377 *link = bpf_program__attach_iter(prog, NULL);
11378 return libbpf_get_error(*link);
11381 struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
11383 struct bpf_link *link = NULL;
11386 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
11387 return libbpf_err_ptr(-EOPNOTSUPP);
11389 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
11391 return libbpf_err_ptr(err);
11393 /* When calling bpf_program__attach() explicitly, auto-attach support
11394 * is expected to work, so NULL returned link is considered an error.
11395 * This is different for skeleton's attach, see comment in
11396 * bpf_object__attach_skeleton().
11399 return libbpf_err_ptr(-EOPNOTSUPP);
11404 static int bpf_link__detach_struct_ops(struct bpf_link *link)
11408 if (bpf_map_delete_elem(link->fd, &zero))
11414 struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)
11416 struct bpf_struct_ops *st_ops;
11417 struct bpf_link *link;
11421 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
11422 return libbpf_err_ptr(-EINVAL);
11424 link = calloc(1, sizeof(*link));
11426 return libbpf_err_ptr(-EINVAL);
11428 st_ops = map->st_ops;
11429 for (i = 0; i < btf_vlen(st_ops->type); i++) {
11430 struct bpf_program *prog = st_ops->progs[i];
11437 prog_fd = bpf_program__fd(prog);
11438 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
11439 *(unsigned long *)kern_data = prog_fd;
11442 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
11446 return libbpf_err_ptr(err);
11449 link->detach = bpf_link__detach_struct_ops;
11450 link->fd = map->fd;
11455 typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr,
11456 void *private_data);
11458 static enum bpf_perf_event_ret
11459 perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
11460 void **copy_mem, size_t *copy_size,
11461 bpf_perf_event_print_t fn, void *private_data)
11463 struct perf_event_mmap_page *header = mmap_mem;
11464 __u64 data_head = ring_buffer_read_head(header);
11465 __u64 data_tail = header->data_tail;
11466 void *base = ((__u8 *)header) + page_size;
11467 int ret = LIBBPF_PERF_EVENT_CONT;
11468 struct perf_event_header *ehdr;
11471 while (data_head != data_tail) {
11472 ehdr = base + (data_tail & (mmap_size - 1));
11473 ehdr_size = ehdr->size;
11475 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
11476 void *copy_start = ehdr;
11477 size_t len_first = base + mmap_size - copy_start;
11478 size_t len_secnd = ehdr_size - len_first;
11480 if (*copy_size < ehdr_size) {
11482 *copy_mem = malloc(ehdr_size);
11485 ret = LIBBPF_PERF_EVENT_ERROR;
11488 *copy_size = ehdr_size;
11491 memcpy(*copy_mem, copy_start, len_first);
11492 memcpy(*copy_mem + len_first, base, len_secnd);
11496 ret = fn(ehdr, private_data);
11497 data_tail += ehdr_size;
11498 if (ret != LIBBPF_PERF_EVENT_CONT)
11502 ring_buffer_write_tail(header, data_tail);
11503 return libbpf_err(ret);
11506 struct perf_buffer;
11508 struct perf_buffer_params {
11509 struct perf_event_attr *attr;
11510 /* if event_cb is specified, it takes precendence */
11511 perf_buffer_event_fn event_cb;
11512 /* sample_cb and lost_cb are higher-level common-case callbacks */
11513 perf_buffer_sample_fn sample_cb;
11514 perf_buffer_lost_fn lost_cb;
11521 struct perf_cpu_buf {
11522 struct perf_buffer *pb;
11523 void *base; /* mmap()'ed memory */
11524 void *buf; /* for reconstructing segmented data */
11531 struct perf_buffer {
11532 perf_buffer_event_fn event_cb;
11533 perf_buffer_sample_fn sample_cb;
11534 perf_buffer_lost_fn lost_cb;
11535 void *ctx; /* passed into callbacks */
11539 struct perf_cpu_buf **cpu_bufs;
11540 struct epoll_event *events;
11541 int cpu_cnt; /* number of allocated CPU buffers */
11542 int epoll_fd; /* perf event FD */
11543 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
11546 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
11547 struct perf_cpu_buf *cpu_buf)
11551 if (cpu_buf->base &&
11552 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
11553 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
11554 if (cpu_buf->fd >= 0) {
11555 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
11556 close(cpu_buf->fd);
11558 free(cpu_buf->buf);
11562 void perf_buffer__free(struct perf_buffer *pb)
11566 if (IS_ERR_OR_NULL(pb))
11568 if (pb->cpu_bufs) {
11569 for (i = 0; i < pb->cpu_cnt; i++) {
11570 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
11575 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
11576 perf_buffer__free_cpu_buf(pb, cpu_buf);
11578 free(pb->cpu_bufs);
11580 if (pb->epoll_fd >= 0)
11581 close(pb->epoll_fd);
11586 static struct perf_cpu_buf *
11587 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
11588 int cpu, int map_key)
11590 struct perf_cpu_buf *cpu_buf;
11591 char msg[STRERR_BUFSIZE];
11594 cpu_buf = calloc(1, sizeof(*cpu_buf));
11596 return ERR_PTR(-ENOMEM);
11599 cpu_buf->cpu = cpu;
11600 cpu_buf->map_key = map_key;
11602 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
11603 -1, PERF_FLAG_FD_CLOEXEC);
11604 if (cpu_buf->fd < 0) {
11606 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
11607 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11611 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
11612 PROT_READ | PROT_WRITE, MAP_SHARED,
11614 if (cpu_buf->base == MAP_FAILED) {
11615 cpu_buf->base = NULL;
11617 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
11618 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11622 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
11624 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
11625 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
11632 perf_buffer__free_cpu_buf(pb, cpu_buf);
11633 return (struct perf_cpu_buf *)ERR_PTR(err);
11636 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
11637 struct perf_buffer_params *p);
11639 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
11640 perf_buffer_sample_fn sample_cb,
11641 perf_buffer_lost_fn lost_cb,
11643 const struct perf_buffer_opts *opts)
11645 const size_t attr_sz = sizeof(struct perf_event_attr);
11646 struct perf_buffer_params p = {};
11647 struct perf_event_attr attr;
11649 if (!OPTS_VALID(opts, perf_buffer_opts))
11650 return libbpf_err_ptr(-EINVAL);
11652 memset(&attr, 0, attr_sz);
11653 attr.size = attr_sz;
11654 attr.config = PERF_COUNT_SW_BPF_OUTPUT;
11655 attr.type = PERF_TYPE_SOFTWARE;
11656 attr.sample_type = PERF_SAMPLE_RAW;
11657 attr.sample_period = 1;
11658 attr.wakeup_events = 1;
11661 p.sample_cb = sample_cb;
11662 p.lost_cb = lost_cb;
11665 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
11668 struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt,
11669 struct perf_event_attr *attr,
11670 perf_buffer_event_fn event_cb, void *ctx,
11671 const struct perf_buffer_raw_opts *opts)
11673 struct perf_buffer_params p = {};
11676 return libbpf_err_ptr(-EINVAL);
11678 if (!OPTS_VALID(opts, perf_buffer_raw_opts))
11679 return libbpf_err_ptr(-EINVAL);
11682 p.event_cb = event_cb;
11684 p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0);
11685 p.cpus = OPTS_GET(opts, cpus, NULL);
11686 p.map_keys = OPTS_GET(opts, map_keys, NULL);
11688 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
11691 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
11692 struct perf_buffer_params *p)
11694 const char *online_cpus_file = "/sys/devices/system/cpu/online";
11695 struct bpf_map_info map;
11696 char msg[STRERR_BUFSIZE];
11697 struct perf_buffer *pb;
11698 bool *online = NULL;
11699 __u32 map_info_len;
11702 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {
11703 pr_warn("page count should be power of two, but is %zu\n",
11705 return ERR_PTR(-EINVAL);
11708 /* best-effort sanity checks */
11709 memset(&map, 0, sizeof(map));
11710 map_info_len = sizeof(map);
11711 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
11714 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
11715 * -EBADFD, -EFAULT, or -E2BIG on real error
11717 if (err != -EINVAL) {
11718 pr_warn("failed to get map info for map FD %d: %s\n",
11719 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
11720 return ERR_PTR(err);
11722 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
11725 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
11726 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
11728 return ERR_PTR(-EINVAL);
11732 pb = calloc(1, sizeof(*pb));
11734 return ERR_PTR(-ENOMEM);
11736 pb->event_cb = p->event_cb;
11737 pb->sample_cb = p->sample_cb;
11738 pb->lost_cb = p->lost_cb;
11741 pb->page_size = getpagesize();
11742 pb->mmap_size = pb->page_size * page_cnt;
11743 pb->map_fd = map_fd;
11745 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
11746 if (pb->epoll_fd < 0) {
11748 pr_warn("failed to create epoll instance: %s\n",
11749 libbpf_strerror_r(err, msg, sizeof(msg)));
11753 if (p->cpu_cnt > 0) {
11754 pb->cpu_cnt = p->cpu_cnt;
11756 pb->cpu_cnt = libbpf_num_possible_cpus();
11757 if (pb->cpu_cnt < 0) {
11761 if (map.max_entries && map.max_entries < pb->cpu_cnt)
11762 pb->cpu_cnt = map.max_entries;
11765 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
11768 pr_warn("failed to allocate events: out of memory\n");
11771 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
11772 if (!pb->cpu_bufs) {
11774 pr_warn("failed to allocate buffers: out of memory\n");
11778 err = parse_cpu_mask_file(online_cpus_file, &online, &n);
11780 pr_warn("failed to get online CPU mask: %d\n", err);
11784 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
11785 struct perf_cpu_buf *cpu_buf;
11788 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
11789 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
11791 /* in case user didn't explicitly requested particular CPUs to
11792 * be attached to, skip offline/not present CPUs
11794 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
11797 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
11798 if (IS_ERR(cpu_buf)) {
11799 err = PTR_ERR(cpu_buf);
11803 pb->cpu_bufs[j] = cpu_buf;
11805 err = bpf_map_update_elem(pb->map_fd, &map_key,
11809 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
11810 cpu, map_key, cpu_buf->fd,
11811 libbpf_strerror_r(err, msg, sizeof(msg)));
11815 pb->events[j].events = EPOLLIN;
11816 pb->events[j].data.ptr = cpu_buf;
11817 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
11818 &pb->events[j]) < 0) {
11820 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
11822 libbpf_strerror_r(err, msg, sizeof(msg)));
11835 perf_buffer__free(pb);
11836 return ERR_PTR(err);
11839 struct perf_sample_raw {
11840 struct perf_event_header header;
11845 struct perf_sample_lost {
11846 struct perf_event_header header;
11849 uint64_t sample_id;
11852 static enum bpf_perf_event_ret
11853 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
11855 struct perf_cpu_buf *cpu_buf = ctx;
11856 struct perf_buffer *pb = cpu_buf->pb;
11859 /* user wants full control over parsing perf event */
11861 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
11864 case PERF_RECORD_SAMPLE: {
11865 struct perf_sample_raw *s = data;
11868 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
11871 case PERF_RECORD_LOST: {
11872 struct perf_sample_lost *s = data;
11875 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
11879 pr_warn("unknown perf sample type %d\n", e->type);
11880 return LIBBPF_PERF_EVENT_ERROR;
11882 return LIBBPF_PERF_EVENT_CONT;
11885 static int perf_buffer__process_records(struct perf_buffer *pb,
11886 struct perf_cpu_buf *cpu_buf)
11888 enum bpf_perf_event_ret ret;
11890 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,
11891 pb->page_size, &cpu_buf->buf,
11892 &cpu_buf->buf_size,
11893 perf_buffer__process_record, cpu_buf);
11894 if (ret != LIBBPF_PERF_EVENT_CONT)
11899 int perf_buffer__epoll_fd(const struct perf_buffer *pb)
11901 return pb->epoll_fd;
11904 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
11908 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
11912 for (i = 0; i < cnt; i++) {
11913 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
11915 err = perf_buffer__process_records(pb, cpu_buf);
11917 pr_warn("error while processing records: %d\n", err);
11918 return libbpf_err(err);
11924 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
11927 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
11929 return pb->cpu_cnt;
11933 * Return perf_event FD of a ring buffer in *buf_idx* slot of
11934 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
11935 * select()/poll()/epoll() Linux syscalls.
11937 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
11939 struct perf_cpu_buf *cpu_buf;
11941 if (buf_idx >= pb->cpu_cnt)
11942 return libbpf_err(-EINVAL);
11944 cpu_buf = pb->cpu_bufs[buf_idx];
11946 return libbpf_err(-ENOENT);
11948 return cpu_buf->fd;
11951 int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size)
11953 struct perf_cpu_buf *cpu_buf;
11955 if (buf_idx >= pb->cpu_cnt)
11956 return libbpf_err(-EINVAL);
11958 cpu_buf = pb->cpu_bufs[buf_idx];
11960 return libbpf_err(-ENOENT);
11962 *buf = cpu_buf->base;
11963 *buf_size = pb->mmap_size;
11968 * Consume data from perf ring buffer corresponding to slot *buf_idx* in
11969 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
11970 * consume, do nothing and return success.
11975 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
11977 struct perf_cpu_buf *cpu_buf;
11979 if (buf_idx >= pb->cpu_cnt)
11980 return libbpf_err(-EINVAL);
11982 cpu_buf = pb->cpu_bufs[buf_idx];
11984 return libbpf_err(-ENOENT);
11986 return perf_buffer__process_records(pb, cpu_buf);
11989 int perf_buffer__consume(struct perf_buffer *pb)
11993 for (i = 0; i < pb->cpu_cnt; i++) {
11994 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
11999 err = perf_buffer__process_records(pb, cpu_buf);
12001 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
12002 return libbpf_err(err);
12008 int bpf_program__set_attach_target(struct bpf_program *prog,
12009 int attach_prog_fd,
12010 const char *attach_func_name)
12012 int btf_obj_fd = 0, btf_id = 0, err;
12014 if (!prog || attach_prog_fd < 0)
12015 return libbpf_err(-EINVAL);
12017 if (prog->obj->loaded)
12018 return libbpf_err(-EINVAL);
12020 if (attach_prog_fd && !attach_func_name) {
12021 /* remember attach_prog_fd and let bpf_program__load() find
12022 * BTF ID during the program load
12024 prog->attach_prog_fd = attach_prog_fd;
12028 if (attach_prog_fd) {
12029 btf_id = libbpf_find_prog_btf_id(attach_func_name,
12032 return libbpf_err(btf_id);
12034 if (!attach_func_name)
12035 return libbpf_err(-EINVAL);
12037 /* load btf_vmlinux, if not yet */
12038 err = bpf_object__load_vmlinux_btf(prog->obj, true);
12040 return libbpf_err(err);
12041 err = find_kernel_btf_id(prog->obj, attach_func_name,
12042 prog->expected_attach_type,
12043 &btf_obj_fd, &btf_id);
12045 return libbpf_err(err);
12048 prog->attach_btf_id = btf_id;
12049 prog->attach_btf_obj_fd = btf_obj_fd;
12050 prog->attach_prog_fd = attach_prog_fd;
12054 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
12056 int err = 0, n, len, start, end = -1;
12062 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
12064 if (*s == ',' || *s == '\n') {
12068 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
12069 if (n <= 0 || n > 2) {
12070 pr_warn("Failed to get CPU range %s: %d\n", s, n);
12073 } else if (n == 1) {
12076 if (start < 0 || start > end) {
12077 pr_warn("Invalid CPU range [%d,%d] in %s\n",
12082 tmp = realloc(*mask, end + 1);
12088 memset(tmp + *mask_sz, 0, start - *mask_sz);
12089 memset(tmp + start, 1, end - start + 1);
12090 *mask_sz = end + 1;
12094 pr_warn("Empty CPU range\n");
12104 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
12106 int fd, err = 0, len;
12109 fd = open(fcpu, O_RDONLY | O_CLOEXEC);
12112 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
12115 len = read(fd, buf, sizeof(buf));
12118 err = len ? -errno : -EINVAL;
12119 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
12122 if (len >= sizeof(buf)) {
12123 pr_warn("CPU mask is too big in file %s\n", fcpu);
12128 return parse_cpu_mask_str(buf, mask, mask_sz);
12131 int libbpf_num_possible_cpus(void)
12133 static const char *fcpu = "/sys/devices/system/cpu/possible";
12135 int err, n, i, tmp_cpus;
12138 tmp_cpus = READ_ONCE(cpus);
12142 err = parse_cpu_mask_file(fcpu, &mask, &n);
12144 return libbpf_err(err);
12147 for (i = 0; i < n; i++) {
12153 WRITE_ONCE(cpus, tmp_cpus);
12157 static int populate_skeleton_maps(const struct bpf_object *obj,
12158 struct bpf_map_skeleton *maps,
12163 for (i = 0; i < map_cnt; i++) {
12164 struct bpf_map **map = maps[i].map;
12165 const char *name = maps[i].name;
12166 void **mmaped = maps[i].mmaped;
12168 *map = bpf_object__find_map_by_name(obj, name);
12170 pr_warn("failed to find skeleton map '%s'\n", name);
12174 /* externs shouldn't be pre-setup from user code */
12175 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
12176 *mmaped = (*map)->mmaped;
12181 static int populate_skeleton_progs(const struct bpf_object *obj,
12182 struct bpf_prog_skeleton *progs,
12187 for (i = 0; i < prog_cnt; i++) {
12188 struct bpf_program **prog = progs[i].prog;
12189 const char *name = progs[i].name;
12191 *prog = bpf_object__find_program_by_name(obj, name);
12193 pr_warn("failed to find skeleton program '%s'\n", name);
12200 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
12201 const struct bpf_object_open_opts *opts)
12203 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
12204 .object_name = s->name,
12206 struct bpf_object *obj;
12209 /* Attempt to preserve opts->object_name, unless overriden by user
12210 * explicitly. Overwriting object name for skeletons is discouraged,
12211 * as it breaks global data maps, because they contain object name
12212 * prefix as their own map name prefix. When skeleton is generated,
12213 * bpftool is making an assumption that this name will stay the same.
12216 memcpy(&skel_opts, opts, sizeof(*opts));
12217 if (!opts->object_name)
12218 skel_opts.object_name = s->name;
12221 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
12222 err = libbpf_get_error(obj);
12224 pr_warn("failed to initialize skeleton BPF object '%s': %d\n",
12226 return libbpf_err(err);
12230 err = populate_skeleton_maps(obj, s->maps, s->map_cnt);
12232 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);
12233 return libbpf_err(err);
12236 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt);
12238 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);
12239 return libbpf_err(err);
12245 int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s)
12247 int err, len, var_idx, i;
12248 const char *var_name;
12249 const struct bpf_map *map;
12252 const struct btf_type *map_type, *var_type;
12253 const struct bpf_var_skeleton *var_skel;
12254 struct btf_var_secinfo *var;
12257 return libbpf_err(-EINVAL);
12259 btf = bpf_object__btf(s->obj);
12261 pr_warn("subskeletons require BTF at runtime (object %s)\n",
12262 bpf_object__name(s->obj));
12263 return libbpf_err(-errno);
12266 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt);
12268 pr_warn("failed to populate subskeleton maps: %d\n", err);
12269 return libbpf_err(err);
12272 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt);
12274 pr_warn("failed to populate subskeleton maps: %d\n", err);
12275 return libbpf_err(err);
12278 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {
12279 var_skel = &s->vars[var_idx];
12280 map = *var_skel->map;
12281 map_type_id = bpf_map__btf_value_type_id(map);
12282 map_type = btf__type_by_id(btf, map_type_id);
12284 if (!btf_is_datasec(map_type)) {
12285 pr_warn("type for map '%1$s' is not a datasec: %2$s",
12286 bpf_map__name(map),
12287 __btf_kind_str(btf_kind(map_type)));
12288 return libbpf_err(-EINVAL);
12291 len = btf_vlen(map_type);
12292 var = btf_var_secinfos(map_type);
12293 for (i = 0; i < len; i++, var++) {
12294 var_type = btf__type_by_id(btf, var->type);
12295 var_name = btf__name_by_offset(btf, var_type->name_off);
12296 if (strcmp(var_name, var_skel->name) == 0) {
12297 *var_skel->addr = map->mmaped + var->offset;
12305 void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s)
12315 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
12319 err = bpf_object__load(*s->obj);
12321 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
12322 return libbpf_err(err);
12325 for (i = 0; i < s->map_cnt; i++) {
12326 struct bpf_map *map = *s->maps[i].map;
12327 size_t mmap_sz = bpf_map_mmap_sz(map);
12328 int prot, map_fd = bpf_map__fd(map);
12329 void **mmaped = s->maps[i].mmaped;
12334 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
12339 if (map->def.map_flags & BPF_F_RDONLY_PROG)
12342 prot = PROT_READ | PROT_WRITE;
12344 /* Remap anonymous mmap()-ed "map initialization image" as
12345 * a BPF map-backed mmap()-ed memory, but preserving the same
12346 * memory address. This will cause kernel to change process'
12347 * page table to point to a different piece of kernel memory,
12348 * but from userspace point of view memory address (and its
12349 * contents, being identical at this point) will stay the
12350 * same. This mapping will be released by bpf_object__close()
12351 * as per normal clean up procedure, so we don't need to worry
12352 * about it from skeleton's clean up perspective.
12354 *mmaped = mmap(map->mmaped, mmap_sz, prot,
12355 MAP_SHARED | MAP_FIXED, map_fd, 0);
12356 if (*mmaped == MAP_FAILED) {
12359 pr_warn("failed to re-mmap() map '%s': %d\n",
12360 bpf_map__name(map), err);
12361 return libbpf_err(err);
12368 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
12372 for (i = 0; i < s->prog_cnt; i++) {
12373 struct bpf_program *prog = *s->progs[i].prog;
12374 struct bpf_link **link = s->progs[i].link;
12376 if (!prog->autoload || !prog->autoattach)
12379 /* auto-attaching not supported for this program */
12380 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12383 /* if user already set the link manually, don't attempt auto-attach */
12387 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
12389 pr_warn("prog '%s': failed to auto-attach: %d\n",
12390 bpf_program__name(prog), err);
12391 return libbpf_err(err);
12394 /* It's possible that for some SEC() definitions auto-attach
12395 * is supported in some cases (e.g., if definition completely
12396 * specifies target information), but is not in other cases.
12397 * SEC("uprobe") is one such case. If user specified target
12398 * binary and function name, such BPF program can be
12399 * auto-attached. But if not, it shouldn't trigger skeleton's
12400 * attach to fail. It should just be skipped.
12401 * attach_fn signals such case with returning 0 (no error) and
12402 * setting link to NULL.
12409 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
12413 for (i = 0; i < s->prog_cnt; i++) {
12414 struct bpf_link **link = s->progs[i].link;
12416 bpf_link__destroy(*link);
12421 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
12427 bpf_object__detach_skeleton(s);
12429 bpf_object__close(*s->obj);