1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
5 * Copyright (c) 2021 Facebook
14 #include <linux/err.h>
15 #include <linux/btf.h>
22 #include "libbpf_internal.h"
25 #define BTF_EXTERN_SEC ".extern"
29 /* positional (not necessarily ELF) index in an array of sections */
31 /* positional (not necessarily ELF) index of a matching section in a final object file */
33 /* section data offset in a matching output section */
35 /* whether section is omitted from the final ELF file */
37 /* whether section is an ephemeral section, not mapped to an ELF section */
46 /* corresponding BTF DATASEC type ID */
54 /* Section header strings section index */
55 size_t shstrs_sec_idx;
56 /* SYMTAB section index */
57 size_t symtab_sec_idx;
60 struct btf_ext *btf_ext;
62 /* List of sections (including ephemeral). Slot zero is unused. */
66 /* mapping of symbol indices from src to dst ELF */
68 /* mapping from the src BTF type IDs to dst ones */
72 /* single .BTF.ext data section */
73 struct btf_ext_sec_data {
80 /* ELF symbol index */
82 /* associated section id for .ksyms, .kconfig, etc, but not .extern */
84 /* extern name offset in STRTAB */
86 /* optional associated BTF type ID */
88 /* BTF type ID to which VAR/FUNC type is pointing to; used for
89 * rewriting types when extern VAR/FUNC is resolved to a concrete
92 int underlying_btf_id;
93 /* sec_var index in the corresponding dst_sec, if exists */
96 /* extern or resolved/global symbol */
98 /* weak or strong symbol, never goes back from strong to weak */
104 /* positional (not necessarily ELF) index in an array of sections */
115 /* final output section size */
117 /* final output contents of the section */
120 /* corresponding STT_SECTION symbol index in SYMTAB */
123 /* section's DATASEC variable info, emitted on BTF finalization */
126 struct btf_var_secinfo *sec_vars;
128 /* section's .BTF.ext data */
129 struct btf_ext_sec_data func_info;
130 struct btf_ext_sec_data line_info;
131 struct btf_ext_sec_data core_relo_info;
140 /* Output sections metadata */
141 struct dst_sec *secs;
144 struct strset *strtab_strs; /* STRTAB unique strings */
145 size_t strtab_sec_idx; /* STRTAB section index */
146 size_t symtab_sec_idx; /* SYMTAB section index */
149 struct btf_ext *btf_ext;
151 /* global (including extern) ELF symbols */
153 struct glob_sym *glob_syms;
156 #define pr_warn_elf(fmt, ...) \
157 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
159 static int init_output_elf(struct bpf_linker *linker, const char *file);
161 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
162 const struct bpf_linker_file_opts *opts,
163 struct src_obj *obj);
164 static int linker_sanity_check_elf(struct src_obj *obj);
165 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
166 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
167 static int linker_sanity_check_btf(struct src_obj *obj);
168 static int linker_sanity_check_btf_ext(struct src_obj *obj);
169 static int linker_fixup_btf(struct src_obj *obj);
170 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
171 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
172 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
173 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
174 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
175 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
176 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
178 static int finalize_btf(struct bpf_linker *linker);
179 static int finalize_btf_ext(struct bpf_linker *linker);
181 void bpf_linker__free(struct bpf_linker *linker)
188 free(linker->filename);
191 elf_end(linker->elf);
196 strset__free(linker->strtab_strs);
198 btf__free(linker->btf);
199 btf_ext__free(linker->btf_ext);
201 for (i = 1; i < linker->sec_cnt; i++) {
202 struct dst_sec *sec = &linker->secs[i];
208 free(sec->func_info.recs);
209 free(sec->line_info.recs);
210 free(sec->core_relo_info.recs);
214 free(linker->glob_syms);
218 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
220 struct bpf_linker *linker;
223 if (!OPTS_VALID(opts, bpf_linker_opts))
224 return errno = EINVAL, NULL;
226 if (elf_version(EV_CURRENT) == EV_NONE) {
227 pr_warn_elf("libelf initialization failed");
228 return errno = EINVAL, NULL;
231 linker = calloc(1, sizeof(*linker));
233 return errno = ENOMEM, NULL;
237 err = init_output_elf(linker, filename);
244 bpf_linker__free(linker);
245 return errno = -err, NULL;
248 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
250 struct dst_sec *secs = linker->secs, *sec;
251 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
253 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
257 /* zero out newly allocated memory */
258 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
261 linker->sec_cnt = new_cnt;
263 sec = &linker->secs[new_cnt - 1];
264 sec->id = new_cnt - 1;
265 sec->sec_name = strdup(sec_name);
272 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
274 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
275 Elf64_Sym *syms, *sym;
276 size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
278 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
282 sym = &syms[sym_cnt];
283 memset(sym, 0, sizeof(*sym));
285 symtab->raw_data = syms;
286 symtab->sec_sz += sizeof(*sym);
287 symtab->shdr->sh_size += sizeof(*sym);
288 symtab->data->d_size += sizeof(*sym);
296 static int init_output_elf(struct bpf_linker *linker, const char *file)
302 linker->filename = strdup(file);
303 if (!linker->filename)
306 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
307 if (linker->fd < 0) {
309 pr_warn("failed to create '%s': %d\n", file, err);
313 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
315 pr_warn_elf("failed to create ELF object");
320 linker->elf_hdr = elf64_newehdr(linker->elf);
321 if (!linker->elf_hdr) {
322 pr_warn_elf("failed to create ELF header");
326 linker->elf_hdr->e_machine = EM_BPF;
327 linker->elf_hdr->e_type = ET_REL;
328 #if __BYTE_ORDER == __LITTLE_ENDIAN
329 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
330 #elif __BYTE_ORDER == __BIG_ENDIAN
331 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
333 #error "Unknown __BYTE_ORDER"
337 /* initialize strset with an empty string to conform to ELF */
338 linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
339 if (libbpf_get_error(linker->strtab_strs))
340 return libbpf_get_error(linker->strtab_strs);
342 sec = add_dst_sec(linker, ".strtab");
346 sec->scn = elf_newscn(linker->elf);
348 pr_warn_elf("failed to create STRTAB section");
352 sec->shdr = elf64_getshdr(sec->scn);
356 sec->data = elf_newdata(sec->scn);
358 pr_warn_elf("failed to create STRTAB data");
362 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
366 sec->sec_idx = elf_ndxscn(sec->scn);
367 linker->elf_hdr->e_shstrndx = sec->sec_idx;
368 linker->strtab_sec_idx = sec->sec_idx;
370 sec->shdr->sh_name = str_off;
371 sec->shdr->sh_type = SHT_STRTAB;
372 sec->shdr->sh_flags = SHF_STRINGS;
373 sec->shdr->sh_offset = 0;
374 sec->shdr->sh_link = 0;
375 sec->shdr->sh_info = 0;
376 sec->shdr->sh_addralign = 1;
377 sec->shdr->sh_size = sec->sec_sz = 0;
378 sec->shdr->sh_entsize = 0;
381 sec = add_dst_sec(linker, ".symtab");
385 sec->scn = elf_newscn(linker->elf);
387 pr_warn_elf("failed to create SYMTAB section");
391 sec->shdr = elf64_getshdr(sec->scn);
395 sec->data = elf_newdata(sec->scn);
397 pr_warn_elf("failed to create SYMTAB data");
401 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
405 sec->sec_idx = elf_ndxscn(sec->scn);
406 linker->symtab_sec_idx = sec->sec_idx;
408 sec->shdr->sh_name = str_off;
409 sec->shdr->sh_type = SHT_SYMTAB;
410 sec->shdr->sh_flags = 0;
411 sec->shdr->sh_offset = 0;
412 sec->shdr->sh_link = linker->strtab_sec_idx;
413 /* sh_info should be one greater than the index of the last local
414 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
416 sec->shdr->sh_info = 0;
417 sec->shdr->sh_addralign = 8;
418 sec->shdr->sh_entsize = sizeof(Elf64_Sym);
421 linker->btf = btf__new_empty();
422 err = libbpf_get_error(linker->btf);
426 /* add the special all-zero symbol */
427 init_sym = add_new_sym(linker, NULL);
431 init_sym->st_name = 0;
432 init_sym->st_info = 0;
433 init_sym->st_other = 0;
434 init_sym->st_shndx = SHN_UNDEF;
435 init_sym->st_value = 0;
436 init_sym->st_size = 0;
441 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
442 const struct bpf_linker_file_opts *opts)
444 struct src_obj obj = {};
447 if (!OPTS_VALID(opts, bpf_linker_file_opts))
448 return libbpf_err(-EINVAL);
451 return libbpf_err(-EINVAL);
453 err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
454 err = err ?: linker_append_sec_data(linker, &obj);
455 err = err ?: linker_append_elf_syms(linker, &obj);
456 err = err ?: linker_append_elf_relos(linker, &obj);
457 err = err ?: linker_append_btf(linker, &obj);
458 err = err ?: linker_append_btf_ext(linker, &obj);
460 /* free up src_obj resources */
461 free(obj.btf_type_map);
463 btf_ext__free(obj.btf_ext);
471 return libbpf_err(err);
474 static bool is_dwarf_sec_name(const char *name)
476 /* approximation, but the actual list is too long */
477 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
480 static bool is_ignored_sec(struct src_sec *sec)
482 Elf64_Shdr *shdr = sec->shdr;
483 const char *name = sec->sec_name;
485 /* no special handling of .strtab */
486 if (shdr->sh_type == SHT_STRTAB)
489 /* ignore .llvm_addrsig section as well */
490 if (shdr->sh_type == SHT_LLVM_ADDRSIG)
493 /* no subprograms will lead to an empty .text section, ignore it */
494 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
495 strcmp(sec->sec_name, ".text") == 0)
499 if (is_dwarf_sec_name(sec->sec_name))
502 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
503 name += sizeof(".rel") - 1;
504 /* DWARF section relocations */
505 if (is_dwarf_sec_name(name))
508 /* .BTF and .BTF.ext don't need relocations */
509 if (strcmp(name, BTF_ELF_SEC) == 0 ||
510 strcmp(name, BTF_EXT_ELF_SEC) == 0)
517 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
519 struct src_sec *secs = obj->secs, *sec;
520 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
522 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
526 /* zero out newly allocated memory */
527 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
530 obj->sec_cnt = new_cnt;
532 sec = &obj->secs[new_cnt - 1];
533 sec->id = new_cnt - 1;
534 sec->sec_name = sec_name;
539 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
540 const struct bpf_linker_file_opts *opts,
543 #if __BYTE_ORDER == __LITTLE_ENDIAN
544 const int host_endianness = ELFDATA2LSB;
545 #elif __BYTE_ORDER == __BIG_ENDIAN
546 const int host_endianness = ELFDATA2MSB;
548 #error "Unknown __BYTE_ORDER"
557 pr_debug("linker: adding object file '%s'...\n", filename);
559 obj->filename = filename;
561 obj->fd = open(filename, O_RDONLY);
564 pr_warn("failed to open file '%s': %d\n", filename, err);
567 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
570 pr_warn_elf("failed to parse ELF file '%s'", filename);
574 /* Sanity check ELF file high-level properties */
575 ehdr = elf64_getehdr(obj->elf);
578 pr_warn_elf("failed to get ELF header for %s", filename);
581 if (ehdr->e_ident[EI_DATA] != host_endianness) {
583 pr_warn_elf("unsupported byte order of ELF file %s", filename);
586 if (ehdr->e_type != ET_REL
587 || ehdr->e_machine != EM_BPF
588 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
590 pr_warn_elf("unsupported kind of ELF file %s", filename);
594 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
596 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
601 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
602 size_t sec_idx = elf_ndxscn(scn);
603 const char *sec_name;
605 shdr = elf64_getshdr(scn);
608 pr_warn_elf("failed to get section #%zu header for %s",
613 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
616 pr_warn_elf("failed to get section #%zu name for %s",
621 data = elf_getdata(scn, 0);
624 pr_warn_elf("failed to get section #%zu (%s) data from %s",
625 sec_idx, sec_name, filename);
629 sec = add_src_sec(obj, sec_name);
636 sec->sec_idx = elf_ndxscn(scn);
638 if (is_ignored_sec(sec)) {
643 switch (shdr->sh_type) {
645 if (obj->symtab_sec_idx) {
647 pr_warn("multiple SYMTAB sections found, not supported\n");
650 obj->symtab_sec_idx = sec_idx;
653 /* we'll construct our own string table */
656 if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
657 obj->btf = btf__new(data->d_buf, shdr->sh_size);
658 err = libbpf_get_error(obj->btf);
660 pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
666 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
667 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
668 err = libbpf_get_error(obj->btf_ext);
670 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
686 pr_warn("unrecognized section #%zu (%s) in %s\n",
687 sec_idx, sec_name, filename);
693 err = err ?: linker_sanity_check_elf(obj);
694 err = err ?: linker_sanity_check_btf(obj);
695 err = err ?: linker_sanity_check_btf_ext(obj);
696 err = err ?: linker_fixup_btf(obj);
701 static bool is_pow_of_2(size_t x)
703 return x && (x & (x - 1)) == 0;
706 static int linker_sanity_check_elf(struct src_obj *obj)
711 if (!obj->symtab_sec_idx) {
712 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
715 if (!obj->shstrs_sec_idx) {
716 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
720 for (i = 1; i < obj->sec_cnt; i++) {
723 if (sec->sec_name[0] == '\0') {
724 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
728 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign))
730 if (sec->shdr->sh_addralign != sec->data->d_align)
733 if (sec->shdr->sh_size != sec->data->d_size)
736 switch (sec->shdr->sh_type) {
738 err = linker_sanity_check_elf_symtab(obj, sec);
745 if (sec->shdr->sh_flags & SHF_EXECINSTR) {
746 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0)
753 err = linker_sanity_check_elf_relos(obj, sec);
757 case SHT_LLVM_ADDRSIG:
760 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
761 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
769 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
771 struct src_sec *link_sec;
775 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
777 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
780 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
781 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
782 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
785 link_sec = &obj->secs[sec->shdr->sh_link];
786 if (link_sec->shdr->sh_type != SHT_STRTAB) {
787 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
788 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
792 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
793 sym = sec->data->d_buf;
794 for (i = 0; i < n; i++, sym++) {
795 int sym_type = ELF64_ST_TYPE(sym->st_info);
796 int sym_bind = ELF64_ST_BIND(sym->st_info);
797 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
800 if (sym->st_name != 0 || sym->st_info != 0
801 || sym->st_other != 0 || sym->st_shndx != 0
802 || sym->st_value != 0 || sym->st_size != 0) {
803 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
808 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
809 pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
810 i, sec->sec_idx, sym_bind);
813 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
814 pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
815 i, sec->sec_idx, sym_vis);
818 if (sym->st_shndx == 0) {
819 if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
820 || sym->st_value != 0 || sym->st_size != 0) {
821 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
828 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
829 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
830 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
833 if (sym_type == STT_SECTION) {
834 if (sym->st_value != 0)
843 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
845 struct src_sec *link_sec, *sym_sec;
849 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
851 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
854 /* SHT_REL's sh_link should point to SYMTAB */
855 if (sec->shdr->sh_link != obj->symtab_sec_idx) {
856 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
857 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
861 /* SHT_REL's sh_info points to relocated section */
862 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
863 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
864 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
867 link_sec = &obj->secs[sec->shdr->sh_info];
869 /* .rel<secname> -> <secname> pattern is followed */
870 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
871 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
872 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
873 sec->sec_idx, obj->filename);
877 /* don't further validate relocations for ignored sections */
878 if (link_sec->skipped)
881 /* relocatable section is data or instructions */
882 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
883 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
884 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
888 /* check sanity of each relocation */
889 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
890 relo = sec->data->d_buf;
891 sym_sec = &obj->secs[obj->symtab_sec_idx];
892 for (i = 0; i < n; i++, relo++) {
893 size_t sym_idx = ELF64_R_SYM(relo->r_info);
894 size_t sym_type = ELF64_R_TYPE(relo->r_info);
896 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
897 sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
898 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
899 i, sec->sec_idx, sym_type, obj->filename);
903 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
904 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
905 i, sec->sec_idx, sym_idx, obj->filename);
909 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
910 if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
911 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
912 i, sec->sec_idx, sym_idx, obj->filename);
921 static int check_btf_type_id(__u32 *type_id, void *ctx)
923 struct btf *btf = ctx;
925 if (*type_id > btf__get_nr_types(btf))
931 static int check_btf_str_off(__u32 *str_off, void *ctx)
933 struct btf *btf = ctx;
936 s = btf__str_by_offset(btf, *str_off);
944 static int linker_sanity_check_btf(struct src_obj *obj)
952 n = btf__get_nr_types(obj->btf);
953 for (i = 1; i <= n; i++) {
954 t = btf_type_by_id(obj->btf, i);
956 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
957 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
965 static int linker_sanity_check_btf_ext(struct src_obj *obj)
972 /* can't use .BTF.ext without .BTF */
976 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
977 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
984 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
992 dst_sec->sec_idx = 0;
993 dst_sec->ephemeral = src_sec->ephemeral;
995 /* ephemeral sections are just thin section shells lacking most parts */
996 if (src_sec->ephemeral)
999 scn = elf_newscn(linker->elf);
1002 data = elf_newdata(scn);
1005 shdr = elf64_getshdr(scn);
1010 dst_sec->shdr = shdr;
1011 dst_sec->data = data;
1012 dst_sec->sec_idx = elf_ndxscn(scn);
1014 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1018 shdr->sh_name = name_off;
1019 shdr->sh_type = src_sec->shdr->sh_type;
1020 shdr->sh_flags = src_sec->shdr->sh_flags;
1022 /* sh_link and sh_info have different meaning for different types of
1023 * sections, so we leave it up to the caller code to fill them in, if
1028 shdr->sh_addralign = src_sec->shdr->sh_addralign;
1029 shdr->sh_entsize = src_sec->shdr->sh_entsize;
1031 data->d_type = src_sec->data->d_type;
1034 data->d_align = src_sec->data->d_align;
1040 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1042 struct dst_sec *sec;
1045 for (i = 1; i < linker->sec_cnt; i++) {
1046 sec = &linker->secs[i];
1048 if (strcmp(sec->sec_name, sec_name) == 0)
1055 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1057 if (dst->ephemeral || src->ephemeral)
1060 if (dst->shdr->sh_type != src->shdr->sh_type) {
1061 pr_warn("sec %s types mismatch\n", dst->sec_name);
1064 if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1065 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1068 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1069 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1076 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1078 if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1080 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1085 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1088 size_t dst_align, src_align;
1089 size_t dst_align_sz, dst_final_sz;
1092 /* Ephemeral source section doesn't contribute anything to ELF
1098 /* Some sections (like .maps) can contain both externs (and thus be
1099 * ephemeral) and non-externs (map definitions). So it's possible that
1100 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1101 * first non-ephemeral entity appears. In such case, we add ELF
1102 * section, data, etc.
1104 if (dst->ephemeral) {
1105 err = init_sec(linker, dst, src);
1110 dst_align = dst->shdr->sh_addralign;
1111 src_align = src->shdr->sh_addralign;
1114 if (dst_align < src_align)
1115 dst_align = src_align;
1117 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1119 /* no need to re-align final size */
1120 dst_final_sz = dst_align_sz + src->shdr->sh_size;
1122 if (src->shdr->sh_type != SHT_NOBITS) {
1123 tmp = realloc(dst->raw_data, dst_final_sz);
1126 dst->raw_data = tmp;
1128 /* pad dst section, if it's alignment forced size increase */
1129 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1130 /* now copy src data at a properly aligned offset */
1131 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1134 dst->sec_sz = dst_final_sz;
1135 dst->shdr->sh_size = dst_final_sz;
1136 dst->data->d_size = dst_final_sz;
1138 dst->shdr->sh_addralign = dst_align;
1139 dst->data->d_align = dst_align;
1141 src->dst_off = dst_align_sz;
1146 static bool is_data_sec(struct src_sec *sec)
1148 if (!sec || sec->skipped)
1150 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1153 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1156 static bool is_relo_sec(struct src_sec *sec)
1158 if (!sec || sec->skipped || sec->ephemeral)
1160 return sec->shdr->sh_type == SHT_REL;
1163 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1167 for (i = 1; i < obj->sec_cnt; i++) {
1168 struct src_sec *src_sec;
1169 struct dst_sec *dst_sec;
1171 src_sec = &obj->secs[i];
1172 if (!is_data_sec(src_sec))
1175 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1177 dst_sec = add_dst_sec(linker, src_sec->sec_name);
1180 err = init_sec(linker, dst_sec, src_sec);
1182 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1186 if (!secs_match(dst_sec, src_sec)) {
1187 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1191 /* "license" and "version" sections are deduped */
1192 if (strcmp(src_sec->sec_name, "license") == 0
1193 || strcmp(src_sec->sec_name, "version") == 0) {
1194 if (!sec_content_is_same(dst_sec, src_sec)) {
1195 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1198 src_sec->skipped = true;
1199 src_sec->dst_id = dst_sec->id;
1204 /* record mapped section index */
1205 src_sec->dst_id = dst_sec->id;
1207 err = extend_sec(linker, dst_sec, src_sec);
1215 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1217 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1218 Elf64_Sym *sym = symtab->data->d_buf;
1219 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1220 int str_sec_idx = symtab->shdr->sh_link;
1221 const char *sym_name;
1223 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1227 for (i = 0; i < n; i++, sym++) {
1228 /* We already validated all-zero symbol #0 and we already
1229 * appended it preventively to the final SYMTAB, so skip it.
1234 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1236 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1240 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1248 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1250 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1251 Elf64_Sym *syms = symtab->raw_data;
1253 return &syms[sym_idx];
1256 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1258 struct glob_sym *glob_sym;
1262 for (i = 0; i < linker->glob_sym_cnt; i++) {
1263 glob_sym = &linker->glob_syms[i];
1264 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1266 if (strcmp(name, sym_name) == 0)
1273 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1275 struct glob_sym *syms, *sym;
1277 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1278 sizeof(*linker->glob_syms));
1282 sym = &syms[linker->glob_sym_cnt];
1283 memset(sym, 0, sizeof(*sym));
1286 linker->glob_syms = syms;
1287 linker->glob_sym_cnt++;
1292 static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1293 const struct btf *btf1, __u32 id1,
1294 const struct btf *btf2, __u32 id2)
1296 const struct btf_type *t1, *t2;
1297 bool is_static1, is_static2;
1298 const char *n1, *n2;
1303 t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1304 t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1306 /* check if only one side is FWD, otherwise handle with common logic */
1307 if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1308 n1 = btf__str_by_offset(btf1, t1->name_off);
1309 n2 = btf__str_by_offset(btf2, t2->name_off);
1310 if (strcmp(n1, n2) != 0) {
1311 pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1315 /* validate if FWD kind matches concrete kind */
1316 if (btf_is_fwd(t1)) {
1317 if (btf_kflag(t1) && btf_is_union(t2))
1319 if (!btf_kflag(t1) && btf_is_struct(t2))
1321 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1322 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1324 if (btf_kflag(t2) && btf_is_union(t1))
1326 if (!btf_kflag(t2) && btf_is_struct(t1))
1328 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1329 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1334 if (btf_kind(t1) != btf_kind(t2)) {
1335 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1336 sym_name, btf_kind_str(t1), btf_kind_str(t2));
1340 switch (btf_kind(t1)) {
1341 case BTF_KIND_STRUCT:
1342 case BTF_KIND_UNION:
1347 n1 = btf__str_by_offset(btf1, t1->name_off);
1348 n2 = btf__str_by_offset(btf2, t2->name_off);
1349 if (strcmp(n1, n2) != 0) {
1350 pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1351 sym_name, btf_kind_str(t1), n1, n2);
1359 switch (btf_kind(t1)) {
1360 case BTF_KIND_UNKN: /* void */
1364 case BTF_KIND_FLOAT:
1366 /* ignore encoding for int and enum values for enum */
1367 if (t1->size != t2->size) {
1368 pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1369 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1374 /* just validate overall shape of the referenced type, so no
1375 * contents comparison for struct/union, and allowd fwd vs
1382 case BTF_KIND_ARRAY:
1383 /* ignore index type and array size */
1384 id1 = btf_array(t1)->type;
1385 id2 = btf_array(t2)->type;
1388 /* extern and global linkages are compatible */
1389 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1390 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1391 if (is_static1 != is_static2) {
1392 pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1400 /* extern and global linkages are compatible */
1401 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1402 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1403 if (is_static1 != is_static2) {
1404 pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1411 case BTF_KIND_STRUCT:
1412 case BTF_KIND_UNION: {
1413 const struct btf_member *m1, *m2;
1418 if (btf_vlen(t1) != btf_vlen(t2)) {
1419 pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1420 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1425 m1 = btf_members(t1);
1426 m2 = btf_members(t2);
1427 for (i = 0; i < n; i++, m1++, m2++) {
1428 n1 = btf__str_by_offset(btf1, m1->name_off);
1429 n2 = btf__str_by_offset(btf2, m2->name_off);
1430 if (strcmp(n1, n2) != 0) {
1431 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1432 sym_name, i, n1, n2);
1435 if (m1->offset != m2->offset) {
1436 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1440 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1446 case BTF_KIND_FUNC_PROTO: {
1447 const struct btf_param *m1, *m2;
1449 if (btf_vlen(t1) != btf_vlen(t2)) {
1450 pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1451 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1456 m1 = btf_params(t1);
1457 m2 = btf_params(t2);
1458 for (i = 0; i < n; i++, m1++, m2++) {
1459 /* ignore func arg names */
1460 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1464 /* now check return type as well */
1470 /* skip_mods_and_typedefs() make this impossible */
1471 case BTF_KIND_TYPEDEF:
1472 case BTF_KIND_VOLATILE:
1473 case BTF_KIND_CONST:
1474 case BTF_KIND_RESTRICT:
1475 /* DATASECs are never compared with each other */
1476 case BTF_KIND_DATASEC:
1478 pr_warn("global '%s': unsupported BTF kind %s\n",
1479 sym_name, btf_kind_str(t1));
1484 static bool map_defs_match(const char *sym_name,
1485 const struct btf *main_btf,
1486 const struct btf_map_def *main_def,
1487 const struct btf_map_def *main_inner_def,
1488 const struct btf *extra_btf,
1489 const struct btf_map_def *extra_def,
1490 const struct btf_map_def *extra_inner_def)
1494 if (main_def->map_type != extra_def->map_type) {
1499 /* check key type/size match */
1500 if (main_def->key_size != extra_def->key_size) {
1501 reason = "key_size";
1504 if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1505 reason = "key type";
1508 if ((main_def->parts & MAP_DEF_KEY_TYPE)
1509 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1510 main_btf, main_def->key_type_id,
1511 extra_btf, extra_def->key_type_id)) {
1512 reason = "key type";
1516 /* validate value type/size match */
1517 if (main_def->value_size != extra_def->value_size) {
1518 reason = "value_size";
1521 if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1522 reason = "value type";
1525 if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1526 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1527 main_btf, main_def->value_type_id,
1528 extra_btf, extra_def->value_type_id)) {
1529 reason = "key type";
1533 if (main_def->max_entries != extra_def->max_entries) {
1534 reason = "max_entries";
1537 if (main_def->map_flags != extra_def->map_flags) {
1538 reason = "map_flags";
1541 if (main_def->numa_node != extra_def->numa_node) {
1542 reason = "numa_node";
1545 if (main_def->pinning != extra_def->pinning) {
1550 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1551 reason = "inner map";
1555 if (main_def->parts & MAP_DEF_INNER_MAP) {
1556 char inner_map_name[128];
1558 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1560 return map_defs_match(inner_map_name,
1561 main_btf, main_inner_def, NULL,
1562 extra_btf, extra_inner_def, NULL);
1568 pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1572 static bool glob_map_defs_match(const char *sym_name,
1573 struct bpf_linker *linker, struct glob_sym *glob_sym,
1574 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1576 struct btf_map_def dst_def = {}, dst_inner_def = {};
1577 struct btf_map_def src_def = {}, src_inner_def = {};
1578 const struct btf_type *t;
1581 t = btf__type_by_id(obj->btf, btf_id);
1582 if (!btf_is_var(t)) {
1583 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1586 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1588 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1590 pr_warn("global '%s': invalid map definition\n", sym_name);
1594 /* re-parse existing map definition */
1595 t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1596 t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1597 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1599 /* this should not happen, because we already validated it */
1600 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1604 /* Currently extern map definition has to be complete and match
1605 * concrete map definition exactly. This restriction might be lifted
1608 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1609 obj->btf, &src_def, &src_inner_def);
1612 static bool glob_syms_match(const char *sym_name,
1613 struct bpf_linker *linker, struct glob_sym *glob_sym,
1614 struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1616 const struct btf_type *src_t;
1618 /* if we are dealing with externs, BTF types describing both global
1619 * and extern VARs/FUNCs should be completely present in all files
1621 if (!glob_sym->btf_id || !btf_id) {
1622 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1626 src_t = btf__type_by_id(obj->btf, btf_id);
1627 if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1628 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1629 btf_kind_str(src_t), sym_name);
1633 /* deal with .maps definitions specially */
1634 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1635 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1637 if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1638 linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1644 static bool btf_is_non_static(const struct btf_type *t)
1646 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1647 || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1650 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1651 int *out_btf_sec_id, int *out_btf_id)
1653 int i, j, n, m, btf_id = 0;
1654 const struct btf_type *t;
1655 const struct btf_var_secinfo *vi;
1659 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1663 n = btf__get_nr_types(obj->btf);
1664 for (i = 1; i <= n; i++) {
1665 t = btf__type_by_id(obj->btf, i);
1667 /* some global and extern FUNCs and VARs might not be associated with any
1668 * DATASEC, so try to detect them in the same pass
1670 if (btf_is_non_static(t)) {
1671 name = btf__str_by_offset(obj->btf, t->name_off);
1672 if (strcmp(name, sym_name) != 0)
1675 /* remember and still try to find DATASEC */
1680 if (!btf_is_datasec(t))
1683 vi = btf_var_secinfos(t);
1684 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1685 t = btf__type_by_id(obj->btf, vi->type);
1686 name = btf__str_by_offset(obj->btf, t->name_off);
1688 if (strcmp(name, sym_name) != 0)
1690 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1692 if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1695 if (btf_id && btf_id != vi->type) {
1696 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1697 sym_name, btf_id, vi->type);
1701 *out_btf_sec_id = i;
1702 *out_btf_id = vi->type;
1708 /* free-floating extern or global FUNC */
1710 *out_btf_sec_id = 0;
1711 *out_btf_id = btf_id;
1715 pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1719 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1721 struct src_sec *sec;
1724 for (i = 1; i < obj->sec_cnt; i++) {
1725 sec = &obj->secs[i];
1727 if (strcmp(sec->sec_name, sec_name) == 0)
1734 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1735 struct btf *src_btf, int src_id)
1737 struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1738 struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1739 struct btf_param *src_p, *dst_p;
1743 /* We already made sure that source and destination types (FUNC or
1744 * VAR) match in terms of types and argument names.
1746 if (btf_is_var(dst_t)) {
1747 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1751 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1753 /* now onto FUNC_PROTO types */
1754 src_t = btf_type_by_id(src_btf, src_t->type);
1755 dst_t = btf_type_by_id(dst_btf, dst_t->type);
1757 /* Fill in all the argument names, which for extern FUNCs are missing.
1758 * We'll end up with two copies of FUNCs/VARs for externs, but that
1759 * will be taken care of by BTF dedup at the very end.
1760 * It might be that BTF types for extern in one file has less/more BTF
1761 * information (e.g., FWD instead of full STRUCT/UNION information),
1762 * but that should be (in most cases, subject to BTF dedup rules)
1763 * handled and resolved by BTF dedup algorithm as well, so we won't
1764 * worry about it. Our only job is to make sure that argument names
1765 * are populated on both sides, otherwise BTF dedup will pedantically
1766 * consider them different.
1768 src_p = btf_params(src_t);
1769 dst_p = btf_params(dst_t);
1770 for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1771 if (!src_p->name_off)
1774 /* src_btf has more complete info, so add name to dst_btf */
1775 s = btf__str_by_offset(src_btf, src_p->name_off);
1776 off = btf__add_str(dst_btf, s);
1779 dst_p->name_off = off;
1784 static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1786 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1789 static void sym_update_type(Elf64_Sym *sym, int sym_type)
1791 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1794 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1796 /* libelf doesn't provide setters for ST_VISIBILITY,
1797 * but it is stored in the lower 2 bits of st_other
1799 sym->st_other &= ~0x03;
1800 sym->st_other |= sym_vis;
1803 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1804 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1806 struct src_sec *src_sec = NULL;
1807 struct dst_sec *dst_sec = NULL;
1808 struct glob_sym *glob_sym = NULL;
1809 int name_off, sym_type, sym_bind, sym_vis, err;
1810 int btf_sec_id = 0, btf_id = 0;
1815 sym_type = ELF64_ST_TYPE(sym->st_info);
1816 sym_bind = ELF64_ST_BIND(sym->st_info);
1817 sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1818 sym_is_extern = sym->st_shndx == SHN_UNDEF;
1820 if (sym_is_extern) {
1822 pr_warn("externs without BTF info are not supported\n");
1825 } else if (sym->st_shndx < SHN_LORESERVE) {
1826 src_sec = &obj->secs[sym->st_shndx];
1827 if (src_sec->skipped)
1829 dst_sec = &linker->secs[src_sec->dst_id];
1831 /* allow only one STT_SECTION symbol per section */
1832 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1833 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1838 if (sym_bind == STB_LOCAL)
1841 /* find matching BTF info */
1842 err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1846 if (sym_is_extern && btf_sec_id) {
1847 const char *sec_name = NULL;
1848 const struct btf_type *t;
1850 t = btf__type_by_id(obj->btf, btf_sec_id);
1851 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1853 /* Clang puts unannotated extern vars into
1854 * '.extern' BTF DATASEC. Treat them the same
1855 * as unannotated extern funcs (which are
1856 * currently not put into any DATASECs).
1857 * Those don't have associated src_sec/dst_sec.
1859 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1860 src_sec = find_src_sec_by_name(obj, sec_name);
1862 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1865 dst_sec = &linker->secs[src_sec->dst_id];
1869 glob_sym = find_glob_sym(linker, sym_name);
1871 /* Preventively resolve to existing symbol. This is
1872 * needed for further relocation symbol remapping in
1873 * the next step of linking.
1875 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1877 /* If both symbols are non-externs, at least one of
1878 * them has to be STB_WEAK, otherwise they are in
1879 * a conflict with each other.
1881 if (!sym_is_extern && !glob_sym->is_extern
1882 && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1883 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1884 src_sym_idx, sym_name, obj->filename);
1888 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1891 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1893 /* If new symbol is strong, then force dst_sym to be strong as
1894 * well; this way a mix of weak and non-weak extern
1895 * definitions will end up being strong.
1897 if (sym_bind == STB_GLOBAL) {
1898 /* We still need to preserve type (NOTYPE or
1899 * OBJECT/FUNC, depending on whether the symbol is
1902 sym_update_bind(dst_sym, STB_GLOBAL);
1903 glob_sym->is_weak = false;
1906 /* Non-default visibility is "contaminating", with stricter
1907 * visibility overwriting more permissive ones, even if more
1908 * permissive visibility comes from just an extern definition.
1909 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1910 * ensured by ELF symbol sanity checks above.
1912 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1913 sym_update_visibility(dst_sym, sym_vis);
1915 /* If the new symbol is extern, then regardless if
1916 * existing symbol is extern or resolved global, just
1917 * keep the existing one untouched.
1922 /* If existing symbol is a strong resolved symbol, bail out,
1923 * because we lost resolution battle have nothing to
1924 * contribute. We already checked abover that there is no
1925 * strong-strong conflict. We also already tightened binding
1926 * and visibility, so nothing else to contribute at that point.
1928 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
1931 /* At this point, new symbol is strong non-extern,
1932 * so overwrite glob_sym with new symbol information.
1933 * Preserve binding and visibility.
1935 sym_update_type(dst_sym, sym_type);
1936 dst_sym->st_shndx = dst_sec->sec_idx;
1937 dst_sym->st_value = src_sec->dst_off + sym->st_value;
1938 dst_sym->st_size = sym->st_size;
1940 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
1941 glob_sym->sec_id = dst_sec->id;
1942 glob_sym->is_extern = false;
1944 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
1948 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
1949 glob_sym->underlying_btf_id = 0;
1951 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1956 name_off = strset__add_str(linker->strtab_strs, sym_name);
1960 dst_sym = add_new_sym(linker, &dst_sym_idx);
1964 dst_sym->st_name = name_off;
1965 dst_sym->st_info = sym->st_info;
1966 dst_sym->st_other = sym->st_other;
1967 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
1968 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1969 dst_sym->st_size = sym->st_size;
1971 obj->sym_map[src_sym_idx] = dst_sym_idx;
1973 if (sym_type == STT_SECTION && dst_sym) {
1974 dst_sec->sec_sym_idx = dst_sym_idx;
1975 dst_sym->st_value = 0;
1978 if (sym_bind != STB_LOCAL) {
1979 glob_sym = add_glob_sym(linker);
1983 glob_sym->sym_idx = dst_sym_idx;
1984 /* we use dst_sec->id (and not dst_sec->sec_idx), because
1985 * ephemeral sections (.kconfig, .ksyms, etc) don't have
1986 * sec_idx (as they don't have corresponding ELF section), but
1987 * still have id. .extern doesn't have even ephemeral section
1988 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
1990 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
1991 glob_sym->name_off = name_off;
1992 /* we will fill btf_id in during BTF merging step */
1993 glob_sym->btf_id = 0;
1994 glob_sym->is_extern = sym_is_extern;
1995 glob_sym->is_weak = sym_bind == STB_WEAK;
2001 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2003 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2004 struct dst_sec *dst_symtab;
2007 for (i = 1; i < obj->sec_cnt; i++) {
2008 struct src_sec *src_sec, *src_linked_sec;
2009 struct dst_sec *dst_sec, *dst_linked_sec;
2010 Elf64_Rel *src_rel, *dst_rel;
2013 src_sec = &obj->secs[i];
2014 if (!is_relo_sec(src_sec))
2017 /* shdr->sh_info points to relocatable section */
2018 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2019 if (src_linked_sec->skipped)
2022 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2024 dst_sec = add_dst_sec(linker, src_sec->sec_name);
2027 err = init_sec(linker, dst_sec, src_sec);
2029 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2032 } else if (!secs_match(dst_sec, src_sec)) {
2033 pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2037 /* add_dst_sec() above could have invalidated linker->secs */
2038 dst_symtab = &linker->secs[linker->symtab_sec_idx];
2040 /* shdr->sh_link points to SYMTAB */
2041 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2043 /* shdr->sh_info points to relocated section */
2044 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2045 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2047 src_sec->dst_id = dst_sec->id;
2048 err = extend_sec(linker, dst_sec, src_sec);
2052 src_rel = src_sec->data->d_buf;
2053 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2054 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2055 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2056 size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2057 size_t sym_type = ELF64_R_TYPE(src_rel->r_info);
2058 Elf64_Sym *src_sym, *dst_sym;
2061 src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2062 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2064 dst_sym_idx = obj->sym_map[src_sym_idx];
2065 dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx;
2066 dst_rel->r_offset += src_linked_sec->dst_off;
2067 sym_type = ELF64_R_TYPE(src_rel->r_info);
2068 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2070 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2071 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2072 struct bpf_insn *insn;
2074 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2075 /* calls to the very first static function inside
2076 * .text section at offset 0 will
2077 * reference section symbol, not the
2078 * function symbol. Fix that up,
2079 * otherwise it won't be possible to
2080 * relocate calls to two different
2081 * static functions with the same name
2082 * (rom two different object files)
2084 insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2085 if (insn->code == (BPF_JMP | BPF_CALL))
2086 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2088 insn->imm += sec->dst_off;
2090 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2101 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2102 int sym_type, const char *sym_name)
2104 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2105 Elf64_Sym *sym = symtab->data->d_buf;
2106 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2107 int str_sec_idx = symtab->shdr->sh_link;
2110 for (i = 0; i < n; i++, sym++) {
2111 if (sym->st_shndx != sec_idx)
2113 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2116 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2120 if (strcmp(sym_name, name) != 0)
2129 static int linker_fixup_btf(struct src_obj *obj)
2131 const char *sec_name;
2132 struct src_sec *sec;
2138 n = btf__get_nr_types(obj->btf);
2139 for (i = 1; i <= n; i++) {
2140 struct btf_var_secinfo *vi;
2143 t = btf_type_by_id(obj->btf, i);
2144 if (btf_kind(t) != BTF_KIND_DATASEC)
2147 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2148 sec = find_src_sec_by_name(obj, sec_name);
2150 /* record actual section size, unless ephemeral */
2152 t->size = sec->shdr->sh_size;
2154 /* BTF can have some sections that are not represented
2155 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2156 * for special extern variables.
2158 * For all but one such special (ephemeral)
2159 * sections, we pre-create "section shells" to be able
2160 * to keep track of extra per-section metadata later
2161 * (e.g., those BTF extern variables).
2163 * .extern is even more special, though, because it
2164 * contains extern variables that need to be resolved
2165 * by static linker, not libbpf and kernel. When such
2166 * externs are resolved, we are going to remove them
2167 * from .extern BTF section and might end up not
2168 * needing it at all. Each resolved extern should have
2169 * matching non-extern VAR/FUNC in other sections.
2171 * We do support leaving some of the externs
2172 * unresolved, though, to support cases of building
2173 * libraries, which will later be linked against final
2174 * BPF applications. So if at finalization we still
2175 * see unresolved externs, we'll create .extern
2176 * section on our own.
2178 if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2181 sec = add_src_sec(obj, sec_name);
2185 sec->ephemeral = true;
2186 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2189 /* remember ELF section and its BTF type ID match */
2190 sec->sec_type_id = i;
2192 /* fix up variable offsets */
2193 vi = btf_var_secinfos(t);
2194 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2195 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2196 const char *var_name = btf__str_by_offset(obj->btf, vt->name_off);
2197 int var_linkage = btf_var(vt)->linkage;
2200 /* no need to patch up static or extern vars */
2201 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2204 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2206 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2210 vi->offset = sym->st_value;
2217 static int remap_type_id(__u32 *type_id, void *ctx)
2220 int new_id = id_map[*type_id];
2222 /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2223 if (new_id == 0 && *type_id != 0) {
2224 pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id);
2228 *type_id = id_map[*type_id];
2233 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2235 const struct btf_type *t;
2236 int i, j, n, start_id, id;
2242 start_id = btf__get_nr_types(linker->btf) + 1;
2243 n = btf__get_nr_types(obj->btf);
2245 obj->btf_type_map = calloc(n + 1, sizeof(int));
2246 if (!obj->btf_type_map)
2249 for (i = 1; i <= n; i++) {
2250 struct glob_sym *glob_sym = NULL;
2252 t = btf__type_by_id(obj->btf, i);
2254 /* DATASECs are handled specially below */
2255 if (btf_kind(t) == BTF_KIND_DATASEC)
2258 if (btf_is_non_static(t)) {
2259 /* there should be glob_sym already */
2260 name = btf__str_by_offset(obj->btf, t->name_off);
2261 glob_sym = find_glob_sym(linker, name);
2263 /* VARs without corresponding glob_sym are those that
2264 * belong to skipped/deduplicated sections (i.e.,
2265 * license and version), so just skip them
2270 /* linker_append_elf_sym() might have requested
2271 * updating underlying type ID, if extern was resolved
2272 * to strong symbol or weak got upgraded to non-weak
2274 if (glob_sym->underlying_btf_id == 0)
2275 glob_sym->underlying_btf_id = -t->type;
2277 /* globals from previous object files that match our
2278 * VAR/FUNC already have a corresponding associated
2279 * BTF type, so just make sure to use it
2281 if (glob_sym->btf_id) {
2282 /* reuse existing BTF type for global var/func */
2283 obj->btf_type_map[i] = glob_sym->btf_id;
2288 id = btf__add_type(linker->btf, obj->btf, t);
2290 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2294 obj->btf_type_map[i] = id;
2296 /* record just appended BTF type for var/func */
2298 glob_sym->btf_id = id;
2299 glob_sym->underlying_btf_id = -t->type;
2303 /* remap all the types except DATASECs */
2304 n = btf__get_nr_types(linker->btf);
2305 for (i = start_id; i <= n; i++) {
2306 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2308 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
2312 /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2313 * actual type), if necessary
2315 for (i = 0; i < linker->glob_sym_cnt; i++) {
2316 struct glob_sym *glob_sym = &linker->glob_syms[i];
2317 struct btf_type *glob_t;
2319 if (glob_sym->underlying_btf_id >= 0)
2322 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2324 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2325 glob_t->type = glob_sym->underlying_btf_id;
2328 /* append DATASEC info */
2329 for (i = 1; i < obj->sec_cnt; i++) {
2330 struct src_sec *src_sec;
2331 struct dst_sec *dst_sec;
2332 const struct btf_var_secinfo *src_var;
2333 struct btf_var_secinfo *dst_var;
2335 src_sec = &obj->secs[i];
2336 if (!src_sec->sec_type_id || src_sec->skipped)
2338 dst_sec = &linker->secs[src_sec->dst_id];
2340 /* Mark section as having BTF regardless of the presence of
2341 * variables. In some cases compiler might generate empty BTF
2342 * with no variables information. E.g., when promoting local
2343 * array/structure variable initial values and BPF object
2344 * file otherwise has no read-only static variables in
2345 * .rodata. We need to preserve such empty BTF and just set
2346 * correct section size.
2348 dst_sec->has_btf = true;
2350 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2351 src_var = btf_var_secinfos(t);
2353 for (j = 0; j < n; j++, src_var++) {
2354 void *sec_vars = dst_sec->sec_vars;
2355 int new_id = obj->btf_type_map[src_var->type];
2356 struct glob_sym *glob_sym = NULL;
2358 t = btf_type_by_id(linker->btf, new_id);
2359 if (btf_is_non_static(t)) {
2360 name = btf__str_by_offset(linker->btf, t->name_off);
2361 glob_sym = find_glob_sym(linker, name);
2362 if (glob_sym->sec_id != dst_sec->id) {
2363 pr_warn("global '%s': section mismatch %d vs %d\n",
2364 name, glob_sym->sec_id, dst_sec->id);
2369 /* If there is already a member (VAR or FUNC) mapped
2370 * to the same type, don't add a duplicate entry.
2371 * This will happen when multiple object files define
2372 * the same extern VARs/FUNCs.
2374 if (glob_sym && glob_sym->var_idx >= 0) {
2377 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2378 /* Because underlying BTF type might have
2379 * changed, so might its size have changed, so
2380 * re-calculate and update it in sec_var.
2382 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2384 pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2392 sec_vars = libbpf_reallocarray(sec_vars,
2393 dst_sec->sec_var_cnt + 1,
2394 sizeof(*dst_sec->sec_vars));
2398 dst_sec->sec_vars = sec_vars;
2399 dst_sec->sec_var_cnt++;
2401 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2402 dst_var->type = obj->btf_type_map[src_var->type];
2403 dst_var->size = src_var->size;
2404 dst_var->offset = src_sec->dst_off + src_var->offset;
2407 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2414 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2418 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2421 ext_data->recs = tmp;
2423 tmp += ext_data->rec_cnt * ext_data->rec_sz;
2424 memcpy(tmp, src_rec, ext_data->rec_sz);
2426 ext_data->rec_cnt++;
2431 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2433 const struct btf_ext_info_sec *ext_sec;
2434 const char *sec_name, *s;
2435 struct src_sec *src_sec;
2436 struct dst_sec *dst_sec;
2437 int rec_sz, str_off, i;
2442 rec_sz = obj->btf_ext->func_info.rec_size;
2443 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2444 struct bpf_func_info_min *src_rec, *dst_rec;
2446 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2447 src_sec = find_src_sec_by_name(obj, sec_name);
2449 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2452 dst_sec = &linker->secs[src_sec->dst_id];
2454 if (dst_sec->func_info.rec_sz == 0)
2455 dst_sec->func_info.rec_sz = rec_sz;
2456 if (dst_sec->func_info.rec_sz != rec_sz) {
2457 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2461 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2462 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2466 dst_rec->insn_off += src_sec->dst_off;
2467 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2471 rec_sz = obj->btf_ext->line_info.rec_size;
2472 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2473 struct bpf_line_info_min *src_rec, *dst_rec;
2475 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2476 src_sec = find_src_sec_by_name(obj, sec_name);
2478 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2481 dst_sec = &linker->secs[src_sec->dst_id];
2483 if (dst_sec->line_info.rec_sz == 0)
2484 dst_sec->line_info.rec_sz = rec_sz;
2485 if (dst_sec->line_info.rec_sz != rec_sz) {
2486 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2490 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2491 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2495 dst_rec->insn_off += src_sec->dst_off;
2497 s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2498 str_off = btf__add_str(linker->btf, s);
2501 dst_rec->file_name_off = str_off;
2503 s = btf__str_by_offset(obj->btf, src_rec->line_off);
2504 str_off = btf__add_str(linker->btf, s);
2507 dst_rec->line_off = str_off;
2509 /* dst_rec->line_col is fine */
2513 rec_sz = obj->btf_ext->core_relo_info.rec_size;
2514 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2515 struct bpf_core_relo *src_rec, *dst_rec;
2517 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2518 src_sec = find_src_sec_by_name(obj, sec_name);
2520 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2523 dst_sec = &linker->secs[src_sec->dst_id];
2525 if (dst_sec->core_relo_info.rec_sz == 0)
2526 dst_sec->core_relo_info.rec_sz = rec_sz;
2527 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2528 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2532 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2533 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2537 dst_rec->insn_off += src_sec->dst_off;
2538 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2540 s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2541 str_off = btf__add_str(linker->btf, s);
2544 dst_rec->access_str_off = str_off;
2546 /* dst_rec->kind is fine */
2553 int bpf_linker__finalize(struct bpf_linker *linker)
2555 struct dst_sec *sec;
2561 return libbpf_err(-EINVAL);
2563 err = finalize_btf(linker);
2565 return libbpf_err(err);
2567 /* Finalize strings */
2568 strs_sz = strset__data_size(linker->strtab_strs);
2569 strs = strset__data(linker->strtab_strs);
2571 sec = &linker->secs[linker->strtab_sec_idx];
2572 sec->data->d_align = 1;
2573 sec->data->d_off = 0LL;
2574 sec->data->d_buf = (void *)strs;
2575 sec->data->d_type = ELF_T_BYTE;
2576 sec->data->d_size = strs_sz;
2577 sec->shdr->sh_size = strs_sz;
2579 for (i = 1; i < linker->sec_cnt; i++) {
2580 sec = &linker->secs[i];
2582 /* STRTAB is handled specially above */
2583 if (sec->sec_idx == linker->strtab_sec_idx)
2586 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2590 sec->data->d_buf = sec->raw_data;
2593 /* Finalize ELF layout */
2594 if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2596 pr_warn_elf("failed to finalize ELF layout");
2597 return libbpf_err(err);
2600 /* Write out final ELF contents */
2601 if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2603 pr_warn_elf("failed to write ELF contents");
2604 return libbpf_err(err);
2607 elf_end(linker->elf);
2616 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2617 size_t align, const void *raw_data, size_t raw_sz)
2624 name_off = strset__add_str(linker->strtab_strs, sec_name);
2628 scn = elf_newscn(linker->elf);
2631 data = elf_newdata(scn);
2634 shdr = elf64_getshdr(scn);
2638 shdr->sh_name = name_off;
2639 shdr->sh_type = SHT_PROGBITS;
2641 shdr->sh_size = raw_sz;
2644 shdr->sh_addralign = align;
2645 shdr->sh_entsize = 0;
2647 data->d_type = ELF_T_BYTE;
2648 data->d_size = raw_sz;
2649 data->d_buf = (void *)raw_data;
2650 data->d_align = align;
2656 static int finalize_btf(struct bpf_linker *linker)
2658 struct btf *btf = linker->btf;
2659 const void *raw_data;
2663 /* bail out if no BTF data was produced */
2664 if (btf__get_nr_types(linker->btf) == 0)
2667 for (i = 1; i < linker->sec_cnt; i++) {
2668 struct dst_sec *sec = &linker->secs[i];
2673 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2675 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2680 for (j = 0; j < sec->sec_var_cnt; j++) {
2681 struct btf_var_secinfo *vi = &sec->sec_vars[j];
2683 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2688 err = finalize_btf_ext(linker);
2690 pr_warn(".BTF.ext generation failed: %d\n", err);
2694 err = btf__dedup(linker->btf, linker->btf_ext, NULL);
2696 pr_warn("BTF dedup failed: %d\n", err);
2700 /* Emit .BTF section */
2701 raw_data = btf__get_raw_data(linker->btf, &raw_sz);
2705 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2707 pr_warn("failed to write out .BTF ELF section: %d\n", err);
2711 /* Emit .BTF.ext section */
2712 if (linker->btf_ext) {
2713 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
2717 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2719 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2727 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2728 const char *sec_name, struct btf_ext_sec_data *sec_data)
2730 struct btf_ext_info_sec *sec_info;
2735 if (!sec_data->rec_cnt)
2738 str_off = btf__add_str(linker->btf, sec_name);
2743 sec_info->sec_name_off = str_off;
2744 sec_info->num_info = sec_data->rec_cnt;
2745 cur += sizeof(struct btf_ext_info_sec);
2747 sz = sec_data->rec_cnt * sec_data->rec_sz;
2748 memcpy(cur, sec_data->recs, sz);
2751 return cur - output;
2754 static int finalize_btf_ext(struct bpf_linker *linker)
2756 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2757 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2758 struct btf_ext_header *hdr;
2762 /* validate that all sections have the same .BTF.ext record sizes
2763 * and calculate total data size for each type of data (func info,
2764 * line info, core relos)
2766 for (i = 1; i < linker->sec_cnt; i++) {
2767 struct dst_sec *sec = &linker->secs[i];
2769 if (sec->func_info.rec_cnt) {
2770 if (func_rec_sz == 0)
2771 func_rec_sz = sec->func_info.rec_sz;
2772 if (func_rec_sz != sec->func_info.rec_sz) {
2773 pr_warn("mismatch in func_info record size %zu != %u\n",
2774 func_rec_sz, sec->func_info.rec_sz);
2778 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2780 if (sec->line_info.rec_cnt) {
2781 if (line_rec_sz == 0)
2782 line_rec_sz = sec->line_info.rec_sz;
2783 if (line_rec_sz != sec->line_info.rec_sz) {
2784 pr_warn("mismatch in line_info record size %zu != %u\n",
2785 line_rec_sz, sec->line_info.rec_sz);
2789 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2791 if (sec->core_relo_info.rec_cnt) {
2792 if (core_relo_rec_sz == 0)
2793 core_relo_rec_sz = sec->core_relo_info.rec_sz;
2794 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2795 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2796 core_relo_rec_sz, sec->core_relo_info.rec_sz);
2800 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2804 if (!funcs_sz && !lines_sz && !core_relos_sz)
2807 total_sz += sizeof(struct btf_ext_header);
2809 funcs_sz += sizeof(__u32); /* record size prefix */
2810 total_sz += funcs_sz;
2813 lines_sz += sizeof(__u32); /* record size prefix */
2814 total_sz += lines_sz;
2816 if (core_relos_sz) {
2817 core_relos_sz += sizeof(__u32); /* record size prefix */
2818 total_sz += core_relos_sz;
2821 cur = data = calloc(1, total_sz);
2826 hdr->magic = BTF_MAGIC;
2827 hdr->version = BTF_VERSION;
2829 hdr->hdr_len = sizeof(struct btf_ext_header);
2830 cur += sizeof(struct btf_ext_header);
2832 /* All offsets are in bytes relative to the end of this header */
2833 hdr->func_info_off = 0;
2834 hdr->func_info_len = funcs_sz;
2835 hdr->line_info_off = funcs_sz;
2836 hdr->line_info_len = lines_sz;
2837 hdr->core_relo_off = funcs_sz + lines_sz;
2838 hdr->core_relo_len = core_relos_sz;
2841 *(__u32 *)cur = func_rec_sz;
2842 cur += sizeof(__u32);
2844 for (i = 1; i < linker->sec_cnt; i++) {
2845 struct dst_sec *sec = &linker->secs[i];
2847 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2858 *(__u32 *)cur = line_rec_sz;
2859 cur += sizeof(__u32);
2861 for (i = 1; i < linker->sec_cnt; i++) {
2862 struct dst_sec *sec = &linker->secs[i];
2864 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2874 if (core_relos_sz) {
2875 *(__u32 *)cur = core_relo_rec_sz;
2876 cur += sizeof(__u32);
2878 for (i = 1; i < linker->sec_cnt; i++) {
2879 struct dst_sec *sec = &linker->secs[i];
2881 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2891 linker->btf_ext = btf_ext__new(data, total_sz);
2892 err = libbpf_get_error(linker->btf_ext);
2894 linker->btf_ext = NULL;
2895 pr_warn("failed to parse final .BTF.ext data: %d\n", err);