tools/lib/bpf/linker.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2 /*
   3  * BPF static linker
   4  *
   5  * Copyright (c) 2021 Facebook
   6  */
   7 #include <stdbool.h>
   8 #include <stddef.h>
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <unistd.h>
  13 #include <errno.h>
  14 #include <linux/err.h>
  15 #include <linux/btf.h>
  16 #include <elf.h>
  17 #include <libelf.h>
  18 #include <gelf.h>
  19 #include <fcntl.h>
  20 #include "libbpf.h"
  21 #include "btf.h"
  22 #include "libbpf_internal.h"
  23 #include "strset.h"
  24
  25 #define BTF_EXTERN_SEC ".extern"
  26
  27 struct src_sec {
  28         const char *sec_name;
  29         /* positional (not necessarily ELF) index in an array of sections */
  30         int id;
  31         /* positional (not necessarily ELF) index of a matching section in a final object file */
  32         int dst_id;
  33         /* section data offset in a matching output section */
  34         int dst_off;
  35         /* whether section is omitted from the final ELF file */
  36         bool skipped;
  37         /* whether section is an ephemeral section, not mapped to an ELF section */
  38         bool ephemeral;
  39
  40         /* ELF info */
  41         size_t sec_idx;
  42         Elf_Scn *scn;
  43         Elf64_Shdr *shdr;
  44         Elf_Data *data;
  45
  46         /* corresponding BTF DATASEC type ID */
  47         int sec_type_id;
  48 };
  49
  50 struct src_obj {
  51         const char *filename;
  52         int fd;
  53         Elf *elf;
  54         /* Section header strings section index */
  55         size_t shstrs_sec_idx;
  56         /* SYMTAB section index */
  57         size_t symtab_sec_idx;
  58
  59         struct btf *btf;
  60         struct btf_ext *btf_ext;
  61
  62         /* List of sections (including ephemeral). Slot zero is unused. */
  63         struct src_sec *secs;
  64         int sec_cnt;
  65
  66         /* mapping of symbol indices from src to dst ELF */
  67         int *sym_map;
  68         /* mapping from the src BTF type IDs to dst ones */
  69         int *btf_type_map;
  70 };
  71
  72 /* single .BTF.ext data section */
  73 struct btf_ext_sec_data {
  74         size_t rec_cnt;
  75         __u32 rec_sz;
  76         void *recs;
  77 };
  78
  79 struct glob_sym {
  80         /* ELF symbol index */
  81         int sym_idx;
  82         /* associated section id for .ksyms, .kconfig, etc, but not .extern */
  83         int sec_id;
  84         /* extern name offset in STRTAB */
  85         int name_off;
  86         /* optional associated BTF type ID */
  87         int btf_id;
  88         /* BTF type ID to which VAR/FUNC type is pointing to; used for
  89          * rewriting types when extern VAR/FUNC is resolved to a concrete
  90          * definition
  91          */
  92         int underlying_btf_id;
  93         /* sec_var index in the corresponding dst_sec, if exists */
  94         int var_idx;
  95
  96         /* extern or resolved/global symbol */
  97         bool is_extern;
  98         /* weak or strong symbol, never goes back from strong to weak */
  99         bool is_weak;
 100 };
 101
 102 struct dst_sec {
 103         char *sec_name;
 104         /* positional (not necessarily ELF) index in an array of sections */
 105         int id;
 106
 107         bool ephemeral;
 108
 109         /* ELF info */
 110         size_t sec_idx;
 111         Elf_Scn *scn;
 112         Elf64_Shdr *shdr;
 113         Elf_Data *data;
 114
 115         /* final output section size */
 116         int sec_sz;
 117         /* final output contents of the section */
 118         void *raw_data;
 119
 120         /* corresponding STT_SECTION symbol index in SYMTAB */
 121         int sec_sym_idx;
 122
 123         /* section's DATASEC variable info, emitted on BTF finalization */
 124         bool has_btf;
 125         int sec_var_cnt;
 126         struct btf_var_secinfo *sec_vars;
 127
 128         /* section's .BTF.ext data */
 129         struct btf_ext_sec_data func_info;
 130         struct btf_ext_sec_data line_info;
 131         struct btf_ext_sec_data core_relo_info;
 132 };
 133
 134 struct bpf_linker {
 135         char *filename;
 136         int fd;
 137         Elf *elf;
 138         Elf64_Ehdr *elf_hdr;
 139
 140         /* Output sections metadata */
 141         struct dst_sec *secs;
 142         int sec_cnt;
 143
 144         struct strset *strtab_strs; /* STRTAB unique strings */
 145         size_t strtab_sec_idx; /* STRTAB section index */
 146         size_t symtab_sec_idx; /* SYMTAB section index */
 147
 148         struct btf *btf;
 149         struct btf_ext *btf_ext;
 150
 151         /* global (including extern) ELF symbols */
 152         int glob_sym_cnt;
 153         struct glob_sym *glob_syms;
 154 };
 155
 156 #define pr_warn_elf(fmt, ...)                                                                   \
 157         libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
 158
 159 static int init_output_elf(struct bpf_linker *linker, const char *file);
 160
 161 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
 162                                 const struct bpf_linker_file_opts *opts,
 163                                 struct src_obj *obj);
 164 static int linker_sanity_check_elf(struct src_obj *obj);
 165 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
 166 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
 167 static int linker_sanity_check_btf(struct src_obj *obj);
 168 static int linker_sanity_check_btf_ext(struct src_obj *obj);
 169 static int linker_fixup_btf(struct src_obj *obj);
 170 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
 171 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
 172 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
 173                                  Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
 174 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
 175 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
 176 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
 177
 178 static int finalize_btf(struct bpf_linker *linker);
 179 static int finalize_btf_ext(struct bpf_linker *linker);
 180
 181 void bpf_linker__free(struct bpf_linker *linker)
 182 {
 183         int i;
 184
 185         if (!linker)
 186                 return;
 187
 188         free(linker->filename);
 189
 190         if (linker->elf)
 191                 elf_end(linker->elf);
 192
 193         if (linker->fd >= 0)
 194                 close(linker->fd);
 195
 196         strset__free(linker->strtab_strs);
 197
 198         btf__free(linker->btf);
 199         btf_ext__free(linker->btf_ext);
 200
 201         for (i = 1; i < linker->sec_cnt; i++) {
 202                 struct dst_sec *sec = &linker->secs[i];
 203
 204                 free(sec->sec_name);
 205                 free(sec->raw_data);
 206                 free(sec->sec_vars);
 207
 208                 free(sec->func_info.recs);
 209                 free(sec->line_info.recs);
 210                 free(sec->core_relo_info.recs);
 211         }
 212         free(linker->secs);
 213
 214         free(linker->glob_syms);
 215         free(linker);
 216 }
 217
 218 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
 219 {
 220         struct bpf_linker *linker;
 221         int err;
 222
 223         if (!OPTS_VALID(opts, bpf_linker_opts))
 224                 return errno = EINVAL, NULL;
 225
 226         if (elf_version(EV_CURRENT) == EV_NONE) {
 227                 pr_warn_elf("libelf initialization failed");
 228                 return errno = EINVAL, NULL;
 229         }
 230
 231         linker = calloc(1, sizeof(*linker));
 232         if (!linker)
 233                 return errno = ENOMEM, NULL;
 234
 235         linker->fd = -1;
 236
 237         err = init_output_elf(linker, filename);
 238         if (err)
 239                 goto err_out;
 240
 241         return linker;
 242
 243 err_out:
 244         bpf_linker__free(linker);
 245         return errno = -err, NULL;
 246 }
 247
 248 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
 249 {
 250         struct dst_sec *secs = linker->secs, *sec;
 251         size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
 252
 253         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 254         if (!secs)
 255                 return NULL;
 256
 257         /* zero out newly allocated memory */
 258         memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
 259
 260         linker->secs = secs;
 261         linker->sec_cnt = new_cnt;
 262
 263         sec = &linker->secs[new_cnt - 1];
 264         sec->id = new_cnt - 1;
 265         sec->sec_name = strdup(sec_name);
 266         if (!sec->sec_name)
 267                 return NULL;
 268
 269         return sec;
 270 }
 271
 272 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
 273 {
 274         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
 275         Elf64_Sym *syms, *sym;
 276         size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
 277
 278         syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
 279         if (!syms)
 280                 return NULL;
 281
 282         sym = &syms[sym_cnt];
 283         memset(sym, 0, sizeof(*sym));
 284
 285         symtab->raw_data = syms;
 286         symtab->sec_sz += sizeof(*sym);
 287         symtab->shdr->sh_size += sizeof(*sym);
 288         symtab->data->d_size += sizeof(*sym);
 289
 290         if (sym_idx)
 291                 *sym_idx = sym_cnt;
 292
 293         return sym;
 294 }
 295
 296 static int init_output_elf(struct bpf_linker *linker, const char *file)
 297 {
 298         int err, str_off;
 299         Elf64_Sym *init_sym;
 300         struct dst_sec *sec;
 301
 302         linker->filename = strdup(file);
 303         if (!linker->filename)
 304                 return -ENOMEM;
 305
 306         linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
 307         if (linker->fd < 0) {
 308                 err = -errno;
 309                 pr_warn("failed to create '%s': %d\n", file, err);
 310                 return err;
 311         }
 312
 313         linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
 314         if (!linker->elf) {
 315                 pr_warn_elf("failed to create ELF object");
 316                 return -EINVAL;
 317         }
 318
 319         /* ELF header */
 320         linker->elf_hdr = elf64_newehdr(linker->elf);
 321         if (!linker->elf_hdr) {
 322                 pr_warn_elf("failed to create ELF header");
 323                 return -EINVAL;
 324         }
 325
 326         linker->elf_hdr->e_machine = EM_BPF;
 327         linker->elf_hdr->e_type = ET_REL;
 328 #if __BYTE_ORDER == __LITTLE_ENDIAN
 329         linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
 330 #elif __BYTE_ORDER == __BIG_ENDIAN
 331         linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
 332 #else
 333 #error "Unknown __BYTE_ORDER"
 334 #endif
 335
 336         /* STRTAB */
 337         /* initialize strset with an empty string to conform to ELF */
 338         linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
 339         if (libbpf_get_error(linker->strtab_strs))
 340                 return libbpf_get_error(linker->strtab_strs);
 341
 342         sec = add_dst_sec(linker, ".strtab");
 343         if (!sec)
 344                 return -ENOMEM;
 345
 346         sec->scn = elf_newscn(linker->elf);
 347         if (!sec->scn) {
 348                 pr_warn_elf("failed to create STRTAB section");
 349                 return -EINVAL;
 350         }
 351
 352         sec->shdr = elf64_getshdr(sec->scn);
 353         if (!sec->shdr)
 354                 return -EINVAL;
 355
 356         sec->data = elf_newdata(sec->scn);
 357         if (!sec->data) {
 358                 pr_warn_elf("failed to create STRTAB data");
 359                 return -EINVAL;
 360         }
 361
 362         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 363         if (str_off < 0)
 364                 return str_off;
 365
 366         sec->sec_idx = elf_ndxscn(sec->scn);
 367         linker->elf_hdr->e_shstrndx = sec->sec_idx;
 368         linker->strtab_sec_idx = sec->sec_idx;
 369
 370         sec->shdr->sh_name = str_off;
 371         sec->shdr->sh_type = SHT_STRTAB;
 372         sec->shdr->sh_flags = SHF_STRINGS;
 373         sec->shdr->sh_offset = 0;
 374         sec->shdr->sh_link = 0;
 375         sec->shdr->sh_info = 0;
 376         sec->shdr->sh_addralign = 1;
 377         sec->shdr->sh_size = sec->sec_sz = 0;
 378         sec->shdr->sh_entsize = 0;
 379
 380         /* SYMTAB */
 381         sec = add_dst_sec(linker, ".symtab");
 382         if (!sec)
 383                 return -ENOMEM;
 384
 385         sec->scn = elf_newscn(linker->elf);
 386         if (!sec->scn) {
 387                 pr_warn_elf("failed to create SYMTAB section");
 388                 return -EINVAL;
 389         }
 390
 391         sec->shdr = elf64_getshdr(sec->scn);
 392         if (!sec->shdr)
 393                 return -EINVAL;
 394
 395         sec->data = elf_newdata(sec->scn);
 396         if (!sec->data) {
 397                 pr_warn_elf("failed to create SYMTAB data");
 398                 return -EINVAL;
 399         }
 400
 401         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 402         if (str_off < 0)
 403                 return str_off;
 404
 405         sec->sec_idx = elf_ndxscn(sec->scn);
 406         linker->symtab_sec_idx = sec->sec_idx;
 407
 408         sec->shdr->sh_name = str_off;
 409         sec->shdr->sh_type = SHT_SYMTAB;
 410         sec->shdr->sh_flags = 0;
 411         sec->shdr->sh_offset = 0;
 412         sec->shdr->sh_link = linker->strtab_sec_idx;
 413         /* sh_info should be one greater than the index of the last local
 414          * symbol (i.e., binding is STB_LOCAL). But why and who cares?
 415          */
 416         sec->shdr->sh_info = 0;
 417         sec->shdr->sh_addralign = 8;
 418         sec->shdr->sh_entsize = sizeof(Elf64_Sym);
 419
 420         /* .BTF */
 421         linker->btf = btf__new_empty();
 422         err = libbpf_get_error(linker->btf);
 423         if (err)
 424                 return err;
 425
 426         /* add the special all-zero symbol */
 427         init_sym = add_new_sym(linker, NULL);
 428         if (!init_sym)
 429                 return -EINVAL;
 430
 431         init_sym->st_name = 0;
 432         init_sym->st_info = 0;
 433         init_sym->st_other = 0;
 434         init_sym->st_shndx = SHN_UNDEF;
 435         init_sym->st_value = 0;
 436         init_sym->st_size = 0;
 437
 438         return 0;
 439 }
 440
 441 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
 442                          const struct bpf_linker_file_opts *opts)
 443 {
 444         struct src_obj obj = {};
 445         int err = 0;
 446
 447         if (!OPTS_VALID(opts, bpf_linker_file_opts))
 448                 return libbpf_err(-EINVAL);
 449
 450         if (!linker->elf)
 451                 return libbpf_err(-EINVAL);
 452
 453         err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
 454         err = err ?: linker_append_sec_data(linker, &obj);
 455         err = err ?: linker_append_elf_syms(linker, &obj);
 456         err = err ?: linker_append_elf_relos(linker, &obj);
 457         err = err ?: linker_append_btf(linker, &obj);
 458         err = err ?: linker_append_btf_ext(linker, &obj);
 459
 460         /* free up src_obj resources */
 461         free(obj.btf_type_map);
 462         btf__free(obj.btf);
 463         btf_ext__free(obj.btf_ext);
 464         free(obj.secs);
 465         free(obj.sym_map);
 466         if (obj.elf)
 467                 elf_end(obj.elf);
 468         if (obj.fd >= 0)
 469                 close(obj.fd);
 470
 471         return libbpf_err(err);
 472 }
 473
 474 static bool is_dwarf_sec_name(const char *name)
 475 {
 476         /* approximation, but the actual list is too long */
 477         return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
 478 }
 479
 480 static bool is_ignored_sec(struct src_sec *sec)
 481 {
 482         Elf64_Shdr *shdr = sec->shdr;
 483         const char *name = sec->sec_name;
 484
 485         /* no special handling of .strtab */
 486         if (shdr->sh_type == SHT_STRTAB)
 487                 return true;
 488
 489         /* ignore .llvm_addrsig section as well */
 490         if (shdr->sh_type == SHT_LLVM_ADDRSIG)
 491                 return true;
 492
 493         /* no subprograms will lead to an empty .text section, ignore it */
 494         if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
 495             strcmp(sec->sec_name, ".text") == 0)
 496                 return true;
 497
 498         /* DWARF sections */
 499         if (is_dwarf_sec_name(sec->sec_name))
 500                 return true;
 501
 502         if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
 503                 name += sizeof(".rel") - 1;
 504                 /* DWARF section relocations */
 505                 if (is_dwarf_sec_name(name))
 506                         return true;
 507
 508                 /* .BTF and .BTF.ext don't need relocations */
 509                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
 510                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
 511                         return true;
 512         }
 513
 514         return false;
 515 }
 516
 517 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
 518 {
 519         struct src_sec *secs = obj->secs, *sec;
 520         size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
 521
 522         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 523         if (!secs)
 524                 return NULL;
 525
 526         /* zero out newly allocated memory */
 527         memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
 528
 529         obj->secs = secs;
 530         obj->sec_cnt = new_cnt;
 531
 532         sec = &obj->secs[new_cnt - 1];
 533         sec->id = new_cnt - 1;
 534         sec->sec_name = sec_name;
 535
 536         return sec;
 537 }
 538
 539 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
 540                                 const struct bpf_linker_file_opts *opts,
 541                                 struct src_obj *obj)
 542 {
 543 #if __BYTE_ORDER == __LITTLE_ENDIAN
 544         const int host_endianness = ELFDATA2LSB;
 545 #elif __BYTE_ORDER == __BIG_ENDIAN
 546         const int host_endianness = ELFDATA2MSB;
 547 #else
 548 #error "Unknown __BYTE_ORDER"
 549 #endif
 550         int err = 0;
 551         Elf_Scn *scn;
 552         Elf_Data *data;
 553         Elf64_Ehdr *ehdr;
 554         Elf64_Shdr *shdr;
 555         struct src_sec *sec;
 556
 557         pr_debug("linker: adding object file '%s'...\n", filename);
 558
 559         obj->filename = filename;
 560
 561         obj->fd = open(filename, O_RDONLY);
 562         if (obj->fd < 0) {
 563                 err = -errno;
 564                 pr_warn("failed to open file '%s': %d\n", filename, err);
 565                 return err;
 566         }
 567         obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
 568         if (!obj->elf) {
 569                 err = -errno;
 570                 pr_warn_elf("failed to parse ELF file '%s'", filename);
 571                 return err;
 572         }
 573
 574         /* Sanity check ELF file high-level properties */
 575         ehdr = elf64_getehdr(obj->elf);
 576         if (!ehdr) {
 577                 err = -errno;
 578                 pr_warn_elf("failed to get ELF header for %s", filename);
 579                 return err;
 580         }
 581         if (ehdr->e_ident[EI_DATA] != host_endianness) {
 582                 err = -EOPNOTSUPP;
 583                 pr_warn_elf("unsupported byte order of ELF file %s", filename);
 584                 return err;
 585         }
 586         if (ehdr->e_type != ET_REL
 587             || ehdr->e_machine != EM_BPF
 588             || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
 589                 err = -EOPNOTSUPP;
 590                 pr_warn_elf("unsupported kind of ELF file %s", filename);
 591                 return err;
 592         }
 593
 594         if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
 595                 err = -errno;
 596                 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
 597                 return err;
 598         }
 599
 600         scn = NULL;
 601         while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
 602                 size_t sec_idx = elf_ndxscn(scn);
 603                 const char *sec_name;
 604
 605                 shdr = elf64_getshdr(scn);
 606                 if (!shdr) {
 607                         err = -errno;
 608                         pr_warn_elf("failed to get section #%zu header for %s",
 609                                     sec_idx, filename);
 610                         return err;
 611                 }
 612
 613                 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
 614                 if (!sec_name) {
 615                         err = -errno;
 616                         pr_warn_elf("failed to get section #%zu name for %s",
 617                                     sec_idx, filename);
 618                         return err;
 619                 }
 620
 621                 data = elf_getdata(scn, 0);
 622                 if (!data) {
 623                         err = -errno;
 624                         pr_warn_elf("failed to get section #%zu (%s) data from %s",
 625                                     sec_idx, sec_name, filename);
 626                         return err;
 627                 }
 628
 629                 sec = add_src_sec(obj, sec_name);
 630                 if (!sec)
 631                         return -ENOMEM;
 632
 633                 sec->scn = scn;
 634                 sec->shdr = shdr;
 635                 sec->data = data;
 636                 sec->sec_idx = elf_ndxscn(scn);
 637
 638                 if (is_ignored_sec(sec)) {
 639                         sec->skipped = true;
 640                         continue;
 641                 }
 642
 643                 switch (shdr->sh_type) {
 644                 case SHT_SYMTAB:
 645                         if (obj->symtab_sec_idx) {
 646                                 err = -EOPNOTSUPP;
 647                                 pr_warn("multiple SYMTAB sections found, not supported\n");
 648                                 return err;
 649                         }
 650                         obj->symtab_sec_idx = sec_idx;
 651                         break;
 652                 case SHT_STRTAB:
 653                         /* we'll construct our own string table */
 654                         break;
 655                 case SHT_PROGBITS:
 656                         if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
 657                                 obj->btf = btf__new(data->d_buf, shdr->sh_size);
 658                                 err = libbpf_get_error(obj->btf);
 659                                 if (err) {
 660                                         pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
 661                                         return err;
 662                                 }
 663                                 sec->skipped = true;
 664                                 continue;
 665                         }
 666                         if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
 667                                 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
 668                                 err = libbpf_get_error(obj->btf_ext);
 669                                 if (err) {
 670                                         pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
 671                                         return err;
 672                                 }
 673                                 sec->skipped = true;
 674                                 continue;
 675                         }
 676
 677                         /* data & code */
 678                         break;
 679                 case SHT_NOBITS:
 680                         /* BSS */
 681                         break;
 682                 case SHT_REL:
 683                         /* relocations */
 684                         break;
 685                 default:
 686                         pr_warn("unrecognized section #%zu (%s) in %s\n",
 687                                 sec_idx, sec_name, filename);
 688                         err = -EINVAL;
 689                         return err;
 690                 }
 691         }
 692
 693         err = err ?: linker_sanity_check_elf(obj);
 694         err = err ?: linker_sanity_check_btf(obj);
 695         err = err ?: linker_sanity_check_btf_ext(obj);
 696         err = err ?: linker_fixup_btf(obj);
 697
 698         return err;
 699 }
 700
 701 static bool is_pow_of_2(size_t x)
 702 {
 703         return x && (x & (x - 1)) == 0;
 704 }
 705
 706 static int linker_sanity_check_elf(struct src_obj *obj)
 707 {
 708         struct src_sec *sec;
 709         int i, err;
 710
 711         if (!obj->symtab_sec_idx) {
 712                 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
 713                 return -EINVAL;
 714         }
 715         if (!obj->shstrs_sec_idx) {
 716                 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
 717                 return -EINVAL;
 718         }
 719
 720         for (i = 1; i < obj->sec_cnt; i++) {
 721                 sec = &obj->secs[i];
 722
 723                 if (sec->sec_name[0] == '\0') {
 724                         pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
 725                         return -EINVAL;
 726                 }
 727
 728                 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign))
 729                         return -EINVAL;
 730                 if (sec->shdr->sh_addralign != sec->data->d_align)
 731                         return -EINVAL;
 732
 733                 if (sec->shdr->sh_size != sec->data->d_size)
 734                         return -EINVAL;
 735
 736                 switch (sec->shdr->sh_type) {
 737                 case SHT_SYMTAB:
 738                         err = linker_sanity_check_elf_symtab(obj, sec);
 739                         if (err)
 740                                 return err;
 741                         break;
 742                 case SHT_STRTAB:
 743                         break;
 744                 case SHT_PROGBITS:
 745                         if (sec->shdr->sh_flags & SHF_EXECINSTR) {
 746                                 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0)
 747                                         return -EINVAL;
 748                         }
 749                         break;
 750                 case SHT_NOBITS:
 751                         break;
 752                 case SHT_REL:
 753                         err = linker_sanity_check_elf_relos(obj, sec);
 754                         if (err)
 755                                 return err;
 756                         break;
 757                 case SHT_LLVM_ADDRSIG:
 758                         break;
 759                 default:
 760                         pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
 761                                 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
 762                         return -EINVAL;
 763                 }
 764         }
 765
 766         return 0;
 767 }
 768
 769 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
 770 {
 771         struct src_sec *link_sec;
 772         Elf64_Sym *sym;
 773         int i, n;
 774
 775         if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
 776                 return -EINVAL;
 777         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 778                 return -EINVAL;
 779
 780         if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
 781                 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
 782                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 783                 return -EINVAL;
 784         }
 785         link_sec = &obj->secs[sec->shdr->sh_link];
 786         if (link_sec->shdr->sh_type != SHT_STRTAB) {
 787                 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
 788                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 789                 return -EINVAL;
 790         }
 791
 792         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 793         sym = sec->data->d_buf;
 794         for (i = 0; i < n; i++, sym++) {
 795                 int sym_type = ELF64_ST_TYPE(sym->st_info);
 796                 int sym_bind = ELF64_ST_BIND(sym->st_info);
 797                 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
 798
 799                 if (i == 0) {
 800                         if (sym->st_name != 0 || sym->st_info != 0
 801                             || sym->st_other != 0 || sym->st_shndx != 0
 802                             || sym->st_value != 0 || sym->st_size != 0) {
 803                                 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
 804                                 return -EINVAL;
 805                         }
 806                         continue;
 807                 }
 808                 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
 809                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
 810                                 i, sec->sec_idx, sym_bind);
 811                         return -EINVAL;
 812                 }
 813                 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
 814                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
 815                                 i, sec->sec_idx, sym_vis);
 816                         return -EINVAL;
 817                 }
 818                 if (sym->st_shndx == 0) {
 819                         if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
 820                             || sym->st_value != 0 || sym->st_size != 0) {
 821                                 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
 822                                         i, obj->filename);
 823
 824                                 return -EINVAL;
 825                         }
 826                         continue;
 827                 }
 828                 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
 829                         pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
 830                                 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
 831                         return -EINVAL;
 832                 }
 833                 if (sym_type == STT_SECTION) {
 834                         if (sym->st_value != 0)
 835                                 return -EINVAL;
 836                         continue;
 837                 }
 838         }
 839
 840         return 0;
 841 }
 842
 843 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
 844 {
 845         struct src_sec *link_sec, *sym_sec;
 846         Elf64_Rel *relo;
 847         int i, n;
 848
 849         if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
 850                 return -EINVAL;
 851         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 852                 return -EINVAL;
 853
 854         /* SHT_REL's sh_link should point to SYMTAB */
 855         if (sec->shdr->sh_link != obj->symtab_sec_idx) {
 856                 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
 857                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 858                 return -EINVAL;
 859         }
 860
 861         /* SHT_REL's sh_info points to relocated section */
 862         if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
 863                 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
 864                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 865                 return -EINVAL;
 866         }
 867         link_sec = &obj->secs[sec->shdr->sh_info];
 868
 869         /* .rel<secname> -> <secname> pattern is followed */
 870         if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
 871             || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
 872                 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
 873                         sec->sec_idx, obj->filename);
 874                 return -EINVAL;
 875         }
 876
 877         /* don't further validate relocations for ignored sections */
 878         if (link_sec->skipped)
 879                 return 0;
 880
 881         /* relocatable section is data or instructions */
 882         if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
 883                 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
 884                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 885                 return -EINVAL;
 886         }
 887
 888         /* check sanity of each relocation */
 889         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 890         relo = sec->data->d_buf;
 891         sym_sec = &obj->secs[obj->symtab_sec_idx];
 892         for (i = 0; i < n; i++, relo++) {
 893                 size_t sym_idx = ELF64_R_SYM(relo->r_info);
 894                 size_t sym_type = ELF64_R_TYPE(relo->r_info);
 895
 896                 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
 897                     sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
 898                         pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
 899                                 i, sec->sec_idx, sym_type, obj->filename);
 900                         return -EINVAL;
 901                 }
 902
 903                 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
 904                         pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
 905                                 i, sec->sec_idx, sym_idx, obj->filename);
 906                         return -EINVAL;
 907                 }
 908
 909                 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
 910                         if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
 911                                 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
 912                                         i, sec->sec_idx, sym_idx, obj->filename);
 913                                 return -EINVAL;
 914                         }
 915                 }
 916         }
 917
 918         return 0;
 919 }
 920
 921 static int check_btf_type_id(__u32 *type_id, void *ctx)
 922 {
 923         struct btf *btf = ctx;
 924
 925         if (*type_id > btf__get_nr_types(btf))
 926                 return -EINVAL;
 927
 928         return 0;
 929 }
 930
 931 static int check_btf_str_off(__u32 *str_off, void *ctx)
 932 {
 933         struct btf *btf = ctx;
 934         const char *s;
 935
 936         s = btf__str_by_offset(btf, *str_off);
 937
 938         if (!s)
 939                 return -EINVAL;
 940
 941         return 0;
 942 }
 943
 944 static int linker_sanity_check_btf(struct src_obj *obj)
 945 {
 946         struct btf_type *t;
 947         int i, n, err = 0;
 948
 949         if (!obj->btf)
 950                 return 0;
 951
 952         n = btf__get_nr_types(obj->btf);
 953         for (i = 1; i <= n; i++) {
 954                 t = btf_type_by_id(obj->btf, i);
 955
 956                 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
 957                 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
 958                 if (err)
 959                         return err;
 960         }
 961
 962         return 0;
 963 }
 964
 965 static int linker_sanity_check_btf_ext(struct src_obj *obj)
 966 {
 967         int err = 0;
 968
 969         if (!obj->btf_ext)
 970                 return 0;
 971
 972         /* can't use .BTF.ext without .BTF */
 973         if (!obj->btf)
 974                 return -EINVAL;
 975
 976         err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
 977         err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
 978         if (err)
 979                 return err;
 980
 981         return 0;
 982 }
 983
 984 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
 985 {
 986         Elf_Scn *scn;
 987         Elf_Data *data;
 988         Elf64_Shdr *shdr;
 989         int name_off;
 990
 991         dst_sec->sec_sz = 0;
 992         dst_sec->sec_idx = 0;
 993         dst_sec->ephemeral = src_sec->ephemeral;
 994
 995         /* ephemeral sections are just thin section shells lacking most parts */
 996         if (src_sec->ephemeral)
 997                 return 0;
 998
 999         scn = elf_newscn(linker->elf);
1000         if (!scn)
1001                 return -ENOMEM;
1002         data = elf_newdata(scn);
1003         if (!data)
1004                 return -ENOMEM;
1005         shdr = elf64_getshdr(scn);
1006         if (!shdr)
1007                 return -ENOMEM;
1008
1009         dst_sec->scn = scn;
1010         dst_sec->shdr = shdr;
1011         dst_sec->data = data;
1012         dst_sec->sec_idx = elf_ndxscn(scn);
1013
1014         name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1015         if (name_off < 0)
1016                 return name_off;
1017
1018         shdr->sh_name = name_off;
1019         shdr->sh_type = src_sec->shdr->sh_type;
1020         shdr->sh_flags = src_sec->shdr->sh_flags;
1021         shdr->sh_size = 0;
1022         /* sh_link and sh_info have different meaning for different types of
1023          * sections, so we leave it up to the caller code to fill them in, if
1024          * necessary
1025          */
1026         shdr->sh_link = 0;
1027         shdr->sh_info = 0;
1028         shdr->sh_addralign = src_sec->shdr->sh_addralign;
1029         shdr->sh_entsize = src_sec->shdr->sh_entsize;
1030
1031         data->d_type = src_sec->data->d_type;
1032         data->d_size = 0;
1033         data->d_buf = NULL;
1034         data->d_align = src_sec->data->d_align;
1035         data->d_off = 0;
1036
1037         return 0;
1038 }
1039
1040 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1041 {
1042         struct dst_sec *sec;
1043         int i;
1044
1045         for (i = 1; i < linker->sec_cnt; i++) {
1046                 sec = &linker->secs[i];
1047
1048                 if (strcmp(sec->sec_name, sec_name) == 0)
1049                         return sec;
1050         }
1051
1052         return NULL;
1053 }
1054
1055 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1056 {
1057         if (dst->ephemeral || src->ephemeral)
1058                 return true;
1059
1060         if (dst->shdr->sh_type != src->shdr->sh_type) {
1061                 pr_warn("sec %s types mismatch\n", dst->sec_name);
1062                 return false;
1063         }
1064         if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1065                 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1066                 return false;
1067         }
1068         if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1069                 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1070                 return false;
1071         }
1072
1073         return true;
1074 }
1075
1076 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1077 {
1078         if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1079                 return false;
1080         if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1081                 return false;
1082         return true;
1083 }
1084
1085 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1086 {
1087         void *tmp;
1088         size_t dst_align, src_align;
1089         size_t dst_align_sz, dst_final_sz;
1090         int err;
1091
1092         /* Ephemeral source section doesn't contribute anything to ELF
1093          * section data.
1094          */
1095         if (src->ephemeral)
1096                 return 0;
1097
1098         /* Some sections (like .maps) can contain both externs (and thus be
1099          * ephemeral) and non-externs (map definitions). So it's possible that
1100          * it has to be "upgraded" from ephemeral to non-ephemeral when the
1101          * first non-ephemeral entity appears. In such case, we add ELF
1102          * section, data, etc.
1103          */
1104         if (dst->ephemeral) {
1105                 err = init_sec(linker, dst, src);
1106                 if (err)
1107                         return err;
1108         }
1109
1110         dst_align = dst->shdr->sh_addralign;
1111         src_align = src->shdr->sh_addralign;
1112         if (dst_align == 0)
1113                 dst_align = 1;
1114         if (dst_align < src_align)
1115                 dst_align = src_align;
1116
1117         dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1118
1119         /* no need to re-align final size */
1120         dst_final_sz = dst_align_sz + src->shdr->sh_size;
1121
1122         if (src->shdr->sh_type != SHT_NOBITS) {
1123                 tmp = realloc(dst->raw_data, dst_final_sz);
1124                 if (!tmp)
1125                         return -ENOMEM;
1126                 dst->raw_data = tmp;
1127
1128                 /* pad dst section, if it's alignment forced size increase */
1129                 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1130                 /* now copy src data at a properly aligned offset */
1131                 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1132         }
1133
1134         dst->sec_sz = dst_final_sz;
1135         dst->shdr->sh_size = dst_final_sz;
1136         dst->data->d_size = dst_final_sz;
1137
1138         dst->shdr->sh_addralign = dst_align;
1139         dst->data->d_align = dst_align;
1140
1141         src->dst_off = dst_align_sz;
1142
1143         return 0;
1144 }
1145
1146 static bool is_data_sec(struct src_sec *sec)
1147 {
1148         if (!sec || sec->skipped)
1149                 return false;
1150         /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1151         if (sec->ephemeral)
1152                 return true;
1153         return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1154 }
1155
1156 static bool is_relo_sec(struct src_sec *sec)
1157 {
1158         if (!sec || sec->skipped || sec->ephemeral)
1159                 return false;
1160         return sec->shdr->sh_type == SHT_REL;
1161 }
1162
1163 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1164 {
1165         int i, err;
1166
1167         for (i = 1; i < obj->sec_cnt; i++) {
1168                 struct src_sec *src_sec;
1169                 struct dst_sec *dst_sec;
1170
1171                 src_sec = &obj->secs[i];
1172                 if (!is_data_sec(src_sec))
1173                         continue;
1174
1175                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1176                 if (!dst_sec) {
1177                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
1178                         if (!dst_sec)
1179                                 return -ENOMEM;
1180                         err = init_sec(linker, dst_sec, src_sec);
1181                         if (err) {
1182                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1183                                 return err;
1184                         }
1185                 } else {
1186                         if (!secs_match(dst_sec, src_sec)) {
1187                                 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1188                                 return -1;
1189                         }
1190
1191                         /* "license" and "version" sections are deduped */
1192                         if (strcmp(src_sec->sec_name, "license") == 0
1193                             || strcmp(src_sec->sec_name, "version") == 0) {
1194                                 if (!sec_content_is_same(dst_sec, src_sec)) {
1195                                         pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1196                                         return -EINVAL;
1197                                 }
1198                                 src_sec->skipped = true;
1199                                 src_sec->dst_id = dst_sec->id;
1200                                 continue;
1201                         }
1202                 }
1203
1204                 /* record mapped section index */
1205                 src_sec->dst_id = dst_sec->id;
1206
1207                 err = extend_sec(linker, dst_sec, src_sec);
1208                 if (err)
1209                         return err;
1210         }
1211
1212         return 0;
1213 }
1214
1215 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1216 {
1217         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1218         Elf64_Sym *sym = symtab->data->d_buf;
1219         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1220         int str_sec_idx = symtab->shdr->sh_link;
1221         const char *sym_name;
1222
1223         obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1224         if (!obj->sym_map)
1225                 return -ENOMEM;
1226
1227         for (i = 0; i < n; i++, sym++) {
1228                 /* We already validated all-zero symbol #0 and we already
1229                  * appended it preventively to the final SYMTAB, so skip it.
1230                  */
1231                 if (i == 0)
1232                         continue;
1233
1234                 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1235                 if (!sym_name) {
1236                         pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1237                         return -EINVAL;
1238                 }
1239
1240                 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1241                 if (err)
1242                         return err;
1243         }
1244
1245         return 0;
1246 }
1247
1248 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1249 {
1250         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1251         Elf64_Sym *syms = symtab->raw_data;
1252
1253         return &syms[sym_idx];
1254 }
1255
1256 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1257 {
1258         struct glob_sym *glob_sym;
1259         const char *name;
1260         int i;
1261
1262         for (i = 0; i < linker->glob_sym_cnt; i++) {
1263                 glob_sym = &linker->glob_syms[i];
1264                 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1265
1266                 if (strcmp(name, sym_name) == 0)
1267                         return glob_sym;
1268         }
1269
1270         return NULL;
1271 }
1272
1273 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1274 {
1275         struct glob_sym *syms, *sym;
1276
1277         syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1278                                    sizeof(*linker->glob_syms));
1279         if (!syms)
1280                 return NULL;
1281
1282         sym = &syms[linker->glob_sym_cnt];
1283         memset(sym, 0, sizeof(*sym));
1284         sym->var_idx = -1;
1285
1286         linker->glob_syms = syms;
1287         linker->glob_sym_cnt++;
1288
1289         return sym;
1290 }
1291
1292 static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1293                                  const struct btf *btf1, __u32 id1,
1294                                  const struct btf *btf2, __u32 id2)
1295 {
1296         const struct btf_type *t1, *t2;
1297         bool is_static1, is_static2;
1298         const char *n1, *n2;
1299         int i, n;
1300
1301 recur:
1302         n1 = n2 = NULL;
1303         t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1304         t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1305
1306         /* check if only one side is FWD, otherwise handle with common logic */
1307         if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1308                 n1 = btf__str_by_offset(btf1, t1->name_off);
1309                 n2 = btf__str_by_offset(btf2, t2->name_off);
1310                 if (strcmp(n1, n2) != 0) {
1311                         pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1312                                 sym_name, n1, n2);
1313                         return false;
1314                 }
1315                 /* validate if FWD kind matches concrete kind */
1316                 if (btf_is_fwd(t1)) {
1317                         if (btf_kflag(t1) && btf_is_union(t2))
1318                                 return true;
1319                         if (!btf_kflag(t1) && btf_is_struct(t2))
1320                                 return true;
1321                         pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1322                                 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1323                 } else {
1324                         if (btf_kflag(t2) && btf_is_union(t1))
1325                                 return true;
1326                         if (!btf_kflag(t2) && btf_is_struct(t1))
1327                                 return true;
1328                         pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1329                                 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1330                 }
1331                 return false;
1332         }
1333
1334         if (btf_kind(t1) != btf_kind(t2)) {
1335                 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1336                         sym_name, btf_kind_str(t1), btf_kind_str(t2));
1337                 return false;
1338         }
1339
1340         switch (btf_kind(t1)) {
1341         case BTF_KIND_STRUCT:
1342         case BTF_KIND_UNION:
1343         case BTF_KIND_ENUM:
1344         case BTF_KIND_FWD:
1345         case BTF_KIND_FUNC:
1346         case BTF_KIND_VAR:
1347                 n1 = btf__str_by_offset(btf1, t1->name_off);
1348                 n2 = btf__str_by_offset(btf2, t2->name_off);
1349                 if (strcmp(n1, n2) != 0) {
1350                         pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1351                                 sym_name, btf_kind_str(t1), n1, n2);
1352                         return false;
1353                 }
1354                 break;
1355         default:
1356                 break;
1357         }
1358
1359         switch (btf_kind(t1)) {
1360         case BTF_KIND_UNKN: /* void */
1361         case BTF_KIND_FWD:
1362                 return true;
1363         case BTF_KIND_INT:
1364         case BTF_KIND_FLOAT:
1365         case BTF_KIND_ENUM:
1366                 /* ignore encoding for int and enum values for enum */
1367                 if (t1->size != t2->size) {
1368                         pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1369                                 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1370                         return false;
1371                 }
1372                 return true;
1373         case BTF_KIND_PTR:
1374                 /* just validate overall shape of the referenced type, so no
1375                  * contents comparison for struct/union, and allowd fwd vs
1376                  * struct/union
1377                  */
1378                 exact = false;
1379                 id1 = t1->type;
1380                 id2 = t2->type;
1381                 goto recur;
1382         case BTF_KIND_ARRAY:
1383                 /* ignore index type and array size */
1384                 id1 = btf_array(t1)->type;
1385                 id2 = btf_array(t2)->type;
1386                 goto recur;
1387         case BTF_KIND_FUNC:
1388                 /* extern and global linkages are compatible */
1389                 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1390                 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1391                 if (is_static1 != is_static2) {
1392                         pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1393                         return false;
1394                 }
1395
1396                 id1 = t1->type;
1397                 id2 = t2->type;
1398                 goto recur;
1399         case BTF_KIND_VAR:
1400                 /* extern and global linkages are compatible */
1401                 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1402                 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1403                 if (is_static1 != is_static2) {
1404                         pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1405                         return false;
1406                 }
1407
1408                 id1 = t1->type;
1409                 id2 = t2->type;
1410                 goto recur;
1411         case BTF_KIND_STRUCT:
1412         case BTF_KIND_UNION: {
1413                 const struct btf_member *m1, *m2;
1414
1415                 if (!exact)
1416                         return true;
1417
1418                 if (btf_vlen(t1) != btf_vlen(t2)) {
1419                         pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1420                                 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1421                         return false;
1422                 }
1423
1424                 n = btf_vlen(t1);
1425                 m1 = btf_members(t1);
1426                 m2 = btf_members(t2);
1427                 for (i = 0; i < n; i++, m1++, m2++) {
1428                         n1 = btf__str_by_offset(btf1, m1->name_off);
1429                         n2 = btf__str_by_offset(btf2, m2->name_off);
1430                         if (strcmp(n1, n2) != 0) {
1431                                 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1432                                         sym_name, i, n1, n2);
1433                                 return false;
1434                         }
1435                         if (m1->offset != m2->offset) {
1436                                 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1437                                         sym_name, i, n1);
1438                                 return false;
1439                         }
1440                         if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1441                                 return false;
1442                 }
1443
1444                 return true;
1445         }
1446         case BTF_KIND_FUNC_PROTO: {
1447                 const struct btf_param *m1, *m2;
1448
1449                 if (btf_vlen(t1) != btf_vlen(t2)) {
1450                         pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1451                                 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1452                         return false;
1453                 }
1454
1455                 n = btf_vlen(t1);
1456                 m1 = btf_params(t1);
1457                 m2 = btf_params(t2);
1458                 for (i = 0; i < n; i++, m1++, m2++) {
1459                         /* ignore func arg names */
1460                         if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1461                                 return false;
1462                 }
1463
1464                 /* now check return type as well */
1465                 id1 = t1->type;
1466                 id2 = t2->type;
1467                 goto recur;
1468         }
1469
1470         /* skip_mods_and_typedefs() make this impossible */
1471         case BTF_KIND_TYPEDEF:
1472         case BTF_KIND_VOLATILE:
1473         case BTF_KIND_CONST:
1474         case BTF_KIND_RESTRICT:
1475         /* DATASECs are never compared with each other */
1476         case BTF_KIND_DATASEC:
1477         default:
1478                 pr_warn("global '%s': unsupported BTF kind %s\n",
1479                         sym_name, btf_kind_str(t1));
1480                 return false;
1481         }
1482 }
1483
1484 static bool map_defs_match(const char *sym_name,
1485                            const struct btf *main_btf,
1486                            const struct btf_map_def *main_def,
1487                            const struct btf_map_def *main_inner_def,
1488                            const struct btf *extra_btf,
1489                            const struct btf_map_def *extra_def,
1490                            const struct btf_map_def *extra_inner_def)
1491 {
1492         const char *reason;
1493
1494         if (main_def->map_type != extra_def->map_type) {
1495                 reason = "type";
1496                 goto mismatch;
1497         }
1498
1499         /* check key type/size match */
1500         if (main_def->key_size != extra_def->key_size) {
1501                 reason = "key_size";
1502                 goto mismatch;
1503         }
1504         if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1505                 reason = "key type";
1506                 goto mismatch;
1507         }
1508         if ((main_def->parts & MAP_DEF_KEY_TYPE)
1509              && !glob_sym_btf_matches(sym_name, true /*exact*/,
1510                                       main_btf, main_def->key_type_id,
1511                                       extra_btf, extra_def->key_type_id)) {
1512                 reason = "key type";
1513                 goto mismatch;
1514         }
1515
1516         /* validate value type/size match */
1517         if (main_def->value_size != extra_def->value_size) {
1518                 reason = "value_size";
1519                 goto mismatch;
1520         }
1521         if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1522                 reason = "value type";
1523                 goto mismatch;
1524         }
1525         if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1526              && !glob_sym_btf_matches(sym_name, true /*exact*/,
1527                                       main_btf, main_def->value_type_id,
1528                                       extra_btf, extra_def->value_type_id)) {
1529                 reason = "key type";
1530                 goto mismatch;
1531         }
1532
1533         if (main_def->max_entries != extra_def->max_entries) {
1534                 reason = "max_entries";
1535                 goto mismatch;
1536         }
1537         if (main_def->map_flags != extra_def->map_flags) {
1538                 reason = "map_flags";
1539                 goto mismatch;
1540         }
1541         if (main_def->numa_node != extra_def->numa_node) {
1542                 reason = "numa_node";
1543                 goto mismatch;
1544         }
1545         if (main_def->pinning != extra_def->pinning) {
1546                 reason = "pinning";
1547                 goto mismatch;
1548         }
1549
1550         if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1551                 reason = "inner map";
1552                 goto mismatch;
1553         }
1554
1555         if (main_def->parts & MAP_DEF_INNER_MAP) {
1556                 char inner_map_name[128];
1557
1558                 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1559
1560                 return map_defs_match(inner_map_name,
1561                                       main_btf, main_inner_def, NULL,
1562                                       extra_btf, extra_inner_def, NULL);
1563         }
1564
1565         return true;
1566
1567 mismatch:
1568         pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1569         return false;
1570 }
1571
1572 static bool glob_map_defs_match(const char *sym_name,
1573                                 struct bpf_linker *linker, struct glob_sym *glob_sym,
1574                                 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1575 {
1576         struct btf_map_def dst_def = {}, dst_inner_def = {};
1577         struct btf_map_def src_def = {}, src_inner_def = {};
1578         const struct btf_type *t;
1579         int err;
1580
1581         t = btf__type_by_id(obj->btf, btf_id);
1582         if (!btf_is_var(t)) {
1583                 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1584                 return false;
1585         }
1586         t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1587
1588         err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1589         if (err) {
1590                 pr_warn("global '%s': invalid map definition\n", sym_name);
1591                 return false;
1592         }
1593
1594         /* re-parse existing map definition */
1595         t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1596         t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1597         err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1598         if (err) {
1599                 /* this should not happen, because we already validated it */
1600                 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1601                 return false;
1602         }
1603
1604         /* Currently extern map definition has to be complete and match
1605          * concrete map definition exactly. This restriction might be lifted
1606          * in the future.
1607          */
1608         return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1609                               obj->btf, &src_def, &src_inner_def);
1610 }
1611
1612 static bool glob_syms_match(const char *sym_name,
1613                             struct bpf_linker *linker, struct glob_sym *glob_sym,
1614                             struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1615 {
1616         const struct btf_type *src_t;
1617
1618         /* if we are dealing with externs, BTF types describing both global
1619          * and extern VARs/FUNCs should be completely present in all files
1620          */
1621         if (!glob_sym->btf_id || !btf_id) {
1622                 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1623                 return false;
1624         }
1625
1626         src_t = btf__type_by_id(obj->btf, btf_id);
1627         if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1628                 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1629                         btf_kind_str(src_t), sym_name);
1630                 return false;
1631         }
1632
1633         /* deal with .maps definitions specially */
1634         if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1635                 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1636
1637         if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1638                                   linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1639                 return false;
1640
1641         return true;
1642 }
1643
1644 static bool btf_is_non_static(const struct btf_type *t)
1645 {
1646         return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1647                || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1648 }
1649
1650 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1651                              int *out_btf_sec_id, int *out_btf_id)
1652 {
1653         int i, j, n, m, btf_id = 0;
1654         const struct btf_type *t;
1655         const struct btf_var_secinfo *vi;
1656         const char *name;
1657
1658         if (!obj->btf) {
1659                 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1660                 return -EINVAL;
1661         }
1662
1663         n = btf__get_nr_types(obj->btf);
1664         for (i = 1; i <= n; i++) {
1665                 t = btf__type_by_id(obj->btf, i);
1666
1667                 /* some global and extern FUNCs and VARs might not be associated with any
1668                  * DATASEC, so try to detect them in the same pass
1669                  */
1670                 if (btf_is_non_static(t)) {
1671                         name = btf__str_by_offset(obj->btf, t->name_off);
1672                         if (strcmp(name, sym_name) != 0)
1673                                 continue;
1674
1675                         /* remember and still try to find DATASEC */
1676                         btf_id = i;
1677                         continue;
1678                 }
1679
1680                 if (!btf_is_datasec(t))
1681                         continue;
1682
1683                 vi = btf_var_secinfos(t);
1684                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1685                         t = btf__type_by_id(obj->btf, vi->type);
1686                         name = btf__str_by_offset(obj->btf, t->name_off);
1687
1688                         if (strcmp(name, sym_name) != 0)
1689                                 continue;
1690                         if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1691                                 continue;
1692                         if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1693                                 continue;
1694
1695                         if (btf_id && btf_id != vi->type) {
1696                                 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1697                                         sym_name, btf_id, vi->type);
1698                                 return -EINVAL;
1699                         }
1700
1701                         *out_btf_sec_id = i;
1702                         *out_btf_id = vi->type;
1703
1704                         return 0;
1705                 }
1706         }
1707
1708         /* free-floating extern or global FUNC */
1709         if (btf_id) {
1710                 *out_btf_sec_id = 0;
1711                 *out_btf_id = btf_id;
1712                 return 0;
1713         }
1714
1715         pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1716         return -ENOENT;
1717 }
1718
1719 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1720 {
1721         struct src_sec *sec;
1722         int i;
1723
1724         for (i = 1; i < obj->sec_cnt; i++) {
1725                 sec = &obj->secs[i];
1726
1727                 if (strcmp(sec->sec_name, sec_name) == 0)
1728                         return sec;
1729         }
1730
1731         return NULL;
1732 }
1733
1734 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1735                                     struct btf *src_btf, int src_id)
1736 {
1737         struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1738         struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1739         struct btf_param *src_p, *dst_p;
1740         const char *s;
1741         int i, n, off;
1742
1743         /* We already made sure that source and destination types (FUNC or
1744          * VAR) match in terms of types and argument names.
1745          */
1746         if (btf_is_var(dst_t)) {
1747                 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1748                 return 0;
1749         }
1750
1751         dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1752
1753         /* now onto FUNC_PROTO types */
1754         src_t = btf_type_by_id(src_btf, src_t->type);
1755         dst_t = btf_type_by_id(dst_btf, dst_t->type);
1756
1757         /* Fill in all the argument names, which for extern FUNCs are missing.
1758          * We'll end up with two copies of FUNCs/VARs for externs, but that
1759          * will be taken care of by BTF dedup at the very end.
1760          * It might be that BTF types for extern in one file has less/more BTF
1761          * information (e.g., FWD instead of full STRUCT/UNION information),
1762          * but that should be (in most cases, subject to BTF dedup rules)
1763          * handled and resolved by BTF dedup algorithm as well, so we won't
1764          * worry about it. Our only job is to make sure that argument names
1765          * are populated on both sides, otherwise BTF dedup will pedantically
1766          * consider them different.
1767          */
1768         src_p = btf_params(src_t);
1769         dst_p = btf_params(dst_t);
1770         for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1771                 if (!src_p->name_off)
1772                         continue;
1773
1774                 /* src_btf has more complete info, so add name to dst_btf */
1775                 s = btf__str_by_offset(src_btf, src_p->name_off);
1776                 off = btf__add_str(dst_btf, s);
1777                 if (off < 0)
1778                         return off;
1779                 dst_p->name_off = off;
1780         }
1781         return 0;
1782 }
1783
1784 static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1785 {
1786         sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1787 }
1788
1789 static void sym_update_type(Elf64_Sym *sym, int sym_type)
1790 {
1791         sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1792 }
1793
1794 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1795 {
1796         /* libelf doesn't provide setters for ST_VISIBILITY,
1797          * but it is stored in the lower 2 bits of st_other
1798          */
1799         sym->st_other &= ~0x03;
1800         sym->st_other |= sym_vis;
1801 }
1802
1803 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1804                                  Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1805 {
1806         struct src_sec *src_sec = NULL;
1807         struct dst_sec *dst_sec = NULL;
1808         struct glob_sym *glob_sym = NULL;
1809         int name_off, sym_type, sym_bind, sym_vis, err;
1810         int btf_sec_id = 0, btf_id = 0;
1811         size_t dst_sym_idx;
1812         Elf64_Sym *dst_sym;
1813         bool sym_is_extern;
1814
1815         sym_type = ELF64_ST_TYPE(sym->st_info);
1816         sym_bind = ELF64_ST_BIND(sym->st_info);
1817         sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1818         sym_is_extern = sym->st_shndx == SHN_UNDEF;
1819
1820         if (sym_is_extern) {
1821                 if (!obj->btf) {
1822                         pr_warn("externs without BTF info are not supported\n");
1823                         return -ENOTSUP;
1824                 }
1825         } else if (sym->st_shndx < SHN_LORESERVE) {
1826                 src_sec = &obj->secs[sym->st_shndx];
1827                 if (src_sec->skipped)
1828                         return 0;
1829                 dst_sec = &linker->secs[src_sec->dst_id];
1830
1831                 /* allow only one STT_SECTION symbol per section */
1832                 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1833                         obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1834                         return 0;
1835                 }
1836         }
1837
1838         if (sym_bind == STB_LOCAL)
1839                 goto add_sym;
1840
1841         /* find matching BTF info */
1842         err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1843         if (err)
1844                 return err;
1845
1846         if (sym_is_extern && btf_sec_id) {
1847                 const char *sec_name = NULL;
1848                 const struct btf_type *t;
1849
1850                 t = btf__type_by_id(obj->btf, btf_sec_id);
1851                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1852
1853                 /* Clang puts unannotated extern vars into
1854                  * '.extern' BTF DATASEC. Treat them the same
1855                  * as unannotated extern funcs (which are
1856                  * currently not put into any DATASECs).
1857                  * Those don't have associated src_sec/dst_sec.
1858                  */
1859                 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1860                         src_sec = find_src_sec_by_name(obj, sec_name);
1861                         if (!src_sec) {
1862                                 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1863                                 return -ENOENT;
1864                         }
1865                         dst_sec = &linker->secs[src_sec->dst_id];
1866                 }
1867         }
1868
1869         glob_sym = find_glob_sym(linker, sym_name);
1870         if (glob_sym) {
1871                 /* Preventively resolve to existing symbol. This is
1872                  * needed for further relocation symbol remapping in
1873                  * the next step of linking.
1874                  */
1875                 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1876
1877                 /* If both symbols are non-externs, at least one of
1878                  * them has to be STB_WEAK, otherwise they are in
1879                  * a conflict with each other.
1880                  */
1881                 if (!sym_is_extern && !glob_sym->is_extern
1882                     && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1883                         pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1884                                 src_sym_idx, sym_name, obj->filename);
1885                         return -EINVAL;
1886                 }
1887
1888                 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1889                         return -EINVAL;
1890
1891                 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1892
1893                 /* If new symbol is strong, then force dst_sym to be strong as
1894                  * well; this way a mix of weak and non-weak extern
1895                  * definitions will end up being strong.
1896                  */
1897                 if (sym_bind == STB_GLOBAL) {
1898                         /* We still need to preserve type (NOTYPE or
1899                          * OBJECT/FUNC, depending on whether the symbol is
1900                          * extern or not)
1901                          */
1902                         sym_update_bind(dst_sym, STB_GLOBAL);
1903                         glob_sym->is_weak = false;
1904                 }
1905
1906                 /* Non-default visibility is "contaminating", with stricter
1907                  * visibility overwriting more permissive ones, even if more
1908                  * permissive visibility comes from just an extern definition.
1909                  * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1910                  * ensured by ELF symbol sanity checks above.
1911                  */
1912                 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1913                         sym_update_visibility(dst_sym, sym_vis);
1914
1915                 /* If the new symbol is extern, then regardless if
1916                  * existing symbol is extern or resolved global, just
1917                  * keep the existing one untouched.
1918                  */
1919                 if (sym_is_extern)
1920                         return 0;
1921
1922                 /* If existing symbol is a strong resolved symbol, bail out,
1923                  * because we lost resolution battle have nothing to
1924                  * contribute. We already checked abover that there is no
1925                  * strong-strong conflict. We also already tightened binding
1926                  * and visibility, so nothing else to contribute at that point.
1927                  */
1928                 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
1929                         return 0;
1930
1931                 /* At this point, new symbol is strong non-extern,
1932                  * so overwrite glob_sym with new symbol information.
1933                  * Preserve binding and visibility.
1934                  */
1935                 sym_update_type(dst_sym, sym_type);
1936                 dst_sym->st_shndx = dst_sec->sec_idx;
1937                 dst_sym->st_value = src_sec->dst_off + sym->st_value;
1938                 dst_sym->st_size = sym->st_size;
1939
1940                 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
1941                 glob_sym->sec_id = dst_sec->id;
1942                 glob_sym->is_extern = false;
1943
1944                 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
1945                                              obj->btf, btf_id))
1946                         return -EINVAL;
1947
1948                 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
1949                 glob_sym->underlying_btf_id = 0;
1950
1951                 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1952                 return 0;
1953         }
1954
1955 add_sym:
1956         name_off = strset__add_str(linker->strtab_strs, sym_name);
1957         if (name_off < 0)
1958                 return name_off;
1959
1960         dst_sym = add_new_sym(linker, &dst_sym_idx);
1961         if (!dst_sym)
1962                 return -ENOMEM;
1963
1964         dst_sym->st_name = name_off;
1965         dst_sym->st_info = sym->st_info;
1966         dst_sym->st_other = sym->st_other;
1967         dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
1968         dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1969         dst_sym->st_size = sym->st_size;
1970
1971         obj->sym_map[src_sym_idx] = dst_sym_idx;
1972
1973         if (sym_type == STT_SECTION && dst_sym) {
1974                 dst_sec->sec_sym_idx = dst_sym_idx;
1975                 dst_sym->st_value = 0;
1976         }
1977
1978         if (sym_bind != STB_LOCAL) {
1979                 glob_sym = add_glob_sym(linker);
1980                 if (!glob_sym)
1981                         return -ENOMEM;
1982
1983                 glob_sym->sym_idx = dst_sym_idx;
1984                 /* we use dst_sec->id (and not dst_sec->sec_idx), because
1985                  * ephemeral sections (.kconfig, .ksyms, etc) don't have
1986                  * sec_idx (as they don't have corresponding ELF section), but
1987                  * still have id. .extern doesn't have even ephemeral section
1988                  * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
1989                  */
1990                 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
1991                 glob_sym->name_off = name_off;
1992                 /* we will fill btf_id in during BTF merging step */
1993                 glob_sym->btf_id = 0;
1994                 glob_sym->is_extern = sym_is_extern;
1995                 glob_sym->is_weak = sym_bind == STB_WEAK;
1996         }
1997
1998         return 0;
1999 }
2000
2001 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2002 {
2003         struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2004         struct dst_sec *dst_symtab;
2005         int i, err;
2006
2007         for (i = 1; i < obj->sec_cnt; i++) {
2008                 struct src_sec *src_sec, *src_linked_sec;
2009                 struct dst_sec *dst_sec, *dst_linked_sec;
2010                 Elf64_Rel *src_rel, *dst_rel;
2011                 int j, n;
2012
2013                 src_sec = &obj->secs[i];
2014                 if (!is_relo_sec(src_sec))
2015                         continue;
2016
2017                 /* shdr->sh_info points to relocatable section */
2018                 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2019                 if (src_linked_sec->skipped)
2020                         continue;
2021
2022                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2023                 if (!dst_sec) {
2024                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
2025                         if (!dst_sec)
2026                                 return -ENOMEM;
2027                         err = init_sec(linker, dst_sec, src_sec);
2028                         if (err) {
2029                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2030                                 return err;
2031                         }
2032                 } else if (!secs_match(dst_sec, src_sec)) {
2033                         pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2034                         return -1;
2035                 }
2036
2037                 /* add_dst_sec() above could have invalidated linker->secs */
2038                 dst_symtab = &linker->secs[linker->symtab_sec_idx];
2039
2040                 /* shdr->sh_link points to SYMTAB */
2041                 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2042
2043                 /* shdr->sh_info points to relocated section */
2044                 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2045                 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2046
2047                 src_sec->dst_id = dst_sec->id;
2048                 err = extend_sec(linker, dst_sec, src_sec);
2049                 if (err)
2050                         return err;
2051
2052                 src_rel = src_sec->data->d_buf;
2053                 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2054                 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2055                 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2056                         size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2057                         size_t sym_type = ELF64_R_TYPE(src_rel->r_info);
2058                         Elf64_Sym *src_sym, *dst_sym;
2059                         size_t dst_sym_idx;
2060
2061                         src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2062                         src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2063
2064                         dst_sym_idx = obj->sym_map[src_sym_idx];
2065                         dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx;
2066                         dst_rel->r_offset += src_linked_sec->dst_off;
2067                         sym_type = ELF64_R_TYPE(src_rel->r_info);
2068                         dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2069
2070                         if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2071                                 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2072                                 struct bpf_insn *insn;
2073
2074                                 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2075                                         /* calls to the very first static function inside
2076                                          * .text section at offset 0 will
2077                                          * reference section symbol, not the
2078                                          * function symbol. Fix that up,
2079                                          * otherwise it won't be possible to
2080                                          * relocate calls to two different
2081                                          * static functions with the same name
2082                                          * (rom two different object files)
2083                                          */
2084                                         insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2085                                         if (insn->code == (BPF_JMP | BPF_CALL))
2086                                                 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2087                                         else
2088                                                 insn->imm += sec->dst_off;
2089                                 } else {
2090                                         pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2091                                         return -EINVAL;
2092                                 }
2093                         }
2094
2095                 }
2096         }
2097
2098         return 0;
2099 }
2100
2101 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2102                                    int sym_type, const char *sym_name)
2103 {
2104         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2105         Elf64_Sym *sym = symtab->data->d_buf;
2106         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2107         int str_sec_idx = symtab->shdr->sh_link;
2108         const char *name;
2109
2110         for (i = 0; i < n; i++, sym++) {
2111                 if (sym->st_shndx != sec_idx)
2112                         continue;
2113                 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2114                         continue;
2115
2116                 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2117                 if (!name)
2118                         return NULL;
2119
2120                 if (strcmp(sym_name, name) != 0)
2121                         continue;
2122
2123                 return sym;
2124         }
2125
2126         return NULL;
2127 }
2128
2129 static int linker_fixup_btf(struct src_obj *obj)
2130 {
2131         const char *sec_name;
2132         struct src_sec *sec;
2133         int i, j, n, m;
2134
2135         if (!obj->btf)
2136                 return 0;
2137
2138         n = btf__get_nr_types(obj->btf);
2139         for (i = 1; i <= n; i++) {
2140                 struct btf_var_secinfo *vi;
2141                 struct btf_type *t;
2142
2143                 t = btf_type_by_id(obj->btf, i);
2144                 if (btf_kind(t) != BTF_KIND_DATASEC)
2145                         continue;
2146
2147                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2148                 sec = find_src_sec_by_name(obj, sec_name);
2149                 if (sec) {
2150                         /* record actual section size, unless ephemeral */
2151                         if (sec->shdr)
2152                                 t->size = sec->shdr->sh_size;
2153                 } else {
2154                         /* BTF can have some sections that are not represented
2155                          * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2156                          * for special extern variables.
2157                          *
2158                          * For all but one such special (ephemeral)
2159                          * sections, we pre-create "section shells" to be able
2160                          * to keep track of extra per-section metadata later
2161                          * (e.g., those BTF extern variables).
2162                          *
2163                          * .extern is even more special, though, because it
2164                          * contains extern variables that need to be resolved
2165                          * by static linker, not libbpf and kernel. When such
2166                          * externs are resolved, we are going to remove them
2167                          * from .extern BTF section and might end up not
2168                          * needing it at all. Each resolved extern should have
2169                          * matching non-extern VAR/FUNC in other sections.
2170                          *
2171                          * We do support leaving some of the externs
2172                          * unresolved, though, to support cases of building
2173                          * libraries, which will later be linked against final
2174                          * BPF applications. So if at finalization we still
2175                          * see unresolved externs, we'll create .extern
2176                          * section on our own.
2177                          */
2178                         if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2179                                 continue;
2180
2181                         sec = add_src_sec(obj, sec_name);
2182                         if (!sec)
2183                                 return -ENOMEM;
2184
2185                         sec->ephemeral = true;
2186                         sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2187                 }
2188
2189                 /* remember ELF section and its BTF type ID match */
2190                 sec->sec_type_id = i;
2191
2192                 /* fix up variable offsets */
2193                 vi = btf_var_secinfos(t);
2194                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2195                         const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2196                         const char *var_name = btf__str_by_offset(obj->btf, vt->name_off);
2197                         int var_linkage = btf_var(vt)->linkage;
2198                         Elf64_Sym *sym;
2199
2200                         /* no need to patch up static or extern vars */
2201                         if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2202                                 continue;
2203
2204                         sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2205                         if (!sym) {
2206                                 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2207                                 return -ENOENT;
2208                         }
2209
2210                         vi->offset = sym->st_value;
2211                 }
2212         }
2213
2214         return 0;
2215 }
2216
2217 static int remap_type_id(__u32 *type_id, void *ctx)
2218 {
2219         int *id_map = ctx;
2220         int new_id = id_map[*type_id];
2221
2222         /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2223         if (new_id == 0 && *type_id != 0) {
2224                 pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id);
2225                 return -EINVAL;
2226         }
2227
2228         *type_id = id_map[*type_id];
2229
2230         return 0;
2231 }
2232
2233 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2234 {
2235         const struct btf_type *t;
2236         int i, j, n, start_id, id;
2237         const char *name;
2238
2239         if (!obj->btf)
2240                 return 0;
2241
2242         start_id = btf__get_nr_types(linker->btf) + 1;
2243         n = btf__get_nr_types(obj->btf);
2244
2245         obj->btf_type_map = calloc(n + 1, sizeof(int));
2246         if (!obj->btf_type_map)
2247                 return -ENOMEM;
2248
2249         for (i = 1; i <= n; i++) {
2250                 struct glob_sym *glob_sym = NULL;
2251
2252                 t = btf__type_by_id(obj->btf, i);
2253
2254                 /* DATASECs are handled specially below */
2255                 if (btf_kind(t) == BTF_KIND_DATASEC)
2256                         continue;
2257
2258                 if (btf_is_non_static(t)) {
2259                         /* there should be glob_sym already */
2260                         name = btf__str_by_offset(obj->btf, t->name_off);
2261                         glob_sym = find_glob_sym(linker, name);
2262
2263                         /* VARs without corresponding glob_sym are those that
2264                          * belong to skipped/deduplicated sections (i.e.,
2265                          * license and version), so just skip them
2266                          */
2267                         if (!glob_sym)
2268                                 continue;
2269
2270                         /* linker_append_elf_sym() might have requested
2271                          * updating underlying type ID, if extern was resolved
2272                          * to strong symbol or weak got upgraded to non-weak
2273                          */
2274                         if (glob_sym->underlying_btf_id == 0)
2275                                 glob_sym->underlying_btf_id = -t->type;
2276
2277                         /* globals from previous object files that match our
2278                          * VAR/FUNC already have a corresponding associated
2279                          * BTF type, so just make sure to use it
2280                          */
2281                         if (glob_sym->btf_id) {
2282                                 /* reuse existing BTF type for global var/func */
2283                                 obj->btf_type_map[i] = glob_sym->btf_id;
2284                                 continue;
2285                         }
2286                 }
2287
2288                 id = btf__add_type(linker->btf, obj->btf, t);
2289                 if (id < 0) {
2290                         pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2291                         return id;
2292                 }
2293
2294                 obj->btf_type_map[i] = id;
2295
2296                 /* record just appended BTF type for var/func */
2297                 if (glob_sym) {
2298                         glob_sym->btf_id = id;
2299                         glob_sym->underlying_btf_id = -t->type;
2300                 }
2301         }
2302
2303         /* remap all the types except DATASECs */
2304         n = btf__get_nr_types(linker->btf);
2305         for (i = start_id; i <= n; i++) {
2306                 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2307
2308                 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
2309                         return -EINVAL;
2310         }
2311
2312         /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2313          * actual type), if necessary
2314          */
2315         for (i = 0; i < linker->glob_sym_cnt; i++) {
2316                 struct glob_sym *glob_sym = &linker->glob_syms[i];
2317                 struct btf_type *glob_t;
2318
2319                 if (glob_sym->underlying_btf_id >= 0)
2320                         continue;
2321
2322                 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2323
2324                 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2325                 glob_t->type = glob_sym->underlying_btf_id;
2326         }
2327
2328         /* append DATASEC info */
2329         for (i = 1; i < obj->sec_cnt; i++) {
2330                 struct src_sec *src_sec;
2331                 struct dst_sec *dst_sec;
2332                 const struct btf_var_secinfo *src_var;
2333                 struct btf_var_secinfo *dst_var;
2334
2335                 src_sec = &obj->secs[i];
2336                 if (!src_sec->sec_type_id || src_sec->skipped)
2337                         continue;
2338                 dst_sec = &linker->secs[src_sec->dst_id];
2339
2340                 /* Mark section as having BTF regardless of the presence of
2341                  * variables. In some cases compiler might generate empty BTF
2342                  * with no variables information. E.g., when promoting local
2343                  * array/structure variable initial values and BPF object
2344                  * file otherwise has no read-only static variables in
2345                  * .rodata. We need to preserve such empty BTF and just set
2346                  * correct section size.
2347                  */
2348                 dst_sec->has_btf = true;
2349
2350                 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2351                 src_var = btf_var_secinfos(t);
2352                 n = btf_vlen(t);
2353                 for (j = 0; j < n; j++, src_var++) {
2354                         void *sec_vars = dst_sec->sec_vars;
2355                         int new_id = obj->btf_type_map[src_var->type];
2356                         struct glob_sym *glob_sym = NULL;
2357
2358                         t = btf_type_by_id(linker->btf, new_id);
2359                         if (btf_is_non_static(t)) {
2360                                 name = btf__str_by_offset(linker->btf, t->name_off);
2361                                 glob_sym = find_glob_sym(linker, name);
2362                                 if (glob_sym->sec_id != dst_sec->id) {
2363                                         pr_warn("global '%s': section mismatch %d vs %d\n",
2364                                                 name, glob_sym->sec_id, dst_sec->id);
2365                                         return -EINVAL;
2366                                 }
2367                         }
2368
2369                         /* If there is already a member (VAR or FUNC) mapped
2370                          * to the same type, don't add a duplicate entry.
2371                          * This will happen when multiple object files define
2372                          * the same extern VARs/FUNCs.
2373                          */
2374                         if (glob_sym && glob_sym->var_idx >= 0) {
2375                                 __s64 sz;
2376
2377                                 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2378                                 /* Because underlying BTF type might have
2379                                  * changed, so might its size have changed, so
2380                                  * re-calculate and update it in sec_var.
2381                                  */
2382                                 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2383                                 if (sz < 0) {
2384                                         pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2385                                                 name, (int)sz);
2386                                         return -EINVAL;
2387                                 }
2388                                 dst_var->size = sz;
2389                                 continue;
2390                         }
2391
2392                         sec_vars = libbpf_reallocarray(sec_vars,
2393                                                        dst_sec->sec_var_cnt + 1,
2394                                                        sizeof(*dst_sec->sec_vars));
2395                         if (!sec_vars)
2396                                 return -ENOMEM;
2397
2398                         dst_sec->sec_vars = sec_vars;
2399                         dst_sec->sec_var_cnt++;
2400
2401                         dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2402                         dst_var->type = obj->btf_type_map[src_var->type];
2403                         dst_var->size = src_var->size;
2404                         dst_var->offset = src_sec->dst_off + src_var->offset;
2405
2406                         if (glob_sym)
2407                                 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2408                 }
2409         }
2410
2411         return 0;
2412 }
2413
2414 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2415 {
2416         void *tmp;
2417
2418         tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2419         if (!tmp)
2420                 return NULL;
2421         ext_data->recs = tmp;
2422
2423         tmp += ext_data->rec_cnt * ext_data->rec_sz;
2424         memcpy(tmp, src_rec, ext_data->rec_sz);
2425
2426         ext_data->rec_cnt++;
2427
2428         return tmp;
2429 }
2430
2431 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2432 {
2433         const struct btf_ext_info_sec *ext_sec;
2434         const char *sec_name, *s;
2435         struct src_sec *src_sec;
2436         struct dst_sec *dst_sec;
2437         int rec_sz, str_off, i;
2438
2439         if (!obj->btf_ext)
2440                 return 0;
2441
2442         rec_sz = obj->btf_ext->func_info.rec_size;
2443         for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2444                 struct bpf_func_info_min *src_rec, *dst_rec;
2445
2446                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2447                 src_sec = find_src_sec_by_name(obj, sec_name);
2448                 if (!src_sec) {
2449                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2450                         return -EINVAL;
2451                 }
2452                 dst_sec = &linker->secs[src_sec->dst_id];
2453
2454                 if (dst_sec->func_info.rec_sz == 0)
2455                         dst_sec->func_info.rec_sz = rec_sz;
2456                 if (dst_sec->func_info.rec_sz != rec_sz) {
2457                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2458                         return -EINVAL;
2459                 }
2460
2461                 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2462                         dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2463                         if (!dst_rec)
2464                                 return -ENOMEM;
2465
2466                         dst_rec->insn_off += src_sec->dst_off;
2467                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2468                 }
2469         }
2470
2471         rec_sz = obj->btf_ext->line_info.rec_size;
2472         for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2473                 struct bpf_line_info_min *src_rec, *dst_rec;
2474
2475                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2476                 src_sec = find_src_sec_by_name(obj, sec_name);
2477                 if (!src_sec) {
2478                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2479                         return -EINVAL;
2480                 }
2481                 dst_sec = &linker->secs[src_sec->dst_id];
2482
2483                 if (dst_sec->line_info.rec_sz == 0)
2484                         dst_sec->line_info.rec_sz = rec_sz;
2485                 if (dst_sec->line_info.rec_sz != rec_sz) {
2486                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2487                         return -EINVAL;
2488                 }
2489
2490                 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2491                         dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2492                         if (!dst_rec)
2493                                 return -ENOMEM;
2494
2495                         dst_rec->insn_off += src_sec->dst_off;
2496
2497                         s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2498                         str_off = btf__add_str(linker->btf, s);
2499                         if (str_off < 0)
2500                                 return -ENOMEM;
2501                         dst_rec->file_name_off = str_off;
2502
2503                         s = btf__str_by_offset(obj->btf, src_rec->line_off);
2504                         str_off = btf__add_str(linker->btf, s);
2505                         if (str_off < 0)
2506                                 return -ENOMEM;
2507                         dst_rec->line_off = str_off;
2508
2509                         /* dst_rec->line_col is fine */
2510                 }
2511         }
2512
2513         rec_sz = obj->btf_ext->core_relo_info.rec_size;
2514         for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2515                 struct bpf_core_relo *src_rec, *dst_rec;
2516
2517                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2518                 src_sec = find_src_sec_by_name(obj, sec_name);
2519                 if (!src_sec) {
2520                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2521                         return -EINVAL;
2522                 }
2523                 dst_sec = &linker->secs[src_sec->dst_id];
2524
2525                 if (dst_sec->core_relo_info.rec_sz == 0)
2526                         dst_sec->core_relo_info.rec_sz = rec_sz;
2527                 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2528                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2529                         return -EINVAL;
2530                 }
2531
2532                 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2533                         dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2534                         if (!dst_rec)
2535                                 return -ENOMEM;
2536
2537                         dst_rec->insn_off += src_sec->dst_off;
2538                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2539
2540                         s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2541                         str_off = btf__add_str(linker->btf, s);
2542                         if (str_off < 0)
2543                                 return -ENOMEM;
2544                         dst_rec->access_str_off = str_off;
2545
2546                         /* dst_rec->kind is fine */
2547                 }
2548         }
2549
2550         return 0;
2551 }
2552
2553 int bpf_linker__finalize(struct bpf_linker *linker)
2554 {
2555         struct dst_sec *sec;
2556         size_t strs_sz;
2557         const void *strs;
2558         int err, i;
2559
2560         if (!linker->elf)
2561                 return libbpf_err(-EINVAL);
2562
2563         err = finalize_btf(linker);
2564         if (err)
2565                 return libbpf_err(err);
2566
2567         /* Finalize strings */
2568         strs_sz = strset__data_size(linker->strtab_strs);
2569         strs = strset__data(linker->strtab_strs);
2570
2571         sec = &linker->secs[linker->strtab_sec_idx];
2572         sec->data->d_align = 1;
2573         sec->data->d_off = 0LL;
2574         sec->data->d_buf = (void *)strs;
2575         sec->data->d_type = ELF_T_BYTE;
2576         sec->data->d_size = strs_sz;
2577         sec->shdr->sh_size = strs_sz;
2578
2579         for (i = 1; i < linker->sec_cnt; i++) {
2580                 sec = &linker->secs[i];
2581
2582                 /* STRTAB is handled specially above */
2583                 if (sec->sec_idx == linker->strtab_sec_idx)
2584                         continue;
2585
2586                 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2587                 if (!sec->scn)
2588                         continue;
2589
2590                 sec->data->d_buf = sec->raw_data;
2591         }
2592
2593         /* Finalize ELF layout */
2594         if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2595                 err = -errno;
2596                 pr_warn_elf("failed to finalize ELF layout");
2597                 return libbpf_err(err);
2598         }
2599
2600         /* Write out final ELF contents */
2601         if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2602                 err = -errno;
2603                 pr_warn_elf("failed to write ELF contents");
2604                 return libbpf_err(err);
2605         }
2606
2607         elf_end(linker->elf);
2608         close(linker->fd);
2609
2610         linker->elf = NULL;
2611         linker->fd = -1;
2612
2613         return 0;
2614 }
2615
2616 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2617                              size_t align, const void *raw_data, size_t raw_sz)
2618 {
2619         Elf_Scn *scn;
2620         Elf_Data *data;
2621         Elf64_Shdr *shdr;
2622         int name_off;
2623
2624         name_off = strset__add_str(linker->strtab_strs, sec_name);
2625         if (name_off < 0)
2626                 return name_off;
2627
2628         scn = elf_newscn(linker->elf);
2629         if (!scn)
2630                 return -ENOMEM;
2631         data = elf_newdata(scn);
2632         if (!data)
2633                 return -ENOMEM;
2634         shdr = elf64_getshdr(scn);
2635         if (!shdr)
2636                 return -EINVAL;
2637
2638         shdr->sh_name = name_off;
2639         shdr->sh_type = SHT_PROGBITS;
2640         shdr->sh_flags = 0;
2641         shdr->sh_size = raw_sz;
2642         shdr->sh_link = 0;
2643         shdr->sh_info = 0;
2644         shdr->sh_addralign = align;
2645         shdr->sh_entsize = 0;
2646
2647         data->d_type = ELF_T_BYTE;
2648         data->d_size = raw_sz;
2649         data->d_buf = (void *)raw_data;
2650         data->d_align = align;
2651         data->d_off = 0;
2652
2653         return 0;
2654 }
2655
2656 static int finalize_btf(struct bpf_linker *linker)
2657 {
2658         struct btf *btf = linker->btf;
2659         const void *raw_data;
2660         int i, j, id, err;
2661         __u32 raw_sz;
2662
2663         /* bail out if no BTF data was produced */
2664         if (btf__get_nr_types(linker->btf) == 0)
2665                 return 0;
2666
2667         for (i = 1; i < linker->sec_cnt; i++) {
2668                 struct dst_sec *sec = &linker->secs[i];
2669
2670                 if (!sec->has_btf)
2671                         continue;
2672
2673                 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2674                 if (id < 0) {
2675                         pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2676                                 sec->sec_name, id);
2677                         return id;
2678                 }
2679
2680                 for (j = 0; j < sec->sec_var_cnt; j++) {
2681                         struct btf_var_secinfo *vi = &sec->sec_vars[j];
2682
2683                         if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2684                                 return -EINVAL;
2685                 }
2686         }
2687
2688         err = finalize_btf_ext(linker);
2689         if (err) {
2690                 pr_warn(".BTF.ext generation failed: %d\n", err);
2691                 return err;
2692         }
2693
2694         err = btf__dedup(linker->btf, linker->btf_ext, NULL);
2695         if (err) {
2696                 pr_warn("BTF dedup failed: %d\n", err);
2697                 return err;
2698         }
2699
2700         /* Emit .BTF section */
2701         raw_data = btf__get_raw_data(linker->btf, &raw_sz);
2702         if (!raw_data)
2703                 return -ENOMEM;
2704
2705         err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2706         if (err) {
2707                 pr_warn("failed to write out .BTF ELF section: %d\n", err);
2708                 return err;
2709         }
2710
2711         /* Emit .BTF.ext section */
2712         if (linker->btf_ext) {
2713                 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
2714                 if (!raw_data)
2715                         return -ENOMEM;
2716
2717                 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2718                 if (err) {
2719                         pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2720                         return err;
2721                 }
2722         }
2723
2724         return 0;
2725 }
2726
2727 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2728                              const char *sec_name, struct btf_ext_sec_data *sec_data)
2729 {
2730         struct btf_ext_info_sec *sec_info;
2731         void *cur = output;
2732         int str_off;
2733         size_t sz;
2734
2735         if (!sec_data->rec_cnt)
2736                 return 0;
2737
2738         str_off = btf__add_str(linker->btf, sec_name);
2739         if (str_off < 0)
2740                 return -ENOMEM;
2741
2742         sec_info = cur;
2743         sec_info->sec_name_off = str_off;
2744         sec_info->num_info = sec_data->rec_cnt;
2745         cur += sizeof(struct btf_ext_info_sec);
2746
2747         sz = sec_data->rec_cnt * sec_data->rec_sz;
2748         memcpy(cur, sec_data->recs, sz);
2749         cur += sz;
2750
2751         return cur - output;
2752 }
2753
2754 static int finalize_btf_ext(struct bpf_linker *linker)
2755 {
2756         size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2757         size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2758         struct btf_ext_header *hdr;
2759         void *data, *cur;
2760         int i, err, sz;
2761
2762         /* validate that all sections have the same .BTF.ext record sizes
2763          * and calculate total data size for each type of data (func info,
2764          * line info, core relos)
2765          */
2766         for (i = 1; i < linker->sec_cnt; i++) {
2767                 struct dst_sec *sec = &linker->secs[i];
2768
2769                 if (sec->func_info.rec_cnt) {
2770                         if (func_rec_sz == 0)
2771                                 func_rec_sz = sec->func_info.rec_sz;
2772                         if (func_rec_sz != sec->func_info.rec_sz) {
2773                                 pr_warn("mismatch in func_info record size %zu != %u\n",
2774                                         func_rec_sz, sec->func_info.rec_sz);
2775                                 return -EINVAL;
2776                         }
2777
2778                         funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2779                 }
2780                 if (sec->line_info.rec_cnt) {
2781                         if (line_rec_sz == 0)
2782                                 line_rec_sz = sec->line_info.rec_sz;
2783                         if (line_rec_sz != sec->line_info.rec_sz) {
2784                                 pr_warn("mismatch in line_info record size %zu != %u\n",
2785                                         line_rec_sz, sec->line_info.rec_sz);
2786                                 return -EINVAL;
2787                         }
2788
2789                         lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2790                 }
2791                 if (sec->core_relo_info.rec_cnt) {
2792                         if (core_relo_rec_sz == 0)
2793                                 core_relo_rec_sz = sec->core_relo_info.rec_sz;
2794                         if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2795                                 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2796                                         core_relo_rec_sz, sec->core_relo_info.rec_sz);
2797                                 return -EINVAL;
2798                         }
2799
2800                         core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2801                 }
2802         }
2803
2804         if (!funcs_sz && !lines_sz && !core_relos_sz)
2805                 return 0;
2806
2807         total_sz += sizeof(struct btf_ext_header);
2808         if (funcs_sz) {
2809                 funcs_sz += sizeof(__u32); /* record size prefix */
2810                 total_sz += funcs_sz;
2811         }
2812         if (lines_sz) {
2813                 lines_sz += sizeof(__u32); /* record size prefix */
2814                 total_sz += lines_sz;
2815         }
2816         if (core_relos_sz) {
2817                 core_relos_sz += sizeof(__u32); /* record size prefix */
2818                 total_sz += core_relos_sz;
2819         }
2820
2821         cur = data = calloc(1, total_sz);
2822         if (!data)
2823                 return -ENOMEM;
2824
2825         hdr = cur;
2826         hdr->magic = BTF_MAGIC;
2827         hdr->version = BTF_VERSION;
2828         hdr->flags = 0;
2829         hdr->hdr_len = sizeof(struct btf_ext_header);
2830         cur += sizeof(struct btf_ext_header);
2831
2832         /* All offsets are in bytes relative to the end of this header */
2833         hdr->func_info_off = 0;
2834         hdr->func_info_len = funcs_sz;
2835         hdr->line_info_off = funcs_sz;
2836         hdr->line_info_len = lines_sz;
2837         hdr->core_relo_off = funcs_sz + lines_sz;
2838         hdr->core_relo_len = core_relos_sz;
2839
2840         if (funcs_sz) {
2841                 *(__u32 *)cur = func_rec_sz;
2842                 cur += sizeof(__u32);
2843
2844                 for (i = 1; i < linker->sec_cnt; i++) {
2845                         struct dst_sec *sec = &linker->secs[i];
2846
2847                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2848                         if (sz < 0) {
2849                                 err = sz;
2850                                 goto out;
2851                         }
2852
2853                         cur += sz;
2854                 }
2855         }
2856
2857         if (lines_sz) {
2858                 *(__u32 *)cur = line_rec_sz;
2859                 cur += sizeof(__u32);
2860
2861                 for (i = 1; i < linker->sec_cnt; i++) {
2862                         struct dst_sec *sec = &linker->secs[i];
2863
2864                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2865                         if (sz < 0) {
2866                                 err = sz;
2867                                 goto out;
2868                         }
2869
2870                         cur += sz;
2871                 }
2872         }
2873
2874         if (core_relos_sz) {
2875                 *(__u32 *)cur = core_relo_rec_sz;
2876                 cur += sizeof(__u32);
2877
2878                 for (i = 1; i < linker->sec_cnt; i++) {
2879                         struct dst_sec *sec = &linker->secs[i];
2880
2881                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2882                         if (sz < 0) {
2883                                 err = sz;
2884                                 goto out;
2885                         }
2886
2887                         cur += sz;
2888                 }
2889         }
2890
2891         linker->btf_ext = btf_ext__new(data, total_sz);
2892         err = libbpf_get_error(linker->btf_ext);
2893         if (err) {
2894                 linker->btf_ext = NULL;
2895                 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2896                 goto out;
2897         }
2898
2899 out:
2900         free(data);
2901         return err;
2902 }