tools/lib/bpf/linker.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2 /*
   3  * BPF static linker
   4  *
   5  * Copyright (c) 2021 Facebook
   6  */
   7 #include <stdbool.h>
   8 #include <stddef.h>
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <unistd.h>
  13 #include <errno.h>
  14 #include <linux/err.h>
  15 #include <linux/btf.h>
  16 #include <elf.h>
  17 #include <libelf.h>
  18 #include <gelf.h>
  19 #include <fcntl.h>
  20 #include "libbpf.h"
  21 #include "btf.h"
  22 #include "libbpf_internal.h"
  23 #include "strset.h"
  24
  25 struct src_sec {
  26         const char *sec_name;
  27         /* positional (not necessarily ELF) index in an array of sections */
  28         int id;
  29         /* positional (not necessarily ELF) index of a matching section in a final object file */
  30         int dst_id;
  31         /* section data offset in a matching output section */
  32         int dst_off;
  33         /* whether section is omitted from the final ELF file */
  34         bool skipped;
  35         /* whether section is an ephemeral section, not mapped to an ELF section */
  36         bool ephemeral;
  37
  38         /* ELF info */
  39         size_t sec_idx;
  40         Elf_Scn *scn;
  41         Elf64_Shdr *shdr;
  42         Elf_Data *data;
  43
  44         /* corresponding BTF DATASEC type ID */
  45         int sec_type_id;
  46 };
  47
  48 struct src_obj {
  49         const char *filename;
  50         int fd;
  51         Elf *elf;
  52         /* Section header strings section index */
  53         size_t shstrs_sec_idx;
  54         /* SYMTAB section index */
  55         size_t symtab_sec_idx;
  56
  57         struct btf *btf;
  58         struct btf_ext *btf_ext;
  59
  60         /* List of sections (including ephemeral). Slot zero is unused. */
  61         struct src_sec *secs;
  62         int sec_cnt;
  63
  64         /* mapping of symbol indices from src to dst ELF */
  65         int *sym_map;
  66         /* mapping from the src BTF type IDs to dst ones */
  67         int *btf_type_map;
  68 };
  69
  70 /* single .BTF.ext data section */
  71 struct btf_ext_sec_data {
  72         size_t rec_cnt;
  73         __u32 rec_sz;
  74         void *recs;
  75 };
  76
  77 struct dst_sec {
  78         char *sec_name;
  79         /* positional (not necessarily ELF) index in an array of sections */
  80         int id;
  81
  82         /* ELF info */
  83         size_t sec_idx;
  84         Elf_Scn *scn;
  85         Elf64_Shdr *shdr;
  86         Elf_Data *data;
  87
  88         /* final output section size */
  89         int sec_sz;
  90         /* final output contents of the section */
  91         void *raw_data;
  92
  93         /* corresponding STT_SECTION symbol index in SYMTAB */
  94         int sec_sym_idx;
  95
  96         /* section's DATASEC variable info, emitted on BTF finalization */
  97         bool has_btf;
  98         int sec_var_cnt;
  99         struct btf_var_secinfo *sec_vars;
 100
 101         /* section's .BTF.ext data */
 102         struct btf_ext_sec_data func_info;
 103         struct btf_ext_sec_data line_info;
 104         struct btf_ext_sec_data core_relo_info;
 105 };
 106
 107 struct bpf_linker {
 108         char *filename;
 109         int fd;
 110         Elf *elf;
 111         Elf64_Ehdr *elf_hdr;
 112
 113         /* Output sections metadata */
 114         struct dst_sec *secs;
 115         int sec_cnt;
 116
 117         struct strset *strtab_strs; /* STRTAB unique strings */
 118         size_t strtab_sec_idx; /* STRTAB section index */
 119         size_t symtab_sec_idx; /* SYMTAB section index */
 120
 121         struct btf *btf;
 122         struct btf_ext *btf_ext;
 123 };
 124
 125 #define pr_warn_elf(fmt, ...)                                                                   \
 126         libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
 127
 128 static int init_output_elf(struct bpf_linker *linker, const char *file);
 129
 130 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj);
 131 static int linker_sanity_check_elf(struct src_obj *obj);
 132 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
 133 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
 134 static int linker_sanity_check_btf(struct src_obj *obj);
 135 static int linker_sanity_check_btf_ext(struct src_obj *obj);
 136 static int linker_fixup_btf(struct src_obj *obj);
 137 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
 138 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
 139 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
 140 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
 141 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
 142
 143 static int finalize_btf(struct bpf_linker *linker);
 144 static int finalize_btf_ext(struct bpf_linker *linker);
 145
 146 void bpf_linker__free(struct bpf_linker *linker)
 147 {
 148         int i;
 149
 150         if (!linker)
 151                 return;
 152
 153         free(linker->filename);
 154
 155         if (linker->elf)
 156                 elf_end(linker->elf);
 157
 158         if (linker->fd >= 0)
 159                 close(linker->fd);
 160
 161         strset__free(linker->strtab_strs);
 162
 163         btf__free(linker->btf);
 164         btf_ext__free(linker->btf_ext);
 165
 166         for (i = 1; i < linker->sec_cnt; i++) {
 167                 struct dst_sec *sec = &linker->secs[i];
 168
 169                 free(sec->sec_name);
 170                 free(sec->raw_data);
 171                 free(sec->sec_vars);
 172
 173                 free(sec->func_info.recs);
 174                 free(sec->line_info.recs);
 175                 free(sec->core_relo_info.recs);
 176         }
 177         free(linker->secs);
 178
 179         free(linker);
 180 }
 181
 182 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
 183 {
 184         struct bpf_linker *linker;
 185         int err;
 186
 187         if (!OPTS_VALID(opts, bpf_linker_opts))
 188                 return NULL;
 189
 190         if (elf_version(EV_CURRENT) == EV_NONE) {
 191                 pr_warn_elf("libelf initialization failed");
 192                 return NULL;
 193         }
 194
 195         linker = calloc(1, sizeof(*linker));
 196         if (!linker)
 197                 return NULL;
 198
 199         linker->fd = -1;
 200
 201         err = init_output_elf(linker, filename);
 202         if (err)
 203                 goto err_out;
 204
 205         return linker;
 206
 207 err_out:
 208         bpf_linker__free(linker);
 209         return NULL;
 210 }
 211
 212 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
 213 {
 214         struct dst_sec *secs = linker->secs, *sec;
 215         size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
 216
 217         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 218         if (!secs)
 219                 return NULL;
 220
 221         /* zero out newly allocated memory */
 222         memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
 223
 224         linker->secs = secs;
 225         linker->sec_cnt = new_cnt;
 226
 227         sec = &linker->secs[new_cnt - 1];
 228         sec->id = new_cnt - 1;
 229         sec->sec_name = strdup(sec_name);
 230         if (!sec->sec_name)
 231                 return NULL;
 232
 233         return sec;
 234 }
 235
 236 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
 237 {
 238         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
 239         Elf64_Sym *syms, *sym;
 240         size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
 241
 242         syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
 243         if (!syms)
 244                 return NULL;
 245
 246         sym = &syms[sym_cnt];
 247         memset(sym, 0, sizeof(*sym));
 248
 249         symtab->raw_data = syms;
 250         symtab->sec_sz += sizeof(*sym);
 251         symtab->shdr->sh_size += sizeof(*sym);
 252         symtab->data->d_size += sizeof(*sym);
 253
 254         if (sym_idx)
 255                 *sym_idx = sym_cnt;
 256
 257         return sym;
 258 }
 259
 260 static int init_output_elf(struct bpf_linker *linker, const char *file)
 261 {
 262         int err, str_off;
 263         Elf64_Sym *init_sym;
 264         struct dst_sec *sec;
 265
 266         linker->filename = strdup(file);
 267         if (!linker->filename)
 268                 return -ENOMEM;
 269
 270         linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
 271         if (linker->fd < 0) {
 272                 err = -errno;
 273                 pr_warn("failed to create '%s': %d\n", file, err);
 274                 return err;
 275         }
 276
 277         linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
 278         if (!linker->elf) {
 279                 pr_warn_elf("failed to create ELF object");
 280                 return -EINVAL;
 281         }
 282
 283         /* ELF header */
 284         linker->elf_hdr = elf64_newehdr(linker->elf);
 285         if (!linker->elf_hdr) {
 286                 pr_warn_elf("failed to create ELF header");
 287                 return -EINVAL;
 288         }
 289
 290         linker->elf_hdr->e_machine = EM_BPF;
 291         linker->elf_hdr->e_type = ET_REL;
 292 #if __BYTE_ORDER == __LITTLE_ENDIAN
 293         linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
 294 #elif __BYTE_ORDER == __BIG_ENDIAN
 295         linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
 296 #else
 297 #error "Unknown __BYTE_ORDER"
 298 #endif
 299
 300         /* STRTAB */
 301         /* initialize strset with an empty string to conform to ELF */
 302         linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
 303         if (libbpf_get_error(linker->strtab_strs))
 304                 return libbpf_get_error(linker->strtab_strs);
 305
 306         sec = add_dst_sec(linker, ".strtab");
 307         if (!sec)
 308                 return -ENOMEM;
 309
 310         sec->scn = elf_newscn(linker->elf);
 311         if (!sec->scn) {
 312                 pr_warn_elf("failed to create STRTAB section");
 313                 return -EINVAL;
 314         }
 315
 316         sec->shdr = elf64_getshdr(sec->scn);
 317         if (!sec->shdr)
 318                 return -EINVAL;
 319
 320         sec->data = elf_newdata(sec->scn);
 321         if (!sec->data) {
 322                 pr_warn_elf("failed to create STRTAB data");
 323                 return -EINVAL;
 324         }
 325
 326         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 327         if (str_off < 0)
 328                 return str_off;
 329
 330         sec->sec_idx = elf_ndxscn(sec->scn);
 331         linker->elf_hdr->e_shstrndx = sec->sec_idx;
 332         linker->strtab_sec_idx = sec->sec_idx;
 333
 334         sec->shdr->sh_name = str_off;
 335         sec->shdr->sh_type = SHT_STRTAB;
 336         sec->shdr->sh_flags = SHF_STRINGS;
 337         sec->shdr->sh_offset = 0;
 338         sec->shdr->sh_link = 0;
 339         sec->shdr->sh_info = 0;
 340         sec->shdr->sh_addralign = 1;
 341         sec->shdr->sh_size = sec->sec_sz = 0;
 342         sec->shdr->sh_entsize = 0;
 343
 344         /* SYMTAB */
 345         sec = add_dst_sec(linker, ".symtab");
 346         if (!sec)
 347                 return -ENOMEM;
 348
 349         sec->scn = elf_newscn(linker->elf);
 350         if (!sec->scn) {
 351                 pr_warn_elf("failed to create SYMTAB section");
 352                 return -EINVAL;
 353         }
 354
 355         sec->shdr = elf64_getshdr(sec->scn);
 356         if (!sec->shdr)
 357                 return -EINVAL;
 358
 359         sec->data = elf_newdata(sec->scn);
 360         if (!sec->data) {
 361                 pr_warn_elf("failed to create SYMTAB data");
 362                 return -EINVAL;
 363         }
 364
 365         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 366         if (str_off < 0)
 367                 return str_off;
 368
 369         sec->sec_idx = elf_ndxscn(sec->scn);
 370         linker->symtab_sec_idx = sec->sec_idx;
 371
 372         sec->shdr->sh_name = str_off;
 373         sec->shdr->sh_type = SHT_SYMTAB;
 374         sec->shdr->sh_flags = 0;
 375         sec->shdr->sh_offset = 0;
 376         sec->shdr->sh_link = linker->strtab_sec_idx;
 377         /* sh_info should be one greater than the index of the last local
 378          * symbol (i.e., binding is STB_LOCAL). But why and who cares?
 379          */
 380         sec->shdr->sh_info = 0;
 381         sec->shdr->sh_addralign = 8;
 382         sec->shdr->sh_entsize = sizeof(Elf64_Sym);
 383
 384         /* .BTF */
 385         linker->btf = btf__new_empty();
 386         err = libbpf_get_error(linker->btf);
 387         if (err)
 388                 return err;
 389
 390         /* add the special all-zero symbol */
 391         init_sym = add_new_sym(linker, NULL);
 392         if (!init_sym)
 393                 return -EINVAL;
 394
 395         init_sym->st_name = 0;
 396         init_sym->st_info = 0;
 397         init_sym->st_other = 0;
 398         init_sym->st_shndx = SHN_UNDEF;
 399         init_sym->st_value = 0;
 400         init_sym->st_size = 0;
 401
 402         return 0;
 403 }
 404
 405 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename)
 406 {
 407         struct src_obj obj = {};
 408         int err = 0;
 409
 410         if (!linker->elf)
 411                 return -EINVAL;
 412
 413         err = err ?: linker_load_obj_file(linker, filename, &obj);
 414         err = err ?: linker_append_sec_data(linker, &obj);
 415         err = err ?: linker_append_elf_syms(linker, &obj);
 416         err = err ?: linker_append_elf_relos(linker, &obj);
 417         err = err ?: linker_append_btf(linker, &obj);
 418         err = err ?: linker_append_btf_ext(linker, &obj);
 419
 420         /* free up src_obj resources */
 421         free(obj.btf_type_map);
 422         btf__free(obj.btf);
 423         btf_ext__free(obj.btf_ext);
 424         free(obj.secs);
 425         free(obj.sym_map);
 426         if (obj.elf)
 427                 elf_end(obj.elf);
 428         if (obj.fd >= 0)
 429                 close(obj.fd);
 430
 431         return err;
 432 }
 433
 434 static bool is_dwarf_sec_name(const char *name)
 435 {
 436         /* approximation, but the actual list is too long */
 437         return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
 438 }
 439
 440 static bool is_ignored_sec(struct src_sec *sec)
 441 {
 442         Elf64_Shdr *shdr = sec->shdr;
 443         const char *name = sec->sec_name;
 444
 445         /* no special handling of .strtab */
 446         if (shdr->sh_type == SHT_STRTAB)
 447                 return true;
 448
 449         /* ignore .llvm_addrsig section as well */
 450         if (shdr->sh_type == SHT_LLVM_ADDRSIG)
 451                 return true;
 452
 453         /* no subprograms will lead to an empty .text section, ignore it */
 454         if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
 455             strcmp(sec->sec_name, ".text") == 0)
 456                 return true;
 457
 458         /* DWARF sections */
 459         if (is_dwarf_sec_name(sec->sec_name))
 460                 return true;
 461
 462         if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
 463                 name += sizeof(".rel") - 1;
 464                 /* DWARF section relocations */
 465                 if (is_dwarf_sec_name(name))
 466                         return true;
 467
 468                 /* .BTF and .BTF.ext don't need relocations */
 469                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
 470                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
 471                         return true;
 472         }
 473
 474         return false;
 475 }
 476
 477 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
 478 {
 479         struct src_sec *secs = obj->secs, *sec;
 480         size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
 481
 482         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 483         if (!secs)
 484                 return NULL;
 485
 486         /* zero out newly allocated memory */
 487         memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
 488
 489         obj->secs = secs;
 490         obj->sec_cnt = new_cnt;
 491
 492         sec = &obj->secs[new_cnt - 1];
 493         sec->id = new_cnt - 1;
 494         sec->sec_name = sec_name;
 495
 496         return sec;
 497 }
 498
 499 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj)
 500 {
 501 #if __BYTE_ORDER == __LITTLE_ENDIAN
 502         const int host_endianness = ELFDATA2LSB;
 503 #elif __BYTE_ORDER == __BIG_ENDIAN
 504         const int host_endianness = ELFDATA2MSB;
 505 #else
 506 #error "Unknown __BYTE_ORDER"
 507 #endif
 508         int err = 0;
 509         Elf_Scn *scn;
 510         Elf_Data *data;
 511         Elf64_Ehdr *ehdr;
 512         Elf64_Shdr *shdr;
 513         struct src_sec *sec;
 514
 515         pr_debug("linker: adding object file '%s'...\n", filename);
 516
 517         obj->filename = filename;
 518
 519         obj->fd = open(filename, O_RDONLY);
 520         if (obj->fd < 0) {
 521                 err = -errno;
 522                 pr_warn("failed to open file '%s': %d\n", filename, err);
 523                 return err;
 524         }
 525         obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
 526         if (!obj->elf) {
 527                 err = -errno;
 528                 pr_warn_elf("failed to parse ELF file '%s'", filename);
 529                 return err;
 530         }
 531
 532         /* Sanity check ELF file high-level properties */
 533         ehdr = elf64_getehdr(obj->elf);
 534         if (!ehdr) {
 535                 err = -errno;
 536                 pr_warn_elf("failed to get ELF header for %s", filename);
 537                 return err;
 538         }
 539         if (ehdr->e_ident[EI_DATA] != host_endianness) {
 540                 err = -EOPNOTSUPP;
 541                 pr_warn_elf("unsupported byte order of ELF file %s", filename);
 542                 return err;
 543         }
 544         if (ehdr->e_type != ET_REL
 545             || ehdr->e_machine != EM_BPF
 546             || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
 547                 err = -EOPNOTSUPP;
 548                 pr_warn_elf("unsupported kind of ELF file %s", filename);
 549                 return err;
 550         }
 551
 552         if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
 553                 err = -errno;
 554                 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
 555                 return err;
 556         }
 557
 558         scn = NULL;
 559         while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
 560                 size_t sec_idx = elf_ndxscn(scn);
 561                 const char *sec_name;
 562
 563                 shdr = elf64_getshdr(scn);
 564                 if (!shdr) {
 565                         err = -errno;
 566                         pr_warn_elf("failed to get section #%zu header for %s",
 567                                     sec_idx, filename);
 568                         return err;
 569                 }
 570
 571                 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
 572                 if (!sec_name) {
 573                         err = -errno;
 574                         pr_warn_elf("failed to get section #%zu name for %s",
 575                                     sec_idx, filename);
 576                         return err;
 577                 }
 578
 579                 data = elf_getdata(scn, 0);
 580                 if (!data) {
 581                         err = -errno;
 582                         pr_warn_elf("failed to get section #%zu (%s) data from %s",
 583                                     sec_idx, sec_name, filename);
 584                         return err;
 585                 }
 586
 587                 sec = add_src_sec(obj, sec_name);
 588                 if (!sec)
 589                         return -ENOMEM;
 590
 591                 sec->scn = scn;
 592                 sec->shdr = shdr;
 593                 sec->data = data;
 594                 sec->sec_idx = elf_ndxscn(scn);
 595
 596                 if (is_ignored_sec(sec)) {
 597                         sec->skipped = true;
 598                         continue;
 599                 }
 600
 601                 switch (shdr->sh_type) {
 602                 case SHT_SYMTAB:
 603                         if (obj->symtab_sec_idx) {
 604                                 err = -EOPNOTSUPP;
 605                                 pr_warn("multiple SYMTAB sections found, not supported\n");
 606                                 return err;
 607                         }
 608                         obj->symtab_sec_idx = sec_idx;
 609                         break;
 610                 case SHT_STRTAB:
 611                         /* we'll construct our own string table */
 612                         break;
 613                 case SHT_PROGBITS:
 614                         if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
 615                                 obj->btf = btf__new(data->d_buf, shdr->sh_size);
 616                                 err = libbpf_get_error(obj->btf);
 617                                 if (err) {
 618                                         pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
 619                                         return err;
 620                                 }
 621                                 sec->skipped = true;
 622                                 continue;
 623                         }
 624                         if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
 625                                 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
 626                                 err = libbpf_get_error(obj->btf_ext);
 627                                 if (err) {
 628                                         pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
 629                                         return err;
 630                                 }
 631                                 sec->skipped = true;
 632                                 continue;
 633                         }
 634
 635                         /* data & code */
 636                         break;
 637                 case SHT_NOBITS:
 638                         /* BSS */
 639                         break;
 640                 case SHT_REL:
 641                         /* relocations */
 642                         break;
 643                 default:
 644                         pr_warn("unrecognized section #%zu (%s) in %s\n",
 645                                 sec_idx, sec_name, filename);
 646                         err = -EINVAL;
 647                         return err;
 648                 }
 649         }
 650
 651         err = err ?: linker_sanity_check_elf(obj);
 652         err = err ?: linker_sanity_check_btf(obj);
 653         err = err ?: linker_sanity_check_btf_ext(obj);
 654         err = err ?: linker_fixup_btf(obj);
 655
 656         return err;
 657 }
 658
 659 static bool is_pow_of_2(size_t x)
 660 {
 661         return x && (x & (x - 1)) == 0;
 662 }
 663
 664 static int linker_sanity_check_elf(struct src_obj *obj)
 665 {
 666         struct src_sec *sec;
 667         int i, err;
 668
 669         if (!obj->symtab_sec_idx) {
 670                 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
 671                 return -EINVAL;
 672         }
 673         if (!obj->shstrs_sec_idx) {
 674                 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
 675                 return -EINVAL;
 676         }
 677
 678         for (i = 1; i < obj->sec_cnt; i++) {
 679                 sec = &obj->secs[i];
 680
 681                 if (sec->sec_name[0] == '\0') {
 682                         pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
 683                         return -EINVAL;
 684                 }
 685
 686                 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign))
 687                         return -EINVAL;
 688                 if (sec->shdr->sh_addralign != sec->data->d_align)
 689                         return -EINVAL;
 690
 691                 if (sec->shdr->sh_size != sec->data->d_size)
 692                         return -EINVAL;
 693
 694                 switch (sec->shdr->sh_type) {
 695                 case SHT_SYMTAB:
 696                         err = linker_sanity_check_elf_symtab(obj, sec);
 697                         if (err)
 698                                 return err;
 699                         break;
 700                 case SHT_STRTAB:
 701                         break;
 702                 case SHT_PROGBITS:
 703                         if (sec->shdr->sh_flags & SHF_EXECINSTR) {
 704                                 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0)
 705                                         return -EINVAL;
 706                         }
 707                         break;
 708                 case SHT_NOBITS:
 709                         break;
 710                 case SHT_REL:
 711                         err = linker_sanity_check_elf_relos(obj, sec);
 712                         if (err)
 713                                 return err;
 714                         break;
 715                 case SHT_LLVM_ADDRSIG:
 716                         break;
 717                 default:
 718                         pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
 719                                 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
 720                         return -EINVAL;
 721                 }
 722         }
 723
 724         return 0;
 725 }
 726
 727 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
 728 {
 729         struct src_sec *link_sec;
 730         Elf64_Sym *sym;
 731         int i, n;
 732
 733         if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
 734                 return -EINVAL;
 735         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 736                 return -EINVAL;
 737
 738         if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
 739                 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
 740                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 741                 return -EINVAL;
 742         }
 743         link_sec = &obj->secs[sec->shdr->sh_link];
 744         if (link_sec->shdr->sh_type != SHT_STRTAB) {
 745                 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
 746                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 747                 return -EINVAL;
 748         }
 749
 750         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 751         sym = sec->data->d_buf;
 752         for (i = 0; i < n; i++, sym++) {
 753                 if (sym->st_shndx
 754                     && sym->st_shndx < SHN_LORESERVE
 755                     && sym->st_shndx >= obj->sec_cnt) {
 756                         pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
 757                                 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
 758                         return -EINVAL;
 759                 }
 760                 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION) {
 761                         if (sym->st_value != 0)
 762                                 return -EINVAL;
 763                         continue;
 764                 }
 765         }
 766
 767         return 0;
 768 }
 769
 770 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
 771 {
 772         struct src_sec *link_sec, *sym_sec;
 773         Elf64_Rel *relo;
 774         int i, n;
 775
 776         if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
 777                 return -EINVAL;
 778         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 779                 return -EINVAL;
 780
 781         /* SHT_REL's sh_link should point to SYMTAB */
 782         if (sec->shdr->sh_link != obj->symtab_sec_idx) {
 783                 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
 784                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 785                 return -EINVAL;
 786         }
 787
 788         /* SHT_REL's sh_info points to relocated section */
 789         if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
 790                 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
 791                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 792                 return -EINVAL;
 793         }
 794         link_sec = &obj->secs[sec->shdr->sh_info];
 795
 796         /* .rel<secname> -> <secname> pattern is followed */
 797         if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
 798             || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
 799                 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
 800                         sec->sec_idx, obj->filename);
 801                 return -EINVAL;
 802         }
 803
 804         /* don't further validate relocations for ignored sections */
 805         if (link_sec->skipped)
 806                 return 0;
 807
 808         /* relocatable section is data or instructions */
 809         if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
 810                 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
 811                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 812                 return -EINVAL;
 813         }
 814
 815         /* check sanity of each relocation */
 816         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 817         relo = sec->data->d_buf;
 818         sym_sec = &obj->secs[obj->symtab_sec_idx];
 819         for (i = 0; i < n; i++, relo++) {
 820                 size_t sym_idx = ELF64_R_SYM(relo->r_info);
 821                 size_t sym_type = ELF64_R_TYPE(relo->r_info);
 822
 823                 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32) {
 824                         pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
 825                                 i, sec->sec_idx, sym_type, obj->filename);
 826                         return -EINVAL;
 827                 }
 828
 829                 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
 830                         pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
 831                                 i, sec->sec_idx, sym_idx, obj->filename);
 832                         return -EINVAL;
 833                 }
 834
 835                 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
 836                         if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
 837                                 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
 838                                         i, sec->sec_idx, sym_idx, obj->filename);
 839                                 return -EINVAL;
 840                         }
 841                 }
 842         }
 843
 844         return 0;
 845 }
 846
 847 static int check_btf_type_id(__u32 *type_id, void *ctx)
 848 {
 849         struct btf *btf = ctx;
 850
 851         if (*type_id > btf__get_nr_types(btf))
 852                 return -EINVAL;
 853
 854         return 0;
 855 }
 856
 857 static int check_btf_str_off(__u32 *str_off, void *ctx)
 858 {
 859         struct btf *btf = ctx;
 860         const char *s;
 861
 862         s = btf__str_by_offset(btf, *str_off);
 863
 864         if (!s)
 865                 return -EINVAL;
 866
 867         return 0;
 868 }
 869
 870 static int linker_sanity_check_btf(struct src_obj *obj)
 871 {
 872         struct btf_type *t;
 873         int i, n, err = 0;
 874
 875         if (!obj->btf)
 876                 return 0;
 877
 878         n = btf__get_nr_types(obj->btf);
 879         for (i = 1; i <= n; i++) {
 880                 t = btf_type_by_id(obj->btf, i);
 881
 882                 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
 883                 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
 884                 if (err)
 885                         return err;
 886         }
 887
 888         return 0;
 889 }
 890
 891 static int linker_sanity_check_btf_ext(struct src_obj *obj)
 892 {
 893         int err = 0;
 894
 895         if (!obj->btf_ext)
 896                 return 0;
 897
 898         /* can't use .BTF.ext without .BTF */
 899         if (!obj->btf)
 900                 return -EINVAL;
 901
 902         err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
 903         err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
 904         if (err)
 905                 return err;
 906
 907         return 0;
 908 }
 909
 910 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
 911 {
 912         Elf_Scn *scn;
 913         Elf_Data *data;
 914         Elf64_Shdr *shdr;
 915         int name_off;
 916
 917         dst_sec->sec_sz = 0;
 918         dst_sec->sec_idx = 0;
 919
 920         /* ephemeral sections are just thin section shells lacking most parts */
 921         if (src_sec->ephemeral)
 922                 return 0;
 923
 924         scn = elf_newscn(linker->elf);
 925         if (!scn)
 926                 return -ENOMEM;
 927         data = elf_newdata(scn);
 928         if (!data)
 929                 return -ENOMEM;
 930         shdr = elf64_getshdr(scn);
 931         if (!shdr)
 932                 return -ENOMEM;
 933
 934         dst_sec->scn = scn;
 935         dst_sec->shdr = shdr;
 936         dst_sec->data = data;
 937         dst_sec->sec_idx = elf_ndxscn(scn);
 938
 939         name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
 940         if (name_off < 0)
 941                 return name_off;
 942
 943         shdr->sh_name = name_off;
 944         shdr->sh_type = src_sec->shdr->sh_type;
 945         shdr->sh_flags = src_sec->shdr->sh_flags;
 946         shdr->sh_size = 0;
 947         /* sh_link and sh_info have different meaning for different types of
 948          * sections, so we leave it up to the caller code to fill them in, if
 949          * necessary
 950          */
 951         shdr->sh_link = 0;
 952         shdr->sh_info = 0;
 953         shdr->sh_addralign = src_sec->shdr->sh_addralign;
 954         shdr->sh_entsize = src_sec->shdr->sh_entsize;
 955
 956         data->d_type = src_sec->data->d_type;
 957         data->d_size = 0;
 958         data->d_buf = NULL;
 959         data->d_align = src_sec->data->d_align;
 960         data->d_off = 0;
 961
 962         return 0;
 963 }
 964
 965 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
 966 {
 967         struct dst_sec *sec;
 968         int i;
 969
 970         for (i = 1; i < linker->sec_cnt; i++) {
 971                 sec = &linker->secs[i];
 972
 973                 if (strcmp(sec->sec_name, sec_name) == 0)
 974                         return sec;
 975         }
 976
 977         return NULL;
 978 }
 979
 980 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
 981 {
 982         if (dst->shdr->sh_type != src->shdr->sh_type) {
 983                 pr_warn("sec %s types mismatch\n", dst->sec_name);
 984                 return false;
 985         }
 986         if (dst->shdr->sh_flags != src->shdr->sh_flags) {
 987                 pr_warn("sec %s flags mismatch\n", dst->sec_name);
 988                 return false;
 989         }
 990         if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
 991                 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
 992                 return false;
 993         }
 994
 995         return true;
 996 }
 997
 998 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
 999 {
1000         if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1001                 return false;
1002         if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1003                 return false;
1004         return true;
1005 }
1006
1007 static int extend_sec(struct dst_sec *dst, struct src_sec *src)
1008 {
1009         void *tmp;
1010         size_t dst_align = dst->shdr->sh_addralign;
1011         size_t src_align = src->shdr->sh_addralign;
1012         size_t dst_align_sz, dst_final_sz;
1013
1014         if (dst_align == 0)
1015                 dst_align = 1;
1016         if (dst_align < src_align)
1017                 dst_align = src_align;
1018
1019         dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1020
1021         /* no need to re-align final size */
1022         dst_final_sz = dst_align_sz + src->shdr->sh_size;
1023
1024         if (src->shdr->sh_type != SHT_NOBITS) {
1025                 tmp = realloc(dst->raw_data, dst_final_sz);
1026                 if (!tmp)
1027                         return -ENOMEM;
1028                 dst->raw_data = tmp;
1029
1030                 /* pad dst section, if it's alignment forced size increase */
1031                 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1032                 /* now copy src data at a properly aligned offset */
1033                 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1034         }
1035
1036         dst->sec_sz = dst_final_sz;
1037         dst->shdr->sh_size = dst_final_sz;
1038         dst->data->d_size = dst_final_sz;
1039
1040         dst->shdr->sh_addralign = dst_align;
1041         dst->data->d_align = dst_align;
1042
1043         src->dst_off = dst_align_sz;
1044
1045         return 0;
1046 }
1047
1048 static bool is_data_sec(struct src_sec *sec)
1049 {
1050         if (!sec || sec->skipped)
1051                 return false;
1052         /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1053         if (sec->ephemeral)
1054                 return true;
1055         return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1056 }
1057
1058 static bool is_relo_sec(struct src_sec *sec)
1059 {
1060         if (!sec || sec->skipped || sec->ephemeral)
1061                 return false;
1062         return sec->shdr->sh_type == SHT_REL;
1063 }
1064
1065 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1066 {
1067         int i, err;
1068
1069         for (i = 1; i < obj->sec_cnt; i++) {
1070                 struct src_sec *src_sec;
1071                 struct dst_sec *dst_sec;
1072
1073                 src_sec = &obj->secs[i];
1074                 if (!is_data_sec(src_sec))
1075                         continue;
1076
1077                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1078                 if (!dst_sec) {
1079                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
1080                         if (!dst_sec)
1081                                 return -ENOMEM;
1082                         err = init_sec(linker, dst_sec, src_sec);
1083                         if (err) {
1084                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1085                                 return err;
1086                         }
1087                 } else {
1088                         if (!secs_match(dst_sec, src_sec)) {
1089                                 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1090                                 return -1;
1091                         }
1092
1093                         /* "license" and "version" sections are deduped */
1094                         if (strcmp(src_sec->sec_name, "license") == 0
1095                             || strcmp(src_sec->sec_name, "version") == 0) {
1096                                 if (!sec_content_is_same(dst_sec, src_sec)) {
1097                                         pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1098                                         return -EINVAL;
1099                                 }
1100                                 src_sec->skipped = true;
1101                                 src_sec->dst_id = dst_sec->id;
1102                                 continue;
1103                         }
1104                 }
1105
1106                 /* record mapped section index */
1107                 src_sec->dst_id = dst_sec->id;
1108
1109                 if (src_sec->ephemeral)
1110                         continue;
1111
1112                 err = extend_sec(dst_sec, src_sec);
1113                 if (err)
1114                         return err;
1115         }
1116
1117         return 0;
1118 }
1119
1120 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1121 {
1122         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1123         Elf64_Sym *sym = symtab->data->d_buf, *dst_sym;
1124         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
1125         int str_sec_idx = symtab->shdr->sh_link;
1126
1127         obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1128         if (!obj->sym_map)
1129                 return -ENOMEM;
1130
1131         for (i = 0; i < n; i++, sym++) {
1132                 struct src_sec *src_sec = NULL;
1133                 struct dst_sec *dst_sec = NULL;
1134                 const char *sym_name;
1135                 size_t dst_sym_idx;
1136                 int name_off;
1137
1138                 /* we already have all-zero initial symbol */
1139                 if (sym->st_name == 0 && sym->st_info == 0 &&
1140                     sym->st_other == 0 && sym->st_shndx == SHN_UNDEF &&
1141                     sym->st_value == 0 && sym->st_size ==0)
1142                         continue;
1143
1144                 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1145                 if (!sym_name) {
1146                         pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1147                         return -1;
1148                 }
1149
1150                 if (sym->st_shndx && sym->st_shndx < SHN_LORESERVE) {
1151                         src_sec = &obj->secs[sym->st_shndx];
1152                         if (src_sec->skipped)
1153                                 continue;
1154                         dst_sec = &linker->secs[src_sec->dst_id];
1155
1156                         /* allow only one STT_SECTION symbol per section */
1157                         if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sec->sec_sym_idx) {
1158                                 obj->sym_map[i] = dst_sec->sec_sym_idx;
1159                                 continue;
1160                         }
1161                 }
1162
1163                 name_off = strset__add_str(linker->strtab_strs, sym_name);
1164                 if (name_off < 0)
1165                         return name_off;
1166
1167                 dst_sym = add_new_sym(linker, &dst_sym_idx);
1168                 if (!dst_sym)
1169                         return -ENOMEM;
1170
1171                 dst_sym->st_name = name_off;
1172                 dst_sym->st_info = sym->st_info;
1173                 dst_sym->st_other = sym->st_other;
1174                 dst_sym->st_shndx = src_sec ? dst_sec->sec_idx : sym->st_shndx;
1175                 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1176                 dst_sym->st_size = sym->st_size;
1177
1178                 obj->sym_map[i] = dst_sym_idx;
1179
1180                 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sym) {
1181                         dst_sec->sec_sym_idx = dst_sym_idx;
1182                         dst_sym->st_value = 0;
1183                 }
1184
1185         }
1186
1187         return 0;
1188 }
1189
1190 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
1191 {
1192         struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
1193         struct dst_sec *dst_symtab = &linker->secs[linker->symtab_sec_idx];
1194         int i, err;
1195
1196         for (i = 1; i < obj->sec_cnt; i++) {
1197                 struct src_sec *src_sec, *src_linked_sec;
1198                 struct dst_sec *dst_sec, *dst_linked_sec;
1199                 Elf64_Rel *src_rel, *dst_rel;
1200                 int j, n;
1201
1202                 src_sec = &obj->secs[i];
1203                 if (!is_relo_sec(src_sec))
1204                         continue;
1205
1206                 /* shdr->sh_info points to relocatable section */
1207                 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
1208                 if (src_linked_sec->skipped)
1209                         continue;
1210
1211                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1212                 if (!dst_sec) {
1213                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
1214                         if (!dst_sec)
1215                                 return -ENOMEM;
1216                         err = init_sec(linker, dst_sec, src_sec);
1217                         if (err) {
1218                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1219                                 return err;
1220                         }
1221                 } else if (!secs_match(dst_sec, src_sec)) {
1222                         pr_warn("sections %s are not compatible\n", src_sec->sec_name);
1223                         return -1;
1224                 }
1225
1226                 /* shdr->sh_link points to SYMTAB */
1227                 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
1228
1229                 /* shdr->sh_info points to relocated section */
1230                 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
1231                 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
1232
1233                 src_sec->dst_id = dst_sec->id;
1234                 err = extend_sec(dst_sec, src_sec);
1235                 if (err)
1236                         return err;
1237
1238                 src_rel = src_sec->data->d_buf;
1239                 dst_rel = dst_sec->raw_data + src_sec->dst_off;
1240                 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
1241                 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
1242                         size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info);
1243                         size_t sym_type = ELF64_R_TYPE(src_rel->r_info);
1244                         Elf64_Sym *src_sym, *dst_sym;
1245                         size_t dst_sym_idx;
1246
1247                         src_sym_idx = ELF64_R_SYM(src_rel->r_info);
1248                         src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
1249
1250                         dst_sym_idx = obj->sym_map[src_sym_idx];
1251                         dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx;
1252                         dst_rel->r_offset += src_linked_sec->dst_off;
1253                         sym_type = ELF64_R_TYPE(src_rel->r_info);
1254                         dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
1255
1256                         if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
1257                                 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
1258                                 struct bpf_insn *insn;
1259
1260                                 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
1261                                         /* calls to the very first static function inside
1262                                          * .text section at offset 0 will
1263                                          * reference section symbol, not the
1264                                          * function symbol. Fix that up,
1265                                          * otherwise it won't be possible to
1266                                          * relocate calls to two different
1267                                          * static functions with the same name
1268                                          * (rom two different object files)
1269                                          */
1270                                         insn = dst_linked_sec->raw_data + dst_rel->r_offset;
1271                                         if (insn->code == (BPF_JMP | BPF_CALL))
1272                                                 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
1273                                         else
1274                                                 insn->imm += sec->dst_off;
1275                                 } else {
1276                                         pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
1277                                         return -EINVAL;
1278                                 }
1279                         }
1280
1281                 }
1282         }
1283
1284         return 0;
1285 }
1286
1287 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1288 {
1289         struct src_sec *sec;
1290         int i;
1291
1292         for (i = 1; i < obj->sec_cnt; i++) {
1293                 sec = &obj->secs[i];
1294
1295                 if (strcmp(sec->sec_name, sec_name) == 0)
1296                         return sec;
1297         }
1298
1299         return NULL;
1300 }
1301
1302 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
1303                                    int sym_type, const char *sym_name)
1304 {
1305         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1306         Elf64_Sym *sym = symtab->data->d_buf;
1307         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
1308         int str_sec_idx = symtab->shdr->sh_link;
1309         const char *name;
1310
1311         for (i = 0; i < n; i++, sym++) {
1312                 if (sym->st_shndx != sec_idx)
1313                         continue;
1314                 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
1315                         continue;
1316
1317                 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1318                 if (!name)
1319                         return NULL;
1320
1321                 if (strcmp(sym_name, name) != 0)
1322                         continue;
1323
1324                 return sym;
1325         }
1326
1327         return NULL;
1328 }
1329
1330 static int linker_fixup_btf(struct src_obj *obj)
1331 {
1332         const char *sec_name;
1333         struct src_sec *sec;
1334         int i, j, n, m;
1335
1336         if (!obj->btf)
1337                 return 0;
1338
1339         n = btf__get_nr_types(obj->btf);
1340         for (i = 1; i <= n; i++) {
1341                 struct btf_var_secinfo *vi;
1342                 struct btf_type *t;
1343
1344                 t = btf_type_by_id(obj->btf, i);
1345                 if (btf_kind(t) != BTF_KIND_DATASEC)
1346                         continue;
1347
1348                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1349                 sec = find_src_sec_by_name(obj, sec_name);
1350                 if (sec) {
1351                         /* record actual section size, unless ephemeral */
1352                         if (sec->shdr)
1353                                 t->size = sec->shdr->sh_size;
1354                 } else {
1355                         /* BTF can have some sections that are not represented
1356                          * in ELF, e.g., .kconfig and .ksyms, which are used
1357                          * for special extern variables.  Here we'll
1358                          * pre-create "section shells" for them to be able to
1359                          * keep track of extra per-section metadata later
1360                          * (e.g., BTF variables).
1361                          */
1362                         sec = add_src_sec(obj, sec_name);
1363                         if (!sec)
1364                                 return -ENOMEM;
1365
1366                         sec->ephemeral = true;
1367                         sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
1368                 }
1369
1370                 /* remember ELF section and its BTF type ID match */
1371                 sec->sec_type_id = i;
1372
1373                 /* fix up variable offsets */
1374                 vi = btf_var_secinfos(t);
1375                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1376                         const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
1377                         const char *var_name = btf__str_by_offset(obj->btf, vt->name_off);
1378                         int var_linkage = btf_var(vt)->linkage;
1379                         Elf64_Sym *sym;
1380
1381                         /* no need to patch up static or extern vars */
1382                         if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
1383                                 continue;
1384
1385                         sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
1386                         if (!sym) {
1387                                 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
1388                                 return -ENOENT;
1389                         }
1390
1391                         vi->offset = sym->st_value;
1392                 }
1393         }
1394
1395         return 0;
1396 }
1397
1398 static int remap_type_id(__u32 *type_id, void *ctx)
1399 {
1400         int *id_map = ctx;
1401
1402         *type_id = id_map[*type_id];
1403
1404         return 0;
1405 }
1406
1407 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
1408 {
1409         const struct btf_type *t;
1410         int i, j, n, start_id, id;
1411
1412         if (!obj->btf)
1413                 return 0;
1414
1415         start_id = btf__get_nr_types(linker->btf) + 1;
1416         n = btf__get_nr_types(obj->btf);
1417
1418         obj->btf_type_map = calloc(n + 1, sizeof(int));
1419         if (!obj->btf_type_map)
1420                 return -ENOMEM;
1421
1422         for (i = 1; i <= n; i++) {
1423                 t = btf__type_by_id(obj->btf, i);
1424
1425                 /* DATASECs are handled specially below */
1426                 if (btf_kind(t) == BTF_KIND_DATASEC)
1427                         continue;
1428
1429                 id = btf__add_type(linker->btf, obj->btf, t);
1430                 if (id < 0) {
1431                         pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
1432                         return id;
1433                 }
1434
1435                 obj->btf_type_map[i] = id;
1436         }
1437
1438         /* remap all the types except DATASECs */
1439         n = btf__get_nr_types(linker->btf);
1440         for (i = start_id; i <= n; i++) {
1441                 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
1442
1443                 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
1444                         return -EINVAL;
1445         }
1446
1447         /* append DATASEC info */
1448         for (i = 1; i < obj->sec_cnt; i++) {
1449                 struct src_sec *src_sec;
1450                 struct dst_sec *dst_sec;
1451                 const struct btf_var_secinfo *src_var;
1452                 struct btf_var_secinfo *dst_var;
1453
1454                 src_sec = &obj->secs[i];
1455                 if (!src_sec->sec_type_id || src_sec->skipped)
1456                         continue;
1457                 dst_sec = &linker->secs[src_sec->dst_id];
1458
1459                 /* Mark section as having BTF regardless of the presence of
1460                  * variables. In some cases compiler might generate empty BTF
1461                  * with no variables information. E.g., when promoting local
1462                  * array/structure variable initial values and BPF object
1463                  * file otherwise has no read-only static variables in
1464                  * .rodata. We need to preserve such empty BTF and just set
1465                  * correct section size.
1466                  */
1467                 dst_sec->has_btf = true;
1468
1469                 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
1470                 src_var = btf_var_secinfos(t);
1471                 n = btf_vlen(t);
1472                 for (j = 0; j < n; j++, src_var++) {
1473                         void *sec_vars = dst_sec->sec_vars;
1474
1475                         sec_vars = libbpf_reallocarray(sec_vars,
1476                                                        dst_sec->sec_var_cnt + 1,
1477                                                        sizeof(*dst_sec->sec_vars));
1478                         if (!sec_vars)
1479                                 return -ENOMEM;
1480
1481                         dst_sec->sec_vars = sec_vars;
1482                         dst_sec->sec_var_cnt++;
1483
1484                         dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
1485                         dst_var->type = obj->btf_type_map[src_var->type];
1486                         dst_var->size = src_var->size;
1487                         dst_var->offset = src_sec->dst_off + src_var->offset;
1488                 }
1489         }
1490
1491         return 0;
1492 }
1493
1494 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
1495 {
1496         void *tmp;
1497
1498         tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
1499         if (!tmp)
1500                 return NULL;
1501         ext_data->recs = tmp;
1502
1503         tmp += ext_data->rec_cnt * ext_data->rec_sz;
1504         memcpy(tmp, src_rec, ext_data->rec_sz);
1505
1506         ext_data->rec_cnt++;
1507
1508         return tmp;
1509 }
1510
1511 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
1512 {
1513         const struct btf_ext_info_sec *ext_sec;
1514         const char *sec_name, *s;
1515         struct src_sec *src_sec;
1516         struct dst_sec *dst_sec;
1517         int rec_sz, str_off, i;
1518
1519         if (!obj->btf_ext)
1520                 return 0;
1521
1522         rec_sz = obj->btf_ext->func_info.rec_size;
1523         for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
1524                 struct bpf_func_info_min *src_rec, *dst_rec;
1525
1526                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
1527                 src_sec = find_src_sec_by_name(obj, sec_name);
1528                 if (!src_sec) {
1529                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
1530                         return -EINVAL;
1531                 }
1532                 dst_sec = &linker->secs[src_sec->dst_id];
1533
1534                 if (dst_sec->func_info.rec_sz == 0)
1535                         dst_sec->func_info.rec_sz = rec_sz;
1536                 if (dst_sec->func_info.rec_sz != rec_sz) {
1537                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
1538                         return -EINVAL;
1539                 }
1540
1541                 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
1542                         dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
1543                         if (!dst_rec)
1544                                 return -ENOMEM;
1545
1546                         dst_rec->insn_off += src_sec->dst_off;
1547                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
1548                 }
1549         }
1550
1551         rec_sz = obj->btf_ext->line_info.rec_size;
1552         for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
1553                 struct bpf_line_info_min *src_rec, *dst_rec;
1554
1555                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
1556                 src_sec = find_src_sec_by_name(obj, sec_name);
1557                 if (!src_sec) {
1558                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
1559                         return -EINVAL;
1560                 }
1561                 dst_sec = &linker->secs[src_sec->dst_id];
1562
1563                 if (dst_sec->line_info.rec_sz == 0)
1564                         dst_sec->line_info.rec_sz = rec_sz;
1565                 if (dst_sec->line_info.rec_sz != rec_sz) {
1566                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
1567                         return -EINVAL;
1568                 }
1569
1570                 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
1571                         dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
1572                         if (!dst_rec)
1573                                 return -ENOMEM;
1574
1575                         dst_rec->insn_off += src_sec->dst_off;
1576
1577                         s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
1578                         str_off = btf__add_str(linker->btf, s);
1579                         if (str_off < 0)
1580                                 return -ENOMEM;
1581                         dst_rec->file_name_off = str_off;
1582
1583                         s = btf__str_by_offset(obj->btf, src_rec->line_off);
1584                         str_off = btf__add_str(linker->btf, s);
1585                         if (str_off < 0)
1586                                 return -ENOMEM;
1587                         dst_rec->line_off = str_off;
1588
1589                         /* dst_rec->line_col is fine */
1590                 }
1591         }
1592
1593         rec_sz = obj->btf_ext->core_relo_info.rec_size;
1594         for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
1595                 struct bpf_core_relo *src_rec, *dst_rec;
1596
1597                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
1598                 src_sec = find_src_sec_by_name(obj, sec_name);
1599                 if (!src_sec) {
1600                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
1601                         return -EINVAL;
1602                 }
1603                 dst_sec = &linker->secs[src_sec->dst_id];
1604
1605                 if (dst_sec->core_relo_info.rec_sz == 0)
1606                         dst_sec->core_relo_info.rec_sz = rec_sz;
1607                 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
1608                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
1609                         return -EINVAL;
1610                 }
1611
1612                 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
1613                         dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
1614                         if (!dst_rec)
1615                                 return -ENOMEM;
1616
1617                         dst_rec->insn_off += src_sec->dst_off;
1618                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
1619
1620                         s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
1621                         str_off = btf__add_str(linker->btf, s);
1622                         if (str_off < 0)
1623                                 return -ENOMEM;
1624                         dst_rec->access_str_off = str_off;
1625
1626                         /* dst_rec->kind is fine */
1627                 }
1628         }
1629
1630         return 0;
1631 }
1632
1633 int bpf_linker__finalize(struct bpf_linker *linker)
1634 {
1635         struct dst_sec *sec;
1636         size_t strs_sz;
1637         const void *strs;
1638         int err, i;
1639
1640         if (!linker->elf)
1641                 return -EINVAL;
1642
1643         err = finalize_btf(linker);
1644         if (err)
1645                 return err;
1646
1647         /* Finalize strings */
1648         strs_sz = strset__data_size(linker->strtab_strs);
1649         strs = strset__data(linker->strtab_strs);
1650
1651         sec = &linker->secs[linker->strtab_sec_idx];
1652         sec->data->d_align = 1;
1653         sec->data->d_off = 0LL;
1654         sec->data->d_buf = (void *)strs;
1655         sec->data->d_type = ELF_T_BYTE;
1656         sec->data->d_size = strs_sz;
1657         sec->shdr->sh_size = strs_sz;
1658
1659         for (i = 1; i < linker->sec_cnt; i++) {
1660                 sec = &linker->secs[i];
1661
1662                 /* STRTAB is handled specially above */
1663                 if (sec->sec_idx == linker->strtab_sec_idx)
1664                         continue;
1665
1666                 /* special ephemeral sections (.ksyms, .kconfig, etc) */
1667                 if (!sec->scn)
1668                         continue;
1669
1670                 sec->data->d_buf = sec->raw_data;
1671         }
1672
1673         /* Finalize ELF layout */
1674         if (elf_update(linker->elf, ELF_C_NULL) < 0) {
1675                 err = -errno;
1676                 pr_warn_elf("failed to finalize ELF layout");
1677                 return err;
1678         }
1679
1680         /* Write out final ELF contents */
1681         if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
1682                 err = -errno;
1683                 pr_warn_elf("failed to write ELF contents");
1684                 return err;
1685         }
1686
1687         elf_end(linker->elf);
1688         close(linker->fd);
1689
1690         linker->elf = NULL;
1691         linker->fd = -1;
1692
1693         return 0;
1694 }
1695
1696 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
1697                              size_t align, const void *raw_data, size_t raw_sz)
1698 {
1699         Elf_Scn *scn;
1700         Elf_Data *data;
1701         Elf64_Shdr *shdr;
1702         int name_off;
1703
1704         name_off = strset__add_str(linker->strtab_strs, sec_name);
1705         if (name_off < 0)
1706                 return name_off;
1707
1708         scn = elf_newscn(linker->elf);
1709         if (!scn)
1710                 return -ENOMEM;
1711         data = elf_newdata(scn);
1712         if (!data)
1713                 return -ENOMEM;
1714         shdr = elf64_getshdr(scn);
1715         if (!shdr)
1716                 return -EINVAL;
1717
1718         shdr->sh_name = name_off;
1719         shdr->sh_type = SHT_PROGBITS;
1720         shdr->sh_flags = 0;
1721         shdr->sh_size = raw_sz;
1722         shdr->sh_link = 0;
1723         shdr->sh_info = 0;
1724         shdr->sh_addralign = align;
1725         shdr->sh_entsize = 0;
1726
1727         data->d_type = ELF_T_BYTE;
1728         data->d_size = raw_sz;
1729         data->d_buf = (void *)raw_data;
1730         data->d_align = align;
1731         data->d_off = 0;
1732
1733         return 0;
1734 }
1735
1736 static int finalize_btf(struct bpf_linker *linker)
1737 {
1738         struct btf *btf = linker->btf;
1739         const void *raw_data;
1740         int i, j, id, err;
1741         __u32 raw_sz;
1742
1743         /* bail out if no BTF data was produced */
1744         if (btf__get_nr_types(linker->btf) == 0)
1745                 return 0;
1746
1747         for (i = 1; i < linker->sec_cnt; i++) {
1748                 struct dst_sec *sec = &linker->secs[i];
1749
1750                 if (!sec->has_btf)
1751                         continue;
1752
1753                 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
1754                 if (id < 0) {
1755                         pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
1756                                 sec->sec_name, id);
1757                         return id;
1758                 }
1759
1760                 for (j = 0; j < sec->sec_var_cnt; j++) {
1761                         struct btf_var_secinfo *vi = &sec->sec_vars[j];
1762
1763                         if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
1764                                 return -EINVAL;
1765                 }
1766         }
1767
1768         err = finalize_btf_ext(linker);
1769         if (err) {
1770                 pr_warn(".BTF.ext generation failed: %d\n", err);
1771                 return err;
1772         }
1773
1774         err = btf__dedup(linker->btf, linker->btf_ext, NULL);
1775         if (err) {
1776                 pr_warn("BTF dedup failed: %d\n", err);
1777                 return err;
1778         }
1779
1780         /* Emit .BTF section */
1781         raw_data = btf__get_raw_data(linker->btf, &raw_sz);
1782         if (!raw_data)
1783                 return -ENOMEM;
1784
1785         err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
1786         if (err) {
1787                 pr_warn("failed to write out .BTF ELF section: %d\n", err);
1788                 return err;
1789         }
1790
1791         /* Emit .BTF.ext section */
1792         if (linker->btf_ext) {
1793                 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
1794                 if (!raw_data)
1795                         return -ENOMEM;
1796
1797                 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
1798                 if (err) {
1799                         pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
1800                         return err;
1801                 }
1802         }
1803
1804         return 0;
1805 }
1806
1807 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
1808                              const char *sec_name, struct btf_ext_sec_data *sec_data)
1809 {
1810         struct btf_ext_info_sec *sec_info;
1811         void *cur = output;
1812         int str_off;
1813         size_t sz;
1814
1815         if (!sec_data->rec_cnt)
1816                 return 0;
1817
1818         str_off = btf__add_str(linker->btf, sec_name);
1819         if (str_off < 0)
1820                 return -ENOMEM;
1821
1822         sec_info = cur;
1823         sec_info->sec_name_off = str_off;
1824         sec_info->num_info = sec_data->rec_cnt;
1825         cur += sizeof(struct btf_ext_info_sec);
1826
1827         sz = sec_data->rec_cnt * sec_data->rec_sz;
1828         memcpy(cur, sec_data->recs, sz);
1829         cur += sz;
1830
1831         return cur - output;
1832 }
1833
1834 static int finalize_btf_ext(struct bpf_linker *linker)
1835 {
1836         size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
1837         size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
1838         struct btf_ext_header *hdr;
1839         void *data, *cur;
1840         int i, err, sz;
1841
1842         /* validate that all sections have the same .BTF.ext record sizes
1843          * and calculate total data size for each type of data (func info,
1844          * line info, core relos)
1845          */
1846         for (i = 1; i < linker->sec_cnt; i++) {
1847                 struct dst_sec *sec = &linker->secs[i];
1848
1849                 if (sec->func_info.rec_cnt) {
1850                         if (func_rec_sz == 0)
1851                                 func_rec_sz = sec->func_info.rec_sz;
1852                         if (func_rec_sz != sec->func_info.rec_sz) {
1853                                 pr_warn("mismatch in func_info record size %zu != %u\n",
1854                                         func_rec_sz, sec->func_info.rec_sz);
1855                                 return -EINVAL;
1856                         }
1857
1858                         funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
1859                 }
1860                 if (sec->line_info.rec_cnt) {
1861                         if (line_rec_sz == 0)
1862                                 line_rec_sz = sec->line_info.rec_sz;
1863                         if (line_rec_sz != sec->line_info.rec_sz) {
1864                                 pr_warn("mismatch in line_info record size %zu != %u\n",
1865                                         line_rec_sz, sec->line_info.rec_sz);
1866                                 return -EINVAL;
1867                         }
1868
1869                         lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
1870                 }
1871                 if (sec->core_relo_info.rec_cnt) {
1872                         if (core_relo_rec_sz == 0)
1873                                 core_relo_rec_sz = sec->core_relo_info.rec_sz;
1874                         if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
1875                                 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
1876                                         core_relo_rec_sz, sec->core_relo_info.rec_sz);
1877                                 return -EINVAL;
1878                         }
1879
1880                         core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
1881                 }
1882         }
1883
1884         if (!funcs_sz && !lines_sz && !core_relos_sz)
1885                 return 0;
1886
1887         total_sz += sizeof(struct btf_ext_header);
1888         if (funcs_sz) {
1889                 funcs_sz += sizeof(__u32); /* record size prefix */
1890                 total_sz += funcs_sz;
1891         }
1892         if (lines_sz) {
1893                 lines_sz += sizeof(__u32); /* record size prefix */
1894                 total_sz += lines_sz;
1895         }
1896         if (core_relos_sz) {
1897                 core_relos_sz += sizeof(__u32); /* record size prefix */
1898                 total_sz += core_relos_sz;
1899         }
1900
1901         cur = data = calloc(1, total_sz);
1902         if (!data)
1903                 return -ENOMEM;
1904
1905         hdr = cur;
1906         hdr->magic = BTF_MAGIC;
1907         hdr->version = BTF_VERSION;
1908         hdr->flags = 0;
1909         hdr->hdr_len = sizeof(struct btf_ext_header);
1910         cur += sizeof(struct btf_ext_header);
1911
1912         /* All offsets are in bytes relative to the end of this header */
1913         hdr->func_info_off = 0;
1914         hdr->func_info_len = funcs_sz;
1915         hdr->line_info_off = funcs_sz;
1916         hdr->line_info_len = lines_sz;
1917         hdr->core_relo_off = funcs_sz + lines_sz;
1918         hdr->core_relo_len = core_relos_sz;
1919
1920         if (funcs_sz) {
1921                 *(__u32 *)cur = func_rec_sz;
1922                 cur += sizeof(__u32);
1923
1924                 for (i = 1; i < linker->sec_cnt; i++) {
1925                         struct dst_sec *sec = &linker->secs[i];
1926
1927                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
1928                         if (sz < 0) {
1929                                 err = sz;
1930                                 goto out;
1931                         }
1932
1933                         cur += sz;
1934                 }
1935         }
1936
1937         if (lines_sz) {
1938                 *(__u32 *)cur = line_rec_sz;
1939                 cur += sizeof(__u32);
1940
1941                 for (i = 1; i < linker->sec_cnt; i++) {
1942                         struct dst_sec *sec = &linker->secs[i];
1943
1944                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
1945                         if (sz < 0) {
1946                                 err = sz;
1947                                 goto out;
1948                         }
1949
1950                         cur += sz;
1951                 }
1952         }
1953
1954         if (core_relos_sz) {
1955                 *(__u32 *)cur = core_relo_rec_sz;
1956                 cur += sizeof(__u32);
1957
1958                 for (i = 1; i < linker->sec_cnt; i++) {
1959                         struct dst_sec *sec = &linker->secs[i];
1960
1961                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
1962                         if (sz < 0) {
1963                                 err = sz;
1964                                 goto out;
1965                         }
1966
1967                         cur += sz;
1968                 }
1969         }
1970
1971         linker->btf_ext = btf_ext__new(data, total_sz);
1972         err = libbpf_get_error(linker->btf_ext);
1973         if (err) {
1974                 linker->btf_ext = NULL;
1975                 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
1976                 goto out;
1977         }
1978
1979 out:
1980         free(data);
1981         return err;
1982 }