tools/objtool/check.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
   4  */
   5
   6 #include <string.h>
   7 #include <stdlib.h>
   8 #include <inttypes.h>
   9 #include <sys/mman.h>
  10
  11 #include <arch/elf.h>
  12 #include <objtool/builtin.h>
  13 #include <objtool/cfi.h>
  14 #include <objtool/arch.h>
  15 #include <objtool/check.h>
  16 #include <objtool/special.h>
  17 #include <objtool/warn.h>
  18 #include <objtool/endianness.h>
  19
  20 #include <linux/objtool.h>
  21 #include <linux/hashtable.h>
  22 #include <linux/kernel.h>
  23 #include <linux/static_call_types.h>
  24
  25 struct alternative {
  26         struct list_head list;
  27         struct instruction *insn;
  28         bool skip_orig;
  29 };
  30
  31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
  32
  33 static struct cfi_init_state initial_func_cfi;
  34 static struct cfi_state init_cfi;
  35 static struct cfi_state func_cfi;
  36
  37 struct instruction *find_insn(struct objtool_file *file,
  38                               struct section *sec, unsigned long offset)
  39 {
  40         struct instruction *insn;
  41
  42         hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
  43                 if (insn->sec == sec && insn->offset == offset)
  44                         return insn;
  45         }
  46
  47         return NULL;
  48 }
  49
  50 static struct instruction *next_insn_same_sec(struct objtool_file *file,
  51                                               struct instruction *insn)
  52 {
  53         struct instruction *next = list_next_entry(insn, list);
  54
  55         if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
  56                 return NULL;
  57
  58         return next;
  59 }
  60
  61 static struct instruction *next_insn_same_func(struct objtool_file *file,
  62                                                struct instruction *insn)
  63 {
  64         struct instruction *next = list_next_entry(insn, list);
  65         struct symbol *func = insn->func;
  66
  67         if (!func)
  68                 return NULL;
  69
  70         if (&next->list != &file->insn_list && next->func == func)
  71                 return next;
  72
  73         /* Check if we're already in the subfunction: */
  74         if (func == func->cfunc)
  75                 return NULL;
  76
  77         /* Move to the subfunction: */
  78         return find_insn(file, func->cfunc->sec, func->cfunc->offset);
  79 }
  80
  81 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
  82                                                struct instruction *insn)
  83 {
  84         struct instruction *prev = list_prev_entry(insn, list);
  85
  86         if (&prev->list != &file->insn_list && prev->func == insn->func)
  87                 return prev;
  88
  89         return NULL;
  90 }
  91
  92 #define func_for_each_insn(file, func, insn)                            \
  93         for (insn = find_insn(file, func->sec, func->offset);           \
  94              insn;                                                      \
  95              insn = next_insn_same_func(file, insn))
  96
  97 #define sym_for_each_insn(file, sym, insn)                              \
  98         for (insn = find_insn(file, sym->sec, sym->offset);             \
  99              insn && &insn->list != &file->insn_list &&                 \
 100                 insn->sec == sym->sec &&                                \
 101                 insn->offset < sym->offset + sym->len;                  \
 102              insn = list_next_entry(insn, list))
 103
 104 #define sym_for_each_insn_continue_reverse(file, sym, insn)             \
 105         for (insn = list_prev_entry(insn, list);                        \
 106              &insn->list != &file->insn_list &&                         \
 107                 insn->sec == sym->sec && insn->offset >= sym->offset;   \
 108              insn = list_prev_entry(insn, list))
 109
 110 #define sec_for_each_insn_from(file, insn)                              \
 111         for (; insn; insn = next_insn_same_sec(file, insn))
 112
 113 #define sec_for_each_insn_continue(file, insn)                          \
 114         for (insn = next_insn_same_sec(file, insn); insn;               \
 115              insn = next_insn_same_sec(file, insn))
 116
 117 static bool is_jump_table_jump(struct instruction *insn)
 118 {
 119         struct alt_group *alt_group = insn->alt_group;
 120
 121         if (insn->jump_table)
 122                 return true;
 123
 124         /* Retpoline alternative for a jump table? */
 125         return alt_group && alt_group->orig_group &&
 126                alt_group->orig_group->first_insn->jump_table;
 127 }
 128
 129 static bool is_sibling_call(struct instruction *insn)
 130 {
 131         /*
 132          * Assume only ELF functions can make sibling calls.  This ensures
 133          * sibling call detection consistency between vmlinux.o and individual
 134          * objects.
 135          */
 136         if (!insn->func)
 137                 return false;
 138
 139         /* An indirect jump is either a sibling call or a jump to a table. */
 140         if (insn->type == INSN_JUMP_DYNAMIC)
 141                 return !is_jump_table_jump(insn);
 142
 143         /* add_jump_destinations() sets insn->call_dest for sibling calls. */
 144         return (is_static_jump(insn) && insn->call_dest);
 145 }
 146
 147 /*
 148  * This checks to see if the given function is a "noreturn" function.
 149  *
 150  * For global functions which are outside the scope of this object file, we
 151  * have to keep a manual list of them.
 152  *
 153  * For local functions, we have to detect them manually by simply looking for
 154  * the lack of a return instruction.
 155  */
 156 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 157                                 int recursion)
 158 {
 159         int i;
 160         struct instruction *insn;
 161         bool empty = true;
 162
 163         /*
 164          * Unfortunately these have to be hard coded because the noreturn
 165          * attribute isn't provided in ELF data.
 166          */
 167         static const char * const global_noreturns[] = {
 168                 "__stack_chk_fail",
 169                 "panic",
 170                 "do_exit",
 171                 "do_task_dead",
 172                 "kthread_exit",
 173                 "make_task_dead",
 174                 "__module_put_and_kthread_exit",
 175                 "kthread_complete_and_exit",
 176                 "__reiserfs_panic",
 177                 "lbug_with_loc",
 178                 "fortify_panic",
 179                 "usercopy_abort",
 180                 "machine_real_restart",
 181                 "rewind_stack_and_make_dead",
 182                 "kunit_try_catch_throw",
 183                 "xen_start_kernel",
 184                 "cpu_bringup_and_idle",
 185                 "do_group_exit",
 186                 "stop_this_cpu",
 187                 "__invalid_creds",
 188                 "cpu_startup_entry",
 189                 "__ubsan_handle_builtin_unreachable",
 190                 "ex_handler_msr_mce",
 191         };
 192
 193         if (!func)
 194                 return false;
 195
 196         if (func->bind == STB_WEAK)
 197                 return false;
 198
 199         if (func->bind == STB_GLOBAL)
 200                 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
 201                         if (!strcmp(func->name, global_noreturns[i]))
 202                                 return true;
 203
 204         if (!func->len)
 205                 return false;
 206
 207         insn = find_insn(file, func->sec, func->offset);
 208         if (!insn->func)
 209                 return false;
 210
 211         func_for_each_insn(file, func, insn) {
 212                 empty = false;
 213
 214                 if (insn->type == INSN_RETURN)
 215                         return false;
 216         }
 217
 218         if (empty)
 219                 return false;
 220
 221         /*
 222          * A function can have a sibling call instead of a return.  In that
 223          * case, the function's dead-end status depends on whether the target
 224          * of the sibling call returns.
 225          */
 226         func_for_each_insn(file, func, insn) {
 227                 if (is_sibling_call(insn)) {
 228                         struct instruction *dest = insn->jump_dest;
 229
 230                         if (!dest)
 231                                 /* sibling call to another file */
 232                                 return false;
 233
 234                         /* local sibling call */
 235                         if (recursion == 5) {
 236                                 /*
 237                                  * Infinite recursion: two functions have
 238                                  * sibling calls to each other.  This is a very
 239                                  * rare case.  It means they aren't dead ends.
 240                                  */
 241                                 return false;
 242                         }
 243
 244                         return __dead_end_function(file, dest->func, recursion+1);
 245                 }
 246         }
 247
 248         return true;
 249 }
 250
 251 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
 252 {
 253         return __dead_end_function(file, func, 0);
 254 }
 255
 256 static void init_cfi_state(struct cfi_state *cfi)
 257 {
 258         int i;
 259
 260         for (i = 0; i < CFI_NUM_REGS; i++) {
 261                 cfi->regs[i].base = CFI_UNDEFINED;
 262                 cfi->vals[i].base = CFI_UNDEFINED;
 263         }
 264         cfi->cfa.base = CFI_UNDEFINED;
 265         cfi->drap_reg = CFI_UNDEFINED;
 266         cfi->drap_offset = -1;
 267 }
 268
 269 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
 270                             struct section *sec)
 271 {
 272         memset(state, 0, sizeof(*state));
 273         init_cfi_state(&state->cfi);
 274
 275         /*
 276          * We need the full vmlinux for noinstr validation, otherwise we can
 277          * not correctly determine insn->call_dest->sec (external symbols do
 278          * not have a section).
 279          */
 280         if (opts.link && opts.noinstr && sec)
 281                 state->noinstr = sec->noinstr;
 282 }
 283
 284 static struct cfi_state *cfi_alloc(void)
 285 {
 286         struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
 287         if (!cfi) {
 288                 WARN("calloc failed");
 289                 exit(1);
 290         }
 291         nr_cfi++;
 292         return cfi;
 293 }
 294
 295 static int cfi_bits;
 296 static struct hlist_head *cfi_hash;
 297
 298 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
 299 {
 300         return memcmp((void *)cfi1 + sizeof(cfi1->hash),
 301                       (void *)cfi2 + sizeof(cfi2->hash),
 302                       sizeof(struct cfi_state) - sizeof(struct hlist_node));
 303 }
 304
 305 static inline u32 cfi_key(struct cfi_state *cfi)
 306 {
 307         return jhash((void *)cfi + sizeof(cfi->hash),
 308                      sizeof(*cfi) - sizeof(cfi->hash), 0);
 309 }
 310
 311 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
 312 {
 313         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 314         struct cfi_state *obj;
 315
 316         hlist_for_each_entry(obj, head, hash) {
 317                 if (!cficmp(cfi, obj)) {
 318                         nr_cfi_cache++;
 319                         return obj;
 320                 }
 321         }
 322
 323         obj = cfi_alloc();
 324         *obj = *cfi;
 325         hlist_add_head(&obj->hash, head);
 326
 327         return obj;
 328 }
 329
 330 static void cfi_hash_add(struct cfi_state *cfi)
 331 {
 332         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 333
 334         hlist_add_head(&cfi->hash, head);
 335 }
 336
 337 static void *cfi_hash_alloc(unsigned long size)
 338 {
 339         cfi_bits = max(10, ilog2(size));
 340         cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
 341                         PROT_READ|PROT_WRITE,
 342                         MAP_PRIVATE|MAP_ANON, -1, 0);
 343         if (cfi_hash == (void *)-1L) {
 344                 WARN("mmap fail cfi_hash");
 345                 cfi_hash = NULL;
 346         }  else if (opts.stats) {
 347                 printf("cfi_bits: %d\n", cfi_bits);
 348         }
 349
 350         return cfi_hash;
 351 }
 352
 353 static unsigned long nr_insns;
 354 static unsigned long nr_insns_visited;
 355
 356 /*
 357  * Call the arch-specific instruction decoder for all the instructions and add
 358  * them to the global instruction list.
 359  */
 360 static int decode_instructions(struct objtool_file *file)
 361 {
 362         struct section *sec;
 363         struct symbol *func;
 364         unsigned long offset;
 365         struct instruction *insn;
 366         int ret;
 367
 368         for_each_sec(file, sec) {
 369
 370                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
 371                         continue;
 372
 373                 if (strcmp(sec->name, ".altinstr_replacement") &&
 374                     strcmp(sec->name, ".altinstr_aux") &&
 375                     strncmp(sec->name, ".discard.", 9))
 376                         sec->text = true;
 377
 378                 if (!strcmp(sec->name, ".noinstr.text") ||
 379                     !strcmp(sec->name, ".entry.text") ||
 380                     !strncmp(sec->name, ".text.__x86.", 12))
 381                         sec->noinstr = true;
 382
 383                 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
 384                         insn = malloc(sizeof(*insn));
 385                         if (!insn) {
 386                                 WARN("malloc failed");
 387                                 return -1;
 388                         }
 389                         memset(insn, 0, sizeof(*insn));
 390                         INIT_LIST_HEAD(&insn->alts);
 391                         INIT_LIST_HEAD(&insn->stack_ops);
 392                         INIT_LIST_HEAD(&insn->call_node);
 393
 394                         insn->sec = sec;
 395                         insn->offset = offset;
 396
 397                         ret = arch_decode_instruction(file, sec, offset,
 398                                                       sec->sh.sh_size - offset,
 399                                                       &insn->len, &insn->type,
 400                                                       &insn->immediate,
 401                                                       &insn->stack_ops);
 402                         if (ret)
 403                                 goto err;
 404
 405                         /*
 406                          * By default, "ud2" is a dead end unless otherwise
 407                          * annotated, because GCC 7 inserts it for certain
 408                          * divide-by-zero cases.
 409                          */
 410                         if (insn->type == INSN_BUG)
 411                                 insn->dead_end = true;
 412
 413                         hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
 414                         list_add_tail(&insn->list, &file->insn_list);
 415                         nr_insns++;
 416                 }
 417
 418                 list_for_each_entry(func, &sec->symbol_list, list) {
 419                         if (func->type != STT_FUNC || func->alias != func)
 420                                 continue;
 421
 422                         if (!find_insn(file, sec, func->offset)) {
 423                                 WARN("%s(): can't find starting instruction",
 424                                      func->name);
 425                                 return -1;
 426                         }
 427
 428                         sym_for_each_insn(file, func, insn) {
 429                                 insn->func = func;
 430                                 if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) {
 431                                         if (insn->offset == insn->func->offset) {
 432                                                 list_add_tail(&insn->call_node, &file->endbr_list);
 433                                                 file->nr_endbr++;
 434                                         } else {
 435                                                 file->nr_endbr_int++;
 436                                         }
 437                                 }
 438                         }
 439                 }
 440         }
 441
 442         if (opts.stats)
 443                 printf("nr_insns: %lu\n", nr_insns);
 444
 445         return 0;
 446
 447 err:
 448         free(insn);
 449         return ret;
 450 }
 451
 452 /*
 453  * Read the pv_ops[] .data table to find the static initialized values.
 454  */
 455 static int add_pv_ops(struct objtool_file *file, const char *symname)
 456 {
 457         struct symbol *sym, *func;
 458         unsigned long off, end;
 459         struct reloc *rel;
 460         int idx;
 461
 462         sym = find_symbol_by_name(file->elf, symname);
 463         if (!sym)
 464                 return 0;
 465
 466         off = sym->offset;
 467         end = off + sym->len;
 468         for (;;) {
 469                 rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
 470                 if (!rel)
 471                         break;
 472
 473                 func = rel->sym;
 474                 if (func->type == STT_SECTION)
 475                         func = find_symbol_by_offset(rel->sym->sec, rel->addend);
 476
 477                 idx = (rel->offset - sym->offset) / sizeof(unsigned long);
 478
 479                 objtool_pv_add(file, idx, func);
 480
 481                 off = rel->offset + 1;
 482                 if (off > end)
 483                         break;
 484         }
 485
 486         return 0;
 487 }
 488
 489 /*
 490  * Allocate and initialize file->pv_ops[].
 491  */
 492 static int init_pv_ops(struct objtool_file *file)
 493 {
 494         static const char *pv_ops_tables[] = {
 495                 "pv_ops",
 496                 "xen_cpu_ops",
 497                 "xen_irq_ops",
 498                 "xen_mmu_ops",
 499                 NULL,
 500         };
 501         const char *pv_ops;
 502         struct symbol *sym;
 503         int idx, nr;
 504
 505         if (!opts.noinstr)
 506                 return 0;
 507
 508         file->pv_ops = NULL;
 509
 510         sym = find_symbol_by_name(file->elf, "pv_ops");
 511         if (!sym)
 512                 return 0;
 513
 514         nr = sym->len / sizeof(unsigned long);
 515         file->pv_ops = calloc(sizeof(struct pv_state), nr);
 516         if (!file->pv_ops)
 517                 return -1;
 518
 519         for (idx = 0; idx < nr; idx++)
 520                 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
 521
 522         for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
 523                 add_pv_ops(file, pv_ops);
 524
 525         return 0;
 526 }
 527
 528 static struct instruction *find_last_insn(struct objtool_file *file,
 529                                           struct section *sec)
 530 {
 531         struct instruction *insn = NULL;
 532         unsigned int offset;
 533         unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
 534
 535         for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
 536                 insn = find_insn(file, sec, offset);
 537
 538         return insn;
 539 }
 540
 541 /*
 542  * Mark "ud2" instructions and manually annotated dead ends.
 543  */
 544 static int add_dead_ends(struct objtool_file *file)
 545 {
 546         struct section *sec;
 547         struct reloc *reloc;
 548         struct instruction *insn;
 549
 550         /*
 551          * Check for manually annotated dead ends.
 552          */
 553         sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
 554         if (!sec)
 555                 goto reachable;
 556
 557         list_for_each_entry(reloc, &sec->reloc_list, list) {
 558                 if (reloc->sym->type != STT_SECTION) {
 559                         WARN("unexpected relocation symbol type in %s", sec->name);
 560                         return -1;
 561                 }
 562                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 563                 if (insn)
 564                         insn = list_prev_entry(insn, list);
 565                 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 566                         insn = find_last_insn(file, reloc->sym->sec);
 567                         if (!insn) {
 568                                 WARN("can't find unreachable insn at %s+0x%" PRIx64,
 569                                      reloc->sym->sec->name, reloc->addend);
 570                                 return -1;
 571                         }
 572                 } else {
 573                         WARN("can't find unreachable insn at %s+0x%" PRIx64,
 574                              reloc->sym->sec->name, reloc->addend);
 575                         return -1;
 576                 }
 577
 578                 insn->dead_end = true;
 579         }
 580
 581 reachable:
 582         /*
 583          * These manually annotated reachable checks are needed for GCC 4.4,
 584          * where the Linux unreachable() macro isn't supported.  In that case
 585          * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
 586          * not a dead end.
 587          */
 588         sec = find_section_by_name(file->elf, ".rela.discard.reachable");
 589         if (!sec)
 590                 return 0;
 591
 592         list_for_each_entry(reloc, &sec->reloc_list, list) {
 593                 if (reloc->sym->type != STT_SECTION) {
 594                         WARN("unexpected relocation symbol type in %s", sec->name);
 595                         return -1;
 596                 }
 597                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 598                 if (insn)
 599                         insn = list_prev_entry(insn, list);
 600                 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 601                         insn = find_last_insn(file, reloc->sym->sec);
 602                         if (!insn) {
 603                                 WARN("can't find reachable insn at %s+0x%" PRIx64,
 604                                      reloc->sym->sec->name, reloc->addend);
 605                                 return -1;
 606                         }
 607                 } else {
 608                         WARN("can't find reachable insn at %s+0x%" PRIx64,
 609                              reloc->sym->sec->name, reloc->addend);
 610                         return -1;
 611                 }
 612
 613                 insn->dead_end = false;
 614         }
 615
 616         return 0;
 617 }
 618
 619 static int create_static_call_sections(struct objtool_file *file)
 620 {
 621         struct section *sec;
 622         struct static_call_site *site;
 623         struct instruction *insn;
 624         struct symbol *key_sym;
 625         char *key_name, *tmp;
 626         int idx;
 627
 628         sec = find_section_by_name(file->elf, ".static_call_sites");
 629         if (sec) {
 630                 INIT_LIST_HEAD(&file->static_call_list);
 631                 WARN("file already has .static_call_sites section, skipping");
 632                 return 0;
 633         }
 634
 635         if (list_empty(&file->static_call_list))
 636                 return 0;
 637
 638         idx = 0;
 639         list_for_each_entry(insn, &file->static_call_list, call_node)
 640                 idx++;
 641
 642         sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
 643                                  sizeof(struct static_call_site), idx);
 644         if (!sec)
 645                 return -1;
 646
 647         idx = 0;
 648         list_for_each_entry(insn, &file->static_call_list, call_node) {
 649
 650                 site = (struct static_call_site *)sec->data->d_buf + idx;
 651                 memset(site, 0, sizeof(struct static_call_site));
 652
 653                 /* populate reloc for 'addr' */
 654                 if (elf_add_reloc_to_insn(file->elf, sec,
 655                                           idx * sizeof(struct static_call_site),
 656                                           R_X86_64_PC32,
 657                                           insn->sec, insn->offset))
 658                         return -1;
 659
 660                 /* find key symbol */
 661                 key_name = strdup(insn->call_dest->name);
 662                 if (!key_name) {
 663                         perror("strdup");
 664                         return -1;
 665                 }
 666                 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
 667                             STATIC_CALL_TRAMP_PREFIX_LEN)) {
 668                         WARN("static_call: trampoline name malformed: %s", key_name);
 669                         return -1;
 670                 }
 671                 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
 672                 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
 673
 674                 key_sym = find_symbol_by_name(file->elf, tmp);
 675                 if (!key_sym) {
 676                         if (!opts.module) {
 677                                 WARN("static_call: can't find static_call_key symbol: %s", tmp);
 678                                 return -1;
 679                         }
 680
 681                         /*
 682                          * For modules(), the key might not be exported, which
 683                          * means the module can make static calls but isn't
 684                          * allowed to change them.
 685                          *
 686                          * In that case we temporarily set the key to be the
 687                          * trampoline address.  This is fixed up in
 688                          * static_call_add_module().
 689                          */
 690                         key_sym = insn->call_dest;
 691                 }
 692                 free(key_name);
 693
 694                 /* populate reloc for 'key' */
 695                 if (elf_add_reloc(file->elf, sec,
 696                                   idx * sizeof(struct static_call_site) + 4,
 697                                   R_X86_64_PC32, key_sym,
 698                                   is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
 699                         return -1;
 700
 701                 idx++;
 702         }
 703
 704         return 0;
 705 }
 706
 707 static int create_retpoline_sites_sections(struct objtool_file *file)
 708 {
 709         struct instruction *insn;
 710         struct section *sec;
 711         int idx;
 712
 713         sec = find_section_by_name(file->elf, ".retpoline_sites");
 714         if (sec) {
 715                 WARN("file already has .retpoline_sites, skipping");
 716                 return 0;
 717         }
 718
 719         idx = 0;
 720         list_for_each_entry(insn, &file->retpoline_call_list, call_node)
 721                 idx++;
 722
 723         if (!idx)
 724                 return 0;
 725
 726         sec = elf_create_section(file->elf, ".retpoline_sites", 0,
 727                                  sizeof(int), idx);
 728         if (!sec) {
 729                 WARN("elf_create_section: .retpoline_sites");
 730                 return -1;
 731         }
 732
 733         idx = 0;
 734         list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
 735
 736                 int *site = (int *)sec->data->d_buf + idx;
 737                 *site = 0;
 738
 739                 if (elf_add_reloc_to_insn(file->elf, sec,
 740                                           idx * sizeof(int),
 741                                           R_X86_64_PC32,
 742                                           insn->sec, insn->offset)) {
 743                         WARN("elf_add_reloc_to_insn: .retpoline_sites");
 744                         return -1;
 745                 }
 746
 747                 idx++;
 748         }
 749
 750         return 0;
 751 }
 752
 753 static int create_return_sites_sections(struct objtool_file *file)
 754 {
 755         struct instruction *insn;
 756         struct section *sec;
 757         int idx;
 758
 759         sec = find_section_by_name(file->elf, ".return_sites");
 760         if (sec) {
 761                 WARN("file already has .return_sites, skipping");
 762                 return 0;
 763         }
 764
 765         idx = 0;
 766         list_for_each_entry(insn, &file->return_thunk_list, call_node)
 767                 idx++;
 768
 769         if (!idx)
 770                 return 0;
 771
 772         sec = elf_create_section(file->elf, ".return_sites", 0,
 773                                  sizeof(int), idx);
 774         if (!sec) {
 775                 WARN("elf_create_section: .return_sites");
 776                 return -1;
 777         }
 778
 779         idx = 0;
 780         list_for_each_entry(insn, &file->return_thunk_list, call_node) {
 781
 782                 int *site = (int *)sec->data->d_buf + idx;
 783                 *site = 0;
 784
 785                 if (elf_add_reloc_to_insn(file->elf, sec,
 786                                           idx * sizeof(int),
 787                                           R_X86_64_PC32,
 788                                           insn->sec, insn->offset)) {
 789                         WARN("elf_add_reloc_to_insn: .return_sites");
 790                         return -1;
 791                 }
 792
 793                 idx++;
 794         }
 795
 796         return 0;
 797 }
 798
 799 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
 800 {
 801         struct instruction *insn;
 802         struct section *sec;
 803         int idx;
 804
 805         sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
 806         if (sec) {
 807                 WARN("file already has .ibt_endbr_seal, skipping");
 808                 return 0;
 809         }
 810
 811         idx = 0;
 812         list_for_each_entry(insn, &file->endbr_list, call_node)
 813                 idx++;
 814
 815         if (opts.stats) {
 816                 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
 817                 printf("ibt: ENDBR inside functions:  %d\n", file->nr_endbr_int);
 818                 printf("ibt: superfluous ENDBR:       %d\n", idx);
 819         }
 820
 821         if (!idx)
 822                 return 0;
 823
 824         sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0,
 825                                  sizeof(int), idx);
 826         if (!sec) {
 827                 WARN("elf_create_section: .ibt_endbr_seal");
 828                 return -1;
 829         }
 830
 831         idx = 0;
 832         list_for_each_entry(insn, &file->endbr_list, call_node) {
 833
 834                 int *site = (int *)sec->data->d_buf + idx;
 835                 *site = 0;
 836
 837                 if (elf_add_reloc_to_insn(file->elf, sec,
 838                                           idx * sizeof(int),
 839                                           R_X86_64_PC32,
 840                                           insn->sec, insn->offset)) {
 841                         WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
 842                         return -1;
 843                 }
 844
 845                 idx++;
 846         }
 847
 848         return 0;
 849 }
 850
 851 static int create_mcount_loc_sections(struct objtool_file *file)
 852 {
 853         struct section *sec;
 854         unsigned long *loc;
 855         struct instruction *insn;
 856         int idx;
 857
 858         sec = find_section_by_name(file->elf, "__mcount_loc");
 859         if (sec) {
 860                 INIT_LIST_HEAD(&file->mcount_loc_list);
 861                 WARN("file already has __mcount_loc section, skipping");
 862                 return 0;
 863         }
 864
 865         if (list_empty(&file->mcount_loc_list))
 866                 return 0;
 867
 868         idx = 0;
 869         list_for_each_entry(insn, &file->mcount_loc_list, call_node)
 870                 idx++;
 871
 872         sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
 873         if (!sec)
 874                 return -1;
 875
 876         idx = 0;
 877         list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
 878
 879                 loc = (unsigned long *)sec->data->d_buf + idx;
 880                 memset(loc, 0, sizeof(unsigned long));
 881
 882                 if (elf_add_reloc_to_insn(file->elf, sec,
 883                                           idx * sizeof(unsigned long),
 884                                           R_X86_64_64,
 885                                           insn->sec, insn->offset))
 886                         return -1;
 887
 888                 idx++;
 889         }
 890
 891         return 0;
 892 }
 893
 894 /*
 895  * Warnings shouldn't be reported for ignored functions.
 896  */
 897 static void add_ignores(struct objtool_file *file)
 898 {
 899         struct instruction *insn;
 900         struct section *sec;
 901         struct symbol *func;
 902         struct reloc *reloc;
 903
 904         sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
 905         if (!sec)
 906                 return;
 907
 908         list_for_each_entry(reloc, &sec->reloc_list, list) {
 909                 switch (reloc->sym->type) {
 910                 case STT_FUNC:
 911                         func = reloc->sym;
 912                         break;
 913
 914                 case STT_SECTION:
 915                         func = find_func_by_offset(reloc->sym->sec, reloc->addend);
 916                         if (!func)
 917                                 continue;
 918                         break;
 919
 920                 default:
 921                         WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
 922                         continue;
 923                 }
 924
 925                 func_for_each_insn(file, func, insn)
 926                         insn->ignore = true;
 927         }
 928 }
 929
 930 /*
 931  * This is a whitelist of functions that is allowed to be called with AC set.
 932  * The list is meant to be minimal and only contains compiler instrumentation
 933  * ABI and a few functions used to implement *_{to,from}_user() functions.
 934  *
 935  * These functions must not directly change AC, but may PUSHF/POPF.
 936  */
 937 static const char *uaccess_safe_builtin[] = {
 938         /* KASAN */
 939         "kasan_report",
 940         "kasan_check_range",
 941         /* KASAN out-of-line */
 942         "__asan_loadN_noabort",
 943         "__asan_load1_noabort",
 944         "__asan_load2_noabort",
 945         "__asan_load4_noabort",
 946         "__asan_load8_noabort",
 947         "__asan_load16_noabort",
 948         "__asan_storeN_noabort",
 949         "__asan_store1_noabort",
 950         "__asan_store2_noabort",
 951         "__asan_store4_noabort",
 952         "__asan_store8_noabort",
 953         "__asan_store16_noabort",
 954         "__kasan_check_read",
 955         "__kasan_check_write",
 956         /* KASAN in-line */
 957         "__asan_report_load_n_noabort",
 958         "__asan_report_load1_noabort",
 959         "__asan_report_load2_noabort",
 960         "__asan_report_load4_noabort",
 961         "__asan_report_load8_noabort",
 962         "__asan_report_load16_noabort",
 963         "__asan_report_store_n_noabort",
 964         "__asan_report_store1_noabort",
 965         "__asan_report_store2_noabort",
 966         "__asan_report_store4_noabort",
 967         "__asan_report_store8_noabort",
 968         "__asan_report_store16_noabort",
 969         /* KCSAN */
 970         "__kcsan_check_access",
 971         "__kcsan_mb",
 972         "__kcsan_wmb",
 973         "__kcsan_rmb",
 974         "__kcsan_release",
 975         "kcsan_found_watchpoint",
 976         "kcsan_setup_watchpoint",
 977         "kcsan_check_scoped_accesses",
 978         "kcsan_disable_current",
 979         "kcsan_enable_current_nowarn",
 980         /* KCSAN/TSAN */
 981         "__tsan_func_entry",
 982         "__tsan_func_exit",
 983         "__tsan_read_range",
 984         "__tsan_write_range",
 985         "__tsan_read1",
 986         "__tsan_read2",
 987         "__tsan_read4",
 988         "__tsan_read8",
 989         "__tsan_read16",
 990         "__tsan_write1",
 991         "__tsan_write2",
 992         "__tsan_write4",
 993         "__tsan_write8",
 994         "__tsan_write16",
 995         "__tsan_read_write1",
 996         "__tsan_read_write2",
 997         "__tsan_read_write4",
 998         "__tsan_read_write8",
 999         "__tsan_read_write16",
1000         "__tsan_atomic8_load",
1001         "__tsan_atomic16_load",
1002         "__tsan_atomic32_load",
1003         "__tsan_atomic64_load",
1004         "__tsan_atomic8_store",
1005         "__tsan_atomic16_store",
1006         "__tsan_atomic32_store",
1007         "__tsan_atomic64_store",
1008         "__tsan_atomic8_exchange",
1009         "__tsan_atomic16_exchange",
1010         "__tsan_atomic32_exchange",
1011         "__tsan_atomic64_exchange",
1012         "__tsan_atomic8_fetch_add",
1013         "__tsan_atomic16_fetch_add",
1014         "__tsan_atomic32_fetch_add",
1015         "__tsan_atomic64_fetch_add",
1016         "__tsan_atomic8_fetch_sub",
1017         "__tsan_atomic16_fetch_sub",
1018         "__tsan_atomic32_fetch_sub",
1019         "__tsan_atomic64_fetch_sub",
1020         "__tsan_atomic8_fetch_and",
1021         "__tsan_atomic16_fetch_and",
1022         "__tsan_atomic32_fetch_and",
1023         "__tsan_atomic64_fetch_and",
1024         "__tsan_atomic8_fetch_or",
1025         "__tsan_atomic16_fetch_or",
1026         "__tsan_atomic32_fetch_or",
1027         "__tsan_atomic64_fetch_or",
1028         "__tsan_atomic8_fetch_xor",
1029         "__tsan_atomic16_fetch_xor",
1030         "__tsan_atomic32_fetch_xor",
1031         "__tsan_atomic64_fetch_xor",
1032         "__tsan_atomic8_fetch_nand",
1033         "__tsan_atomic16_fetch_nand",
1034         "__tsan_atomic32_fetch_nand",
1035         "__tsan_atomic64_fetch_nand",
1036         "__tsan_atomic8_compare_exchange_strong",
1037         "__tsan_atomic16_compare_exchange_strong",
1038         "__tsan_atomic32_compare_exchange_strong",
1039         "__tsan_atomic64_compare_exchange_strong",
1040         "__tsan_atomic8_compare_exchange_weak",
1041         "__tsan_atomic16_compare_exchange_weak",
1042         "__tsan_atomic32_compare_exchange_weak",
1043         "__tsan_atomic64_compare_exchange_weak",
1044         "__tsan_atomic8_compare_exchange_val",
1045         "__tsan_atomic16_compare_exchange_val",
1046         "__tsan_atomic32_compare_exchange_val",
1047         "__tsan_atomic64_compare_exchange_val",
1048         "__tsan_atomic_thread_fence",
1049         "__tsan_atomic_signal_fence",
1050         /* KCOV */
1051         "write_comp_data",
1052         "check_kcov_mode",
1053         "__sanitizer_cov_trace_pc",
1054         "__sanitizer_cov_trace_const_cmp1",
1055         "__sanitizer_cov_trace_const_cmp2",
1056         "__sanitizer_cov_trace_const_cmp4",
1057         "__sanitizer_cov_trace_const_cmp8",
1058         "__sanitizer_cov_trace_cmp1",
1059         "__sanitizer_cov_trace_cmp2",
1060         "__sanitizer_cov_trace_cmp4",
1061         "__sanitizer_cov_trace_cmp8",
1062         "__sanitizer_cov_trace_switch",
1063         /* UBSAN */
1064         "ubsan_type_mismatch_common",
1065         "__ubsan_handle_type_mismatch",
1066         "__ubsan_handle_type_mismatch_v1",
1067         "__ubsan_handle_shift_out_of_bounds",
1068         /* misc */
1069         "csum_partial_copy_generic",
1070         "copy_mc_fragile",
1071         "copy_mc_fragile_handle_tail",
1072         "copy_mc_enhanced_fast_string",
1073         "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1074         NULL
1075 };
1076
1077 static void add_uaccess_safe(struct objtool_file *file)
1078 {
1079         struct symbol *func;
1080         const char **name;
1081
1082         if (!opts.uaccess)
1083                 return;
1084
1085         for (name = uaccess_safe_builtin; *name; name++) {
1086                 func = find_symbol_by_name(file->elf, *name);
1087                 if (!func)
1088                         continue;
1089
1090                 func->uaccess_safe = true;
1091         }
1092 }
1093
1094 /*
1095  * FIXME: For now, just ignore any alternatives which add retpolines.  This is
1096  * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1097  * But it at least allows objtool to understand the control flow *around* the
1098  * retpoline.
1099  */
1100 static int add_ignore_alternatives(struct objtool_file *file)
1101 {
1102         struct section *sec;
1103         struct reloc *reloc;
1104         struct instruction *insn;
1105
1106         sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1107         if (!sec)
1108                 return 0;
1109
1110         list_for_each_entry(reloc, &sec->reloc_list, list) {
1111                 if (reloc->sym->type != STT_SECTION) {
1112                         WARN("unexpected relocation symbol type in %s", sec->name);
1113                         return -1;
1114                 }
1115
1116                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1117                 if (!insn) {
1118                         WARN("bad .discard.ignore_alts entry");
1119                         return -1;
1120                 }
1121
1122                 insn->ignore_alts = true;
1123         }
1124
1125         return 0;
1126 }
1127
1128 __weak bool arch_is_retpoline(struct symbol *sym)
1129 {
1130         return false;
1131 }
1132
1133 __weak bool arch_is_rethunk(struct symbol *sym)
1134 {
1135         return false;
1136 }
1137
1138 #define NEGATIVE_RELOC  ((void *)-1L)
1139
1140 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1141 {
1142         if (insn->reloc == NEGATIVE_RELOC)
1143                 return NULL;
1144
1145         if (!insn->reloc) {
1146                 if (!file)
1147                         return NULL;
1148
1149                 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1150                                                        insn->offset, insn->len);
1151                 if (!insn->reloc) {
1152                         insn->reloc = NEGATIVE_RELOC;
1153                         return NULL;
1154                 }
1155         }
1156
1157         return insn->reloc;
1158 }
1159
1160 static void remove_insn_ops(struct instruction *insn)
1161 {
1162         struct stack_op *op, *tmp;
1163
1164         list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
1165                 list_del(&op->list);
1166                 free(op);
1167         }
1168 }
1169
1170 static void annotate_call_site(struct objtool_file *file,
1171                                struct instruction *insn, bool sibling)
1172 {
1173         struct reloc *reloc = insn_reloc(file, insn);
1174         struct symbol *sym = insn->call_dest;
1175
1176         if (!sym)
1177                 sym = reloc->sym;
1178
1179         /*
1180          * Alternative replacement code is just template code which is
1181          * sometimes copied to the original instruction. For now, don't
1182          * annotate it. (In the future we might consider annotating the
1183          * original instruction if/when it ever makes sense to do so.)
1184          */
1185         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1186                 return;
1187
1188         if (sym->static_call_tramp) {
1189                 list_add_tail(&insn->call_node, &file->static_call_list);
1190                 return;
1191         }
1192
1193         if (sym->retpoline_thunk) {
1194                 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1195                 return;
1196         }
1197
1198         /*
1199          * Many compilers cannot disable KCOV or sanitizer calls with a function
1200          * attribute so they need a little help, NOP out any such calls from
1201          * noinstr text.
1202          */
1203         if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1204                 if (reloc) {
1205                         reloc->type = R_NONE;
1206                         elf_write_reloc(file->elf, reloc);
1207                 }
1208
1209                 elf_write_insn(file->elf, insn->sec,
1210                                insn->offset, insn->len,
1211                                sibling ? arch_ret_insn(insn->len)
1212                                        : arch_nop_insn(insn->len));
1213
1214                 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1215
1216                 if (sibling) {
1217                         /*
1218                          * We've replaced the tail-call JMP insn by two new
1219                          * insn: RET; INT3, except we only have a single struct
1220                          * insn here. Mark it retpoline_safe to avoid the SLS
1221                          * warning, instead of adding another insn.
1222                          */
1223                         insn->retpoline_safe = true;
1224                 }
1225
1226                 return;
1227         }
1228
1229         if (opts.mcount && sym->fentry) {
1230                 if (sibling)
1231                         WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
1232
1233                 if (reloc) {
1234                         reloc->type = R_NONE;
1235                         elf_write_reloc(file->elf, reloc);
1236                 }
1237
1238                 elf_write_insn(file->elf, insn->sec,
1239                                insn->offset, insn->len,
1240                                arch_nop_insn(insn->len));
1241
1242                 insn->type = INSN_NOP;
1243
1244                 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1245                 return;
1246         }
1247
1248         if (!sibling && dead_end_function(file, sym))
1249                 insn->dead_end = true;
1250 }
1251
1252 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1253                           struct symbol *dest, bool sibling)
1254 {
1255         insn->call_dest = dest;
1256         if (!dest)
1257                 return;
1258
1259         /*
1260          * Whatever stack impact regular CALLs have, should be undone
1261          * by the RETURN of the called function.
1262          *
1263          * Annotated intra-function calls retain the stack_ops but
1264          * are converted to JUMP, see read_intra_function_calls().
1265          */
1266         remove_insn_ops(insn);
1267
1268         annotate_call_site(file, insn, sibling);
1269 }
1270
1271 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1272 {
1273         /*
1274          * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1275          * so convert them accordingly.
1276          */
1277         switch (insn->type) {
1278         case INSN_CALL:
1279                 insn->type = INSN_CALL_DYNAMIC;
1280                 break;
1281         case INSN_JUMP_UNCONDITIONAL:
1282                 insn->type = INSN_JUMP_DYNAMIC;
1283                 break;
1284         case INSN_JUMP_CONDITIONAL:
1285                 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1286                 break;
1287         default:
1288                 return;
1289         }
1290
1291         insn->retpoline_safe = true;
1292
1293         /*
1294          * Whatever stack impact regular CALLs have, should be undone
1295          * by the RETURN of the called function.
1296          *
1297          * Annotated intra-function calls retain the stack_ops but
1298          * are converted to JUMP, see read_intra_function_calls().
1299          */
1300         remove_insn_ops(insn);
1301
1302         annotate_call_site(file, insn, false);
1303 }
1304
1305 static void add_return_call(struct objtool_file *file, struct instruction *insn)
1306 {
1307         /*
1308          * Return thunk tail calls are really just returns in disguise,
1309          * so convert them accordingly.
1310          */
1311         insn->type = INSN_RETURN;
1312         insn->retpoline_safe = true;
1313
1314         list_add_tail(&insn->call_node, &file->return_thunk_list);
1315 }
1316
1317 static bool same_function(struct instruction *insn1, struct instruction *insn2)
1318 {
1319         return insn1->func->pfunc == insn2->func->pfunc;
1320 }
1321
1322 static bool is_first_func_insn(struct objtool_file *file, struct instruction *insn)
1323 {
1324         if (insn->offset == insn->func->offset)
1325                 return true;
1326
1327         if (opts.ibt) {
1328                 struct instruction *prev = prev_insn_same_sym(file, insn);
1329
1330                 if (prev && prev->type == INSN_ENDBR &&
1331                     insn->offset == insn->func->offset + prev->len)
1332                         return true;
1333         }
1334
1335         return false;
1336 }
1337
1338 /*
1339  * Find the destination instructions for all jumps.
1340  */
1341 static int add_jump_destinations(struct objtool_file *file)
1342 {
1343         struct instruction *insn, *jump_dest;
1344         struct reloc *reloc;
1345         struct section *dest_sec;
1346         unsigned long dest_off;
1347
1348         for_each_insn(file, insn) {
1349                 if (insn->jump_dest) {
1350                         /*
1351                          * handle_group_alt() may have previously set
1352                          * 'jump_dest' for some alternatives.
1353                          */
1354                         continue;
1355                 }
1356                 if (!is_static_jump(insn))
1357                         continue;
1358
1359                 reloc = insn_reloc(file, insn);
1360                 if (!reloc) {
1361                         dest_sec = insn->sec;
1362                         dest_off = arch_jump_destination(insn);
1363                 } else if (reloc->sym->type == STT_SECTION) {
1364                         dest_sec = reloc->sym->sec;
1365                         dest_off = arch_dest_reloc_offset(reloc->addend);
1366                 } else if (reloc->sym->retpoline_thunk) {
1367                         add_retpoline_call(file, insn);
1368                         continue;
1369                 } else if (reloc->sym->return_thunk) {
1370                         add_return_call(file, insn);
1371                         continue;
1372                 } else if (insn->func) {
1373                         /*
1374                          * External sibling call or internal sibling call with
1375                          * STT_FUNC reloc.
1376                          */
1377                         add_call_dest(file, insn, reloc->sym, true);
1378                         continue;
1379                 } else if (reloc->sym->sec->idx) {
1380                         dest_sec = reloc->sym->sec;
1381                         dest_off = reloc->sym->sym.st_value +
1382                                    arch_dest_reloc_offset(reloc->addend);
1383                 } else {
1384                         /* non-func asm code jumping to another file */
1385                         continue;
1386                 }
1387
1388                 jump_dest = find_insn(file, dest_sec, dest_off);
1389                 if (!jump_dest) {
1390                         WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
1391                                   insn->sec, insn->offset, dest_sec->name,
1392                                   dest_off);
1393                         return -1;
1394                 }
1395
1396                 /*
1397                  * Cross-function jump.
1398                  */
1399                 if (insn->func && jump_dest->func &&
1400                     insn->func != jump_dest->func) {
1401
1402                         /*
1403                          * For GCC 8+, create parent/child links for any cold
1404                          * subfunctions.  This is _mostly_ redundant with a
1405                          * similar initialization in read_symbols().
1406                          *
1407                          * If a function has aliases, we want the *first* such
1408                          * function in the symbol table to be the subfunction's
1409                          * parent.  In that case we overwrite the
1410                          * initialization done in read_symbols().
1411                          *
1412                          * However this code can't completely replace the
1413                          * read_symbols() code because this doesn't detect the
1414                          * case where the parent function's only reference to a
1415                          * subfunction is through a jump table.
1416                          */
1417                         if (!strstr(insn->func->name, ".cold") &&
1418                             strstr(jump_dest->func->name, ".cold")) {
1419                                 insn->func->cfunc = jump_dest->func;
1420                                 jump_dest->func->pfunc = insn->func;
1421
1422                         } else if (!same_function(insn, jump_dest) &&
1423                                    is_first_func_insn(file, jump_dest)) {
1424                                 /*
1425                                  * Internal sibling call without reloc or with
1426                                  * STT_SECTION reloc.
1427                                  */
1428                                 add_call_dest(file, insn, jump_dest->func, true);
1429                                 continue;
1430                         }
1431                 }
1432
1433                 insn->jump_dest = jump_dest;
1434         }
1435
1436         return 0;
1437 }
1438
1439 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1440 {
1441         struct symbol *call_dest;
1442
1443         call_dest = find_func_by_offset(sec, offset);
1444         if (!call_dest)
1445                 call_dest = find_symbol_by_offset(sec, offset);
1446
1447         return call_dest;
1448 }
1449
1450 /*
1451  * Find the destination instructions for all calls.
1452  */
1453 static int add_call_destinations(struct objtool_file *file)
1454 {
1455         struct instruction *insn;
1456         unsigned long dest_off;
1457         struct symbol *dest;
1458         struct reloc *reloc;
1459
1460         for_each_insn(file, insn) {
1461                 if (insn->type != INSN_CALL)
1462                         continue;
1463
1464                 reloc = insn_reloc(file, insn);
1465                 if (!reloc) {
1466                         dest_off = arch_jump_destination(insn);
1467                         dest = find_call_destination(insn->sec, dest_off);
1468
1469                         add_call_dest(file, insn, dest, false);
1470
1471                         if (insn->ignore)
1472                                 continue;
1473
1474                         if (!insn->call_dest) {
1475                                 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
1476                                 return -1;
1477                         }
1478
1479                         if (insn->func && insn->call_dest->type != STT_FUNC) {
1480                                 WARN_FUNC("unsupported call to non-function",
1481                                           insn->sec, insn->offset);
1482                                 return -1;
1483                         }
1484
1485                 } else if (reloc->sym->type == STT_SECTION) {
1486                         dest_off = arch_dest_reloc_offset(reloc->addend);
1487                         dest = find_call_destination(reloc->sym->sec, dest_off);
1488                         if (!dest) {
1489                                 WARN_FUNC("can't find call dest symbol at %s+0x%lx",
1490                                           insn->sec, insn->offset,
1491                                           reloc->sym->sec->name,
1492                                           dest_off);
1493                                 return -1;
1494                         }
1495
1496                         add_call_dest(file, insn, dest, false);
1497
1498                 } else if (reloc->sym->retpoline_thunk) {
1499                         add_retpoline_call(file, insn);
1500
1501                 } else
1502                         add_call_dest(file, insn, reloc->sym, false);
1503         }
1504
1505         return 0;
1506 }
1507
1508 /*
1509  * The .alternatives section requires some extra special care over and above
1510  * other special sections because alternatives are patched in place.
1511  */
1512 static int handle_group_alt(struct objtool_file *file,
1513                             struct special_alt *special_alt,
1514                             struct instruction *orig_insn,
1515                             struct instruction **new_insn)
1516 {
1517         struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
1518         struct alt_group *orig_alt_group, *new_alt_group;
1519         unsigned long dest_off;
1520
1521
1522         orig_alt_group = malloc(sizeof(*orig_alt_group));
1523         if (!orig_alt_group) {
1524                 WARN("malloc failed");
1525                 return -1;
1526         }
1527         orig_alt_group->cfi = calloc(special_alt->orig_len,
1528                                      sizeof(struct cfi_state *));
1529         if (!orig_alt_group->cfi) {
1530                 WARN("calloc failed");
1531                 return -1;
1532         }
1533
1534         last_orig_insn = NULL;
1535         insn = orig_insn;
1536         sec_for_each_insn_from(file, insn) {
1537                 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1538                         break;
1539
1540                 insn->alt_group = orig_alt_group;
1541                 last_orig_insn = insn;
1542         }
1543         orig_alt_group->orig_group = NULL;
1544         orig_alt_group->first_insn = orig_insn;
1545         orig_alt_group->last_insn = last_orig_insn;
1546
1547
1548         new_alt_group = malloc(sizeof(*new_alt_group));
1549         if (!new_alt_group) {
1550                 WARN("malloc failed");
1551                 return -1;
1552         }
1553
1554         if (special_alt->new_len < special_alt->orig_len) {
1555                 /*
1556                  * Insert a fake nop at the end to make the replacement
1557                  * alt_group the same size as the original.  This is needed to
1558                  * allow propagate_alt_cfi() to do its magic.  When the last
1559                  * instruction affects the stack, the instruction after it (the
1560                  * nop) will propagate the new state to the shared CFI array.
1561                  */
1562                 nop = malloc(sizeof(*nop));
1563                 if (!nop) {
1564                         WARN("malloc failed");
1565                         return -1;
1566                 }
1567                 memset(nop, 0, sizeof(*nop));
1568                 INIT_LIST_HEAD(&nop->alts);
1569                 INIT_LIST_HEAD(&nop->stack_ops);
1570
1571                 nop->sec = special_alt->new_sec;
1572                 nop->offset = special_alt->new_off + special_alt->new_len;
1573                 nop->len = special_alt->orig_len - special_alt->new_len;
1574                 nop->type = INSN_NOP;
1575                 nop->func = orig_insn->func;
1576                 nop->alt_group = new_alt_group;
1577                 nop->ignore = orig_insn->ignore_alts;
1578         }
1579
1580         if (!special_alt->new_len) {
1581                 *new_insn = nop;
1582                 goto end;
1583         }
1584
1585         insn = *new_insn;
1586         sec_for_each_insn_from(file, insn) {
1587                 struct reloc *alt_reloc;
1588
1589                 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1590                         break;
1591
1592                 last_new_insn = insn;
1593
1594                 insn->ignore = orig_insn->ignore_alts;
1595                 insn->func = orig_insn->func;
1596                 insn->alt_group = new_alt_group;
1597
1598                 /*
1599                  * Since alternative replacement code is copy/pasted by the
1600                  * kernel after applying relocations, generally such code can't
1601                  * have relative-address relocation references to outside the
1602                  * .altinstr_replacement section, unless the arch's
1603                  * alternatives code can adjust the relative offsets
1604                  * accordingly.
1605                  */
1606                 alt_reloc = insn_reloc(file, insn);
1607                 if (alt_reloc &&
1608                     !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1609
1610                         WARN_FUNC("unsupported relocation in alternatives section",
1611                                   insn->sec, insn->offset);
1612                         return -1;
1613                 }
1614
1615                 if (!is_static_jump(insn))
1616                         continue;
1617
1618                 if (!insn->immediate)
1619                         continue;
1620
1621                 dest_off = arch_jump_destination(insn);
1622                 if (dest_off == special_alt->new_off + special_alt->new_len) {
1623                         insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
1624                         if (!insn->jump_dest) {
1625                                 WARN_FUNC("can't find alternative jump destination",
1626                                           insn->sec, insn->offset);
1627                                 return -1;
1628                         }
1629                 }
1630         }
1631
1632         if (!last_new_insn) {
1633                 WARN_FUNC("can't find last new alternative instruction",
1634                           special_alt->new_sec, special_alt->new_off);
1635                 return -1;
1636         }
1637
1638         if (nop)
1639                 list_add(&nop->list, &last_new_insn->list);
1640 end:
1641         new_alt_group->orig_group = orig_alt_group;
1642         new_alt_group->first_insn = *new_insn;
1643         new_alt_group->last_insn = nop ? : last_new_insn;
1644         new_alt_group->cfi = orig_alt_group->cfi;
1645         return 0;
1646 }
1647
1648 /*
1649  * A jump table entry can either convert a nop to a jump or a jump to a nop.
1650  * If the original instruction is a jump, make the alt entry an effective nop
1651  * by just skipping the original instruction.
1652  */
1653 static int handle_jump_alt(struct objtool_file *file,
1654                            struct special_alt *special_alt,
1655                            struct instruction *orig_insn,
1656                            struct instruction **new_insn)
1657 {
1658         if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1659             orig_insn->type != INSN_NOP) {
1660
1661                 WARN_FUNC("unsupported instruction at jump label",
1662                           orig_insn->sec, orig_insn->offset);
1663                 return -1;
1664         }
1665
1666         if (opts.hack_jump_label && special_alt->key_addend & 2) {
1667                 struct reloc *reloc = insn_reloc(file, orig_insn);
1668
1669                 if (reloc) {
1670                         reloc->type = R_NONE;
1671                         elf_write_reloc(file->elf, reloc);
1672                 }
1673                 elf_write_insn(file->elf, orig_insn->sec,
1674                                orig_insn->offset, orig_insn->len,
1675                                arch_nop_insn(orig_insn->len));
1676                 orig_insn->type = INSN_NOP;
1677         }
1678
1679         if (orig_insn->type == INSN_NOP) {
1680                 if (orig_insn->len == 2)
1681                         file->jl_nop_short++;
1682                 else
1683                         file->jl_nop_long++;
1684
1685                 return 0;
1686         }
1687
1688         if (orig_insn->len == 2)
1689                 file->jl_short++;
1690         else
1691                 file->jl_long++;
1692
1693         *new_insn = list_next_entry(orig_insn, list);
1694         return 0;
1695 }
1696
1697 /*
1698  * Read all the special sections which have alternate instructions which can be
1699  * patched in or redirected to at runtime.  Each instruction having alternate
1700  * instruction(s) has them added to its insn->alts list, which will be
1701  * traversed in validate_branch().
1702  */
1703 static int add_special_section_alts(struct objtool_file *file)
1704 {
1705         struct list_head special_alts;
1706         struct instruction *orig_insn, *new_insn;
1707         struct special_alt *special_alt, *tmp;
1708         struct alternative *alt;
1709         int ret;
1710
1711         ret = special_get_alts(file->elf, &special_alts);
1712         if (ret)
1713                 return ret;
1714
1715         list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1716
1717                 orig_insn = find_insn(file, special_alt->orig_sec,
1718                                       special_alt->orig_off);
1719                 if (!orig_insn) {
1720                         WARN_FUNC("special: can't find orig instruction",
1721                                   special_alt->orig_sec, special_alt->orig_off);
1722                         ret = -1;
1723                         goto out;
1724                 }
1725
1726                 new_insn = NULL;
1727                 if (!special_alt->group || special_alt->new_len) {
1728                         new_insn = find_insn(file, special_alt->new_sec,
1729                                              special_alt->new_off);
1730                         if (!new_insn) {
1731                                 WARN_FUNC("special: can't find new instruction",
1732                                           special_alt->new_sec,
1733                                           special_alt->new_off);
1734                                 ret = -1;
1735                                 goto out;
1736                         }
1737                 }
1738
1739                 if (special_alt->group) {
1740                         if (!special_alt->orig_len) {
1741                                 WARN_FUNC("empty alternative entry",
1742                                           orig_insn->sec, orig_insn->offset);
1743                                 continue;
1744                         }
1745
1746                         ret = handle_group_alt(file, special_alt, orig_insn,
1747                                                &new_insn);
1748                         if (ret)
1749                                 goto out;
1750                 } else if (special_alt->jump_or_nop) {
1751                         ret = handle_jump_alt(file, special_alt, orig_insn,
1752                                               &new_insn);
1753                         if (ret)
1754                                 goto out;
1755                 }
1756
1757                 alt = malloc(sizeof(*alt));
1758                 if (!alt) {
1759                         WARN("malloc failed");
1760                         ret = -1;
1761                         goto out;
1762                 }
1763
1764                 alt->insn = new_insn;
1765                 alt->skip_orig = special_alt->skip_orig;
1766                 orig_insn->ignore_alts |= special_alt->skip_alt;
1767                 list_add_tail(&alt->list, &orig_insn->alts);
1768
1769                 list_del(&special_alt->list);
1770                 free(special_alt);
1771         }
1772
1773         if (opts.stats) {
1774                 printf("jl\\\tNOP\tJMP\n");
1775                 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1776                 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1777         }
1778
1779 out:
1780         return ret;
1781 }
1782
1783 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
1784                             struct reloc *table)
1785 {
1786         struct reloc *reloc = table;
1787         struct instruction *dest_insn;
1788         struct alternative *alt;
1789         struct symbol *pfunc = insn->func->pfunc;
1790         unsigned int prev_offset = 0;
1791
1792         /*
1793          * Each @reloc is a switch table relocation which points to the target
1794          * instruction.
1795          */
1796         list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
1797
1798                 /* Check for the end of the table: */
1799                 if (reloc != table && reloc->jump_table_start)
1800                         break;
1801
1802                 /* Make sure the table entries are consecutive: */
1803                 if (prev_offset && reloc->offset != prev_offset + 8)
1804                         break;
1805
1806                 /* Detect function pointers from contiguous objects: */
1807                 if (reloc->sym->sec == pfunc->sec &&
1808                     reloc->addend == pfunc->offset)
1809                         break;
1810
1811                 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
1812                 if (!dest_insn)
1813                         break;
1814
1815                 /* Make sure the destination is in the same function: */
1816                 if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
1817                         break;
1818
1819                 alt = malloc(sizeof(*alt));
1820                 if (!alt) {
1821                         WARN("malloc failed");
1822                         return -1;
1823                 }
1824
1825                 alt->insn = dest_insn;
1826                 list_add_tail(&alt->list, &insn->alts);
1827                 prev_offset = reloc->offset;
1828         }
1829
1830         if (!prev_offset) {
1831                 WARN_FUNC("can't find switch jump table",
1832                           insn->sec, insn->offset);
1833                 return -1;
1834         }
1835
1836         return 0;
1837 }
1838
1839 /*
1840  * find_jump_table() - Given a dynamic jump, find the switch jump table
1841  * associated with it.
1842  */
1843 static struct reloc *find_jump_table(struct objtool_file *file,
1844                                       struct symbol *func,
1845                                       struct instruction *insn)
1846 {
1847         struct reloc *table_reloc;
1848         struct instruction *dest_insn, *orig_insn = insn;
1849
1850         /*
1851          * Backward search using the @first_jump_src links, these help avoid
1852          * much of the 'in between' code. Which avoids us getting confused by
1853          * it.
1854          */
1855         for (;
1856              insn && insn->func && insn->func->pfunc == func;
1857              insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
1858
1859                 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
1860                         break;
1861
1862                 /* allow small jumps within the range */
1863                 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
1864                     insn->jump_dest &&
1865                     (insn->jump_dest->offset <= insn->offset ||
1866                      insn->jump_dest->offset > orig_insn->offset))
1867                     break;
1868
1869                 table_reloc = arch_find_switch_table(file, insn);
1870                 if (!table_reloc)
1871                         continue;
1872                 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
1873                 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
1874                         continue;
1875
1876                 return table_reloc;
1877         }
1878
1879         return NULL;
1880 }
1881
1882 /*
1883  * First pass: Mark the head of each jump table so that in the next pass,
1884  * we know when a given jump table ends and the next one starts.
1885  */
1886 static void mark_func_jump_tables(struct objtool_file *file,
1887                                     struct symbol *func)
1888 {
1889         struct instruction *insn, *last = NULL;
1890         struct reloc *reloc;
1891
1892         func_for_each_insn(file, func, insn) {
1893                 if (!last)
1894                         last = insn;
1895
1896                 /*
1897                  * Store back-pointers for unconditional forward jumps such
1898                  * that find_jump_table() can back-track using those and
1899                  * avoid some potentially confusing code.
1900                  */
1901                 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
1902                     insn->offset > last->offset &&
1903                     insn->jump_dest->offset > insn->offset &&
1904                     !insn->jump_dest->first_jump_src) {
1905
1906                         insn->jump_dest->first_jump_src = insn;
1907                         last = insn->jump_dest;
1908                 }
1909
1910                 if (insn->type != INSN_JUMP_DYNAMIC)
1911                         continue;
1912
1913                 reloc = find_jump_table(file, func, insn);
1914                 if (reloc) {
1915                         reloc->jump_table_start = true;
1916                         insn->jump_table = reloc;
1917                 }
1918         }
1919 }
1920
1921 static int add_func_jump_tables(struct objtool_file *file,
1922                                   struct symbol *func)
1923 {
1924         struct instruction *insn;
1925         int ret;
1926
1927         func_for_each_insn(file, func, insn) {
1928                 if (!insn->jump_table)
1929                         continue;
1930
1931                 ret = add_jump_table(file, insn, insn->jump_table);
1932                 if (ret)
1933                         return ret;
1934         }
1935
1936         return 0;
1937 }
1938
1939 /*
1940  * For some switch statements, gcc generates a jump table in the .rodata
1941  * section which contains a list of addresses within the function to jump to.
1942  * This finds these jump tables and adds them to the insn->alts lists.
1943  */
1944 static int add_jump_table_alts(struct objtool_file *file)
1945 {
1946         struct section *sec;
1947         struct symbol *func;
1948         int ret;
1949
1950         if (!file->rodata)
1951                 return 0;
1952
1953         for_each_sec(file, sec) {
1954                 list_for_each_entry(func, &sec->symbol_list, list) {
1955                         if (func->type != STT_FUNC)
1956                                 continue;
1957
1958                         mark_func_jump_tables(file, func);
1959                         ret = add_func_jump_tables(file, func);
1960                         if (ret)
1961                                 return ret;
1962                 }
1963         }
1964
1965         return 0;
1966 }
1967
1968 static void set_func_state(struct cfi_state *state)
1969 {
1970         state->cfa = initial_func_cfi.cfa;
1971         memcpy(&state->regs, &initial_func_cfi.regs,
1972                CFI_NUM_REGS * sizeof(struct cfi_reg));
1973         state->stack_size = initial_func_cfi.cfa.offset;
1974 }
1975
1976 static int read_unwind_hints(struct objtool_file *file)
1977 {
1978         struct cfi_state cfi = init_cfi;
1979         struct section *sec, *relocsec;
1980         struct unwind_hint *hint;
1981         struct instruction *insn;
1982         struct reloc *reloc;
1983         int i;
1984
1985         sec = find_section_by_name(file->elf, ".discard.unwind_hints");
1986         if (!sec)
1987                 return 0;
1988
1989         relocsec = sec->reloc;
1990         if (!relocsec) {
1991                 WARN("missing .rela.discard.unwind_hints section");
1992                 return -1;
1993         }
1994
1995         if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
1996                 WARN("struct unwind_hint size mismatch");
1997                 return -1;
1998         }
1999
2000         file->hints = true;
2001
2002         for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2003                 hint = (struct unwind_hint *)sec->data->d_buf + i;
2004
2005                 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2006                 if (!reloc) {
2007                         WARN("can't find reloc for unwind_hints[%d]", i);
2008                         return -1;
2009                 }
2010
2011                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2012                 if (!insn) {
2013                         WARN("can't find insn for unwind_hints[%d]", i);
2014                         return -1;
2015                 }
2016
2017                 insn->hint = true;
2018
2019                 if (opts.ibt && hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2020                         struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2021
2022                         if (sym && sym->bind == STB_GLOBAL &&
2023                             insn->type != INSN_ENDBR && !insn->noendbr) {
2024                                 WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR",
2025                                           insn->sec, insn->offset);
2026                         }
2027                 }
2028
2029                 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2030                         insn->cfi = &func_cfi;
2031                         continue;
2032                 }
2033
2034                 if (insn->cfi)
2035                         cfi = *(insn->cfi);
2036
2037                 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2038                         WARN_FUNC("unsupported unwind_hint sp base reg %d",
2039                                   insn->sec, insn->offset, hint->sp_reg);
2040                         return -1;
2041                 }
2042
2043                 cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
2044                 cfi.type = hint->type;
2045                 cfi.end = hint->end;
2046
2047                 insn->cfi = cfi_hash_find_or_add(&cfi);
2048         }
2049
2050         return 0;
2051 }
2052
2053 static int read_noendbr_hints(struct objtool_file *file)
2054 {
2055         struct section *sec;
2056         struct instruction *insn;
2057         struct reloc *reloc;
2058
2059         sec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2060         if (!sec)
2061                 return 0;
2062
2063         list_for_each_entry(reloc, &sec->reloc_list, list) {
2064                 insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend);
2065                 if (!insn) {
2066                         WARN("bad .discard.noendbr entry");
2067                         return -1;
2068                 }
2069
2070                 if (insn->type == INSN_ENDBR)
2071                         WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset);
2072
2073                 insn->noendbr = 1;
2074         }
2075
2076         return 0;
2077 }
2078
2079 static int read_retpoline_hints(struct objtool_file *file)
2080 {
2081         struct section *sec;
2082         struct instruction *insn;
2083         struct reloc *reloc;
2084
2085         sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2086         if (!sec)
2087                 return 0;
2088
2089         list_for_each_entry(reloc, &sec->reloc_list, list) {
2090                 if (reloc->sym->type != STT_SECTION) {
2091                         WARN("unexpected relocation symbol type in %s", sec->name);
2092                         return -1;
2093                 }
2094
2095                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2096                 if (!insn) {
2097                         WARN("bad .discard.retpoline_safe entry");
2098                         return -1;
2099                 }
2100
2101                 if (insn->type != INSN_JUMP_DYNAMIC &&
2102                     insn->type != INSN_CALL_DYNAMIC) {
2103                         WARN_FUNC("retpoline_safe hint not an indirect jump/call",
2104                                   insn->sec, insn->offset);
2105                         return -1;
2106                 }
2107
2108                 insn->retpoline_safe = true;
2109         }
2110
2111         return 0;
2112 }
2113
2114 static int read_instr_hints(struct objtool_file *file)
2115 {
2116         struct section *sec;
2117         struct instruction *insn;
2118         struct reloc *reloc;
2119
2120         sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2121         if (!sec)
2122                 return 0;
2123
2124         list_for_each_entry(reloc, &sec->reloc_list, list) {
2125                 if (reloc->sym->type != STT_SECTION) {
2126                         WARN("unexpected relocation symbol type in %s", sec->name);
2127                         return -1;
2128                 }
2129
2130                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2131                 if (!insn) {
2132                         WARN("bad .discard.instr_end entry");
2133                         return -1;
2134                 }
2135
2136                 insn->instr--;
2137         }
2138
2139         sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2140         if (!sec)
2141                 return 0;
2142
2143         list_for_each_entry(reloc, &sec->reloc_list, list) {
2144                 if (reloc->sym->type != STT_SECTION) {
2145                         WARN("unexpected relocation symbol type in %s", sec->name);
2146                         return -1;
2147                 }
2148
2149                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2150                 if (!insn) {
2151                         WARN("bad .discard.instr_begin entry");
2152                         return -1;
2153                 }
2154
2155                 insn->instr++;
2156         }
2157
2158         return 0;
2159 }
2160
2161 static int read_intra_function_calls(struct objtool_file *file)
2162 {
2163         struct instruction *insn;
2164         struct section *sec;
2165         struct reloc *reloc;
2166
2167         sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2168         if (!sec)
2169                 return 0;
2170
2171         list_for_each_entry(reloc, &sec->reloc_list, list) {
2172                 unsigned long dest_off;
2173
2174                 if (reloc->sym->type != STT_SECTION) {
2175                         WARN("unexpected relocation symbol type in %s",
2176                              sec->name);
2177                         return -1;
2178                 }
2179
2180                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2181                 if (!insn) {
2182                         WARN("bad .discard.intra_function_call entry");
2183                         return -1;
2184                 }
2185
2186                 if (insn->type != INSN_CALL) {
2187                         WARN_FUNC("intra_function_call not a direct call",
2188                                   insn->sec, insn->offset);
2189                         return -1;
2190                 }
2191
2192                 /*
2193                  * Treat intra-function CALLs as JMPs, but with a stack_op.
2194                  * See add_call_destinations(), which strips stack_ops from
2195                  * normal CALLs.
2196                  */
2197                 insn->type = INSN_JUMP_UNCONDITIONAL;
2198
2199                 dest_off = insn->offset + insn->len + insn->immediate;
2200                 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2201                 if (!insn->jump_dest) {
2202                         WARN_FUNC("can't find call dest at %s+0x%lx",
2203                                   insn->sec, insn->offset,
2204                                   insn->sec->name, dest_off);
2205                         return -1;
2206                 }
2207         }
2208
2209         return 0;
2210 }
2211
2212 /*
2213  * Return true if name matches an instrumentation function, where calls to that
2214  * function from noinstr code can safely be removed, but compilers won't do so.
2215  */
2216 static bool is_profiling_func(const char *name)
2217 {
2218         /*
2219          * Many compilers cannot disable KCOV with a function attribute.
2220          */
2221         if (!strncmp(name, "__sanitizer_cov_", 16))
2222                 return true;
2223
2224         /*
2225          * Some compilers currently do not remove __tsan_func_entry/exit nor
2226          * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2227          * the __no_sanitize_thread attribute, remove them. Once the kernel's
2228          * minimum Clang version is 14.0, this can be removed.
2229          */
2230         if (!strncmp(name, "__tsan_func_", 12) ||
2231             !strcmp(name, "__tsan_atomic_signal_fence"))
2232                 return true;
2233
2234         return false;
2235 }
2236
2237 static int classify_symbols(struct objtool_file *file)
2238 {
2239         struct section *sec;
2240         struct symbol *func;
2241
2242         for_each_sec(file, sec) {
2243                 list_for_each_entry(func, &sec->symbol_list, list) {
2244                         if (func->bind != STB_GLOBAL)
2245                                 continue;
2246
2247                         if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2248                                      strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2249                                 func->static_call_tramp = true;
2250
2251                         if (arch_is_retpoline(func))
2252                                 func->retpoline_thunk = true;
2253
2254                         if (arch_is_rethunk(func))
2255                                 func->return_thunk = true;
2256
2257                         if (!strcmp(func->name, "__fentry__"))
2258                                 func->fentry = true;
2259
2260                         if (is_profiling_func(func->name))
2261                                 func->profiling_func = true;
2262                 }
2263         }
2264
2265         return 0;
2266 }
2267
2268 static void mark_rodata(struct objtool_file *file)
2269 {
2270         struct section *sec;
2271         bool found = false;
2272
2273         /*
2274          * Search for the following rodata sections, each of which can
2275          * potentially contain jump tables:
2276          *
2277          * - .rodata: can contain GCC switch tables
2278          * - .rodata.<func>: same, if -fdata-sections is being used
2279          * - .rodata..c_jump_table: contains C annotated jump tables
2280          *
2281          * .rodata.str1.* sections are ignored; they don't contain jump tables.
2282          */
2283         for_each_sec(file, sec) {
2284                 if (!strncmp(sec->name, ".rodata", 7) &&
2285                     !strstr(sec->name, ".str1.")) {
2286                         sec->rodata = true;
2287                         found = true;
2288                 }
2289         }
2290
2291         file->rodata = found;
2292 }
2293
2294 static int decode_sections(struct objtool_file *file)
2295 {
2296         int ret;
2297
2298         mark_rodata(file);
2299
2300         ret = init_pv_ops(file);
2301         if (ret)
2302                 return ret;
2303
2304         ret = decode_instructions(file);
2305         if (ret)
2306                 return ret;
2307
2308         add_ignores(file);
2309         add_uaccess_safe(file);
2310
2311         ret = add_ignore_alternatives(file);
2312         if (ret)
2313                 return ret;
2314
2315         /*
2316          * Must be before read_unwind_hints() since that needs insn->noendbr.
2317          */
2318         ret = read_noendbr_hints(file);
2319         if (ret)
2320                 return ret;
2321
2322         /*
2323          * Must be before add_{jump_call}_destination.
2324          */
2325         ret = classify_symbols(file);
2326         if (ret)
2327                 return ret;
2328
2329         /*
2330          * Must be before add_jump_destinations(), which depends on 'func'
2331          * being set for alternatives, to enable proper sibling call detection.
2332          */
2333         ret = add_special_section_alts(file);
2334         if (ret)
2335                 return ret;
2336
2337         ret = add_jump_destinations(file);
2338         if (ret)
2339                 return ret;
2340
2341         /*
2342          * Must be before add_call_destination(); it changes INSN_CALL to
2343          * INSN_JUMP.
2344          */
2345         ret = read_intra_function_calls(file);
2346         if (ret)
2347                 return ret;
2348
2349         ret = add_call_destinations(file);
2350         if (ret)
2351                 return ret;
2352
2353         /*
2354          * Must be after add_call_destinations() such that it can override
2355          * dead_end_function() marks.
2356          */
2357         ret = add_dead_ends(file);
2358         if (ret)
2359                 return ret;
2360
2361         ret = add_jump_table_alts(file);
2362         if (ret)
2363                 return ret;
2364
2365         ret = read_unwind_hints(file);
2366         if (ret)
2367                 return ret;
2368
2369         ret = read_retpoline_hints(file);
2370         if (ret)
2371                 return ret;
2372
2373         ret = read_instr_hints(file);
2374         if (ret)
2375                 return ret;
2376
2377         return 0;
2378 }
2379
2380 static bool is_fentry_call(struct instruction *insn)
2381 {
2382         if (insn->type == INSN_CALL &&
2383             insn->call_dest &&
2384             insn->call_dest->fentry)
2385                 return true;
2386
2387         return false;
2388 }
2389
2390 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2391 {
2392         struct cfi_state *cfi = &state->cfi;
2393         int i;
2394
2395         if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2396                 return true;
2397
2398         if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2399                 return true;
2400
2401         if (cfi->stack_size != initial_func_cfi.cfa.offset)
2402                 return true;
2403
2404         for (i = 0; i < CFI_NUM_REGS; i++) {
2405                 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2406                     cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2407                         return true;
2408         }
2409
2410         return false;
2411 }
2412
2413 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2414                                 int expected_offset)
2415 {
2416         return reg->base == CFI_CFA &&
2417                reg->offset == expected_offset;
2418 }
2419
2420 static bool has_valid_stack_frame(struct insn_state *state)
2421 {
2422         struct cfi_state *cfi = &state->cfi;
2423
2424         if (cfi->cfa.base == CFI_BP &&
2425             check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2426             check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2427                 return true;
2428
2429         if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2430                 return true;
2431
2432         return false;
2433 }
2434
2435 static int update_cfi_state_regs(struct instruction *insn,
2436                                   struct cfi_state *cfi,
2437                                   struct stack_op *op)
2438 {
2439         struct cfi_reg *cfa = &cfi->cfa;
2440
2441         if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2442                 return 0;
2443
2444         /* push */
2445         if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2446                 cfa->offset += 8;
2447
2448         /* pop */
2449         if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2450                 cfa->offset -= 8;
2451
2452         /* add immediate to sp */
2453         if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2454             op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2455                 cfa->offset -= op->src.offset;
2456
2457         return 0;
2458 }
2459
2460 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2461 {
2462         if (arch_callee_saved_reg(reg) &&
2463             cfi->regs[reg].base == CFI_UNDEFINED) {
2464                 cfi->regs[reg].base = base;
2465                 cfi->regs[reg].offset = offset;
2466         }
2467 }
2468
2469 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2470 {
2471         cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2472         cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2473 }
2474
2475 /*
2476  * A note about DRAP stack alignment:
2477  *
2478  * GCC has the concept of a DRAP register, which is used to help keep track of
2479  * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2480  * register.  The typical DRAP pattern is:
2481  *
2482  *   4c 8d 54 24 08             lea    0x8(%rsp),%r10
2483  *   48 83 e4 c0                and    $0xffffffffffffffc0,%rsp
2484  *   41 ff 72 f8                pushq  -0x8(%r10)
2485  *   55                         push   %rbp
2486  *   48 89 e5                   mov    %rsp,%rbp
2487  *                              (more pushes)
2488  *   41 52                      push   %r10
2489  *                              ...
2490  *   41 5a                      pop    %r10
2491  *                              (more pops)
2492  *   5d                         pop    %rbp
2493  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2494  *   c3                         retq
2495  *
2496  * There are some variations in the epilogues, like:
2497  *
2498  *   5b                         pop    %rbx
2499  *   41 5a                      pop    %r10
2500  *   41 5c                      pop    %r12
2501  *   41 5d                      pop    %r13
2502  *   41 5e                      pop    %r14
2503  *   c9                         leaveq
2504  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2505  *   c3                         retq
2506  *
2507  * and:
2508  *
2509  *   4c 8b 55 e8                mov    -0x18(%rbp),%r10
2510  *   48 8b 5d e0                mov    -0x20(%rbp),%rbx
2511  *   4c 8b 65 f0                mov    -0x10(%rbp),%r12
2512  *   4c 8b 6d f8                mov    -0x8(%rbp),%r13
2513  *   c9                         leaveq
2514  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2515  *   c3                         retq
2516  *
2517  * Sometimes r13 is used as the DRAP register, in which case it's saved and
2518  * restored beforehand:
2519  *
2520  *   41 55                      push   %r13
2521  *   4c 8d 6c 24 10             lea    0x10(%rsp),%r13
2522  *   48 83 e4 f0                and    $0xfffffffffffffff0,%rsp
2523  *                              ...
2524  *   49 8d 65 f0                lea    -0x10(%r13),%rsp
2525  *   41 5d                      pop    %r13
2526  *   c3                         retq
2527  */
2528 static int update_cfi_state(struct instruction *insn,
2529                             struct instruction *next_insn,
2530                             struct cfi_state *cfi, struct stack_op *op)
2531 {
2532         struct cfi_reg *cfa = &cfi->cfa;
2533         struct cfi_reg *regs = cfi->regs;
2534
2535         /* stack operations don't make sense with an undefined CFA */
2536         if (cfa->base == CFI_UNDEFINED) {
2537                 if (insn->func) {
2538                         WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2539                         return -1;
2540                 }
2541                 return 0;
2542         }
2543
2544         if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2545             cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2546                 return update_cfi_state_regs(insn, cfi, op);
2547
2548         switch (op->dest.type) {
2549
2550         case OP_DEST_REG:
2551                 switch (op->src.type) {
2552
2553                 case OP_SRC_REG:
2554                         if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2555                             cfa->base == CFI_SP &&
2556                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2557
2558                                 /* mov %rsp, %rbp */
2559                                 cfa->base = op->dest.reg;
2560                                 cfi->bp_scratch = false;
2561                         }
2562
2563                         else if (op->src.reg == CFI_SP &&
2564                                  op->dest.reg == CFI_BP && cfi->drap) {
2565
2566                                 /* drap: mov %rsp, %rbp */
2567                                 regs[CFI_BP].base = CFI_BP;
2568                                 regs[CFI_BP].offset = -cfi->stack_size;
2569                                 cfi->bp_scratch = false;
2570                         }
2571
2572                         else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2573
2574                                 /*
2575                                  * mov %rsp, %reg
2576                                  *
2577                                  * This is needed for the rare case where GCC
2578                                  * does:
2579                                  *
2580                                  *   mov    %rsp, %rax
2581                                  *   ...
2582                                  *   mov    %rax, %rsp
2583                                  */
2584                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2585                                 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2586                         }
2587
2588                         else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2589                                  (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2590
2591                                 /*
2592                                  * mov %rbp, %rsp
2593                                  *
2594                                  * Restore the original stack pointer (Clang).
2595                                  */
2596                                 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2597                         }
2598
2599                         else if (op->dest.reg == cfa->base) {
2600
2601                                 /* mov %reg, %rsp */
2602                                 if (cfa->base == CFI_SP &&
2603                                     cfi->vals[op->src.reg].base == CFI_CFA) {
2604
2605                                         /*
2606                                          * This is needed for the rare case
2607                                          * where GCC does something dumb like:
2608                                          *
2609                                          *   lea    0x8(%rsp), %rcx
2610                                          *   ...
2611                                          *   mov    %rcx, %rsp
2612                                          */
2613                                         cfa->offset = -cfi->vals[op->src.reg].offset;
2614                                         cfi->stack_size = cfa->offset;
2615
2616                                 } else if (cfa->base == CFI_SP &&
2617                                            cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2618                                            cfi->vals[op->src.reg].offset == cfa->offset) {
2619
2620                                         /*
2621                                          * Stack swizzle:
2622                                          *
2623                                          * 1: mov %rsp, (%[tos])
2624                                          * 2: mov %[tos], %rsp
2625                                          *    ...
2626                                          * 3: pop %rsp
2627                                          *
2628                                          * Where:
2629                                          *
2630                                          * 1 - places a pointer to the previous
2631                                          *     stack at the Top-of-Stack of the
2632                                          *     new stack.
2633                                          *
2634                                          * 2 - switches to the new stack.
2635                                          *
2636                                          * 3 - pops the Top-of-Stack to restore
2637                                          *     the original stack.
2638                                          *
2639                                          * Note: we set base to SP_INDIRECT
2640                                          * here and preserve offset. Therefore
2641                                          * when the unwinder reaches ToS it
2642                                          * will dereference SP and then add the
2643                                          * offset to find the next frame, IOW:
2644                                          * (%rsp) + offset.
2645                                          */
2646                                         cfa->base = CFI_SP_INDIRECT;
2647
2648                                 } else {
2649                                         cfa->base = CFI_UNDEFINED;
2650                                         cfa->offset = 0;
2651                                 }
2652                         }
2653
2654                         else if (op->dest.reg == CFI_SP &&
2655                                  cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2656                                  cfi->vals[op->src.reg].offset == cfa->offset) {
2657
2658                                 /*
2659                                  * The same stack swizzle case 2) as above. But
2660                                  * because we can't change cfa->base, case 3)
2661                                  * will become a regular POP. Pretend we're a
2662                                  * PUSH so things don't go unbalanced.
2663                                  */
2664                                 cfi->stack_size += 8;
2665                         }
2666
2667
2668                         break;
2669
2670                 case OP_SRC_ADD:
2671                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2672
2673                                 /* add imm, %rsp */
2674                                 cfi->stack_size -= op->src.offset;
2675                                 if (cfa->base == CFI_SP)
2676                                         cfa->offset -= op->src.offset;
2677                                 break;
2678                         }
2679
2680                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2681
2682                                 /* lea disp(%rbp), %rsp */
2683                                 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2684                                 break;
2685                         }
2686
2687                         if (!cfi->drap && op->src.reg == CFI_SP &&
2688                             op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2689                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset + op->src.offset)) {
2690
2691                                 /* lea disp(%rsp), %rbp */
2692                                 cfa->base = CFI_BP;
2693                                 cfa->offset -= op->src.offset;
2694                                 cfi->bp_scratch = false;
2695                                 break;
2696                         }
2697
2698                         if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2699
2700                                 /* drap: lea disp(%rsp), %drap */
2701                                 cfi->drap_reg = op->dest.reg;
2702
2703                                 /*
2704                                  * lea disp(%rsp), %reg
2705                                  *
2706                                  * This is needed for the rare case where GCC
2707                                  * does something dumb like:
2708                                  *
2709                                  *   lea    0x8(%rsp), %rcx
2710                                  *   ...
2711                                  *   mov    %rcx, %rsp
2712                                  */
2713                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2714                                 cfi->vals[op->dest.reg].offset = \
2715                                         -cfi->stack_size + op->src.offset;
2716
2717                                 break;
2718                         }
2719
2720                         if (cfi->drap && op->dest.reg == CFI_SP &&
2721                             op->src.reg == cfi->drap_reg) {
2722
2723                                  /* drap: lea disp(%drap), %rsp */
2724                                 cfa->base = CFI_SP;
2725                                 cfa->offset = cfi->stack_size = -op->src.offset;
2726                                 cfi->drap_reg = CFI_UNDEFINED;
2727                                 cfi->drap = false;
2728                                 break;
2729                         }
2730
2731                         if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2732                                 WARN_FUNC("unsupported stack register modification",
2733                                           insn->sec, insn->offset);
2734                                 return -1;
2735                         }
2736
2737                         break;
2738
2739                 case OP_SRC_AND:
2740                         if (op->dest.reg != CFI_SP ||
2741                             (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2742                             (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2743                                 WARN_FUNC("unsupported stack pointer realignment",
2744                                           insn->sec, insn->offset);
2745                                 return -1;
2746                         }
2747
2748                         if (cfi->drap_reg != CFI_UNDEFINED) {
2749                                 /* drap: and imm, %rsp */
2750                                 cfa->base = cfi->drap_reg;
2751                                 cfa->offset = cfi->stack_size = 0;
2752                                 cfi->drap = true;
2753                         }
2754
2755                         /*
2756                          * Older versions of GCC (4.8ish) realign the stack
2757                          * without DRAP, with a frame pointer.
2758                          */
2759
2760                         break;
2761
2762                 case OP_SRC_POP:
2763                 case OP_SRC_POPF:
2764                         if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2765
2766                                 /* pop %rsp; # restore from a stack swizzle */
2767                                 cfa->base = CFI_SP;
2768                                 break;
2769                         }
2770
2771                         if (!cfi->drap && op->dest.reg == cfa->base) {
2772
2773                                 /* pop %rbp */
2774                                 cfa->base = CFI_SP;
2775                         }
2776
2777                         if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2778                             op->dest.reg == cfi->drap_reg &&
2779                             cfi->drap_offset == -cfi->stack_size) {
2780
2781                                 /* drap: pop %drap */
2782                                 cfa->base = cfi->drap_reg;
2783                                 cfa->offset = 0;
2784                                 cfi->drap_offset = -1;
2785
2786                         } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
2787
2788                                 /* pop %reg */
2789                                 restore_reg(cfi, op->dest.reg);
2790                         }
2791
2792                         cfi->stack_size -= 8;
2793                         if (cfa->base == CFI_SP)
2794                                 cfa->offset -= 8;
2795
2796                         break;
2797
2798                 case OP_SRC_REG_INDIRECT:
2799                         if (!cfi->drap && op->dest.reg == cfa->base &&
2800                             op->dest.reg == CFI_BP) {
2801
2802                                 /* mov disp(%rsp), %rbp */
2803                                 cfa->base = CFI_SP;
2804                                 cfa->offset = cfi->stack_size;
2805                         }
2806
2807                         if (cfi->drap && op->src.reg == CFI_BP &&
2808                             op->src.offset == cfi->drap_offset) {
2809
2810                                 /* drap: mov disp(%rbp), %drap */
2811                                 cfa->base = cfi->drap_reg;
2812                                 cfa->offset = 0;
2813                                 cfi->drap_offset = -1;
2814                         }
2815
2816                         if (cfi->drap && op->src.reg == CFI_BP &&
2817                             op->src.offset == regs[op->dest.reg].offset) {
2818
2819                                 /* drap: mov disp(%rbp), %reg */
2820                                 restore_reg(cfi, op->dest.reg);
2821
2822                         } else if (op->src.reg == cfa->base &&
2823                             op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
2824
2825                                 /* mov disp(%rbp), %reg */
2826                                 /* mov disp(%rsp), %reg */
2827                                 restore_reg(cfi, op->dest.reg);
2828
2829                         } else if (op->src.reg == CFI_SP &&
2830                                    op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
2831
2832                                 /* mov disp(%rsp), %reg */
2833                                 restore_reg(cfi, op->dest.reg);
2834                         }
2835
2836                         break;
2837
2838                 default:
2839                         WARN_FUNC("unknown stack-related instruction",
2840                                   insn->sec, insn->offset);
2841                         return -1;
2842                 }
2843
2844                 break;
2845
2846         case OP_DEST_PUSH:
2847         case OP_DEST_PUSHF:
2848                 cfi->stack_size += 8;
2849                 if (cfa->base == CFI_SP)
2850                         cfa->offset += 8;
2851
2852                 if (op->src.type != OP_SRC_REG)
2853                         break;
2854
2855                 if (cfi->drap) {
2856                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2857
2858                                 /* drap: push %drap */
2859                                 cfa->base = CFI_BP_INDIRECT;
2860                                 cfa->offset = -cfi->stack_size;
2861
2862                                 /* save drap so we know when to restore it */
2863                                 cfi->drap_offset = -cfi->stack_size;
2864
2865                         } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
2866
2867                                 /* drap: push %rbp */
2868                                 cfi->stack_size = 0;
2869
2870                         } else {
2871
2872                                 /* drap: push %reg */
2873                                 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
2874                         }
2875
2876                 } else {
2877
2878                         /* push %reg */
2879                         save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
2880                 }
2881
2882                 /* detect when asm code uses rbp as a scratch register */
2883                 if (opts.stackval && insn->func && op->src.reg == CFI_BP &&
2884                     cfa->base != CFI_BP)
2885                         cfi->bp_scratch = true;
2886                 break;
2887
2888         case OP_DEST_REG_INDIRECT:
2889
2890                 if (cfi->drap) {
2891                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2892
2893                                 /* drap: mov %drap, disp(%rbp) */
2894                                 cfa->base = CFI_BP_INDIRECT;
2895                                 cfa->offset = op->dest.offset;
2896
2897                                 /* save drap offset so we know when to restore it */
2898                                 cfi->drap_offset = op->dest.offset;
2899                         } else {
2900
2901                                 /* drap: mov reg, disp(%rbp) */
2902                                 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
2903                         }
2904
2905                 } else if (op->dest.reg == cfa->base) {
2906
2907                         /* mov reg, disp(%rbp) */
2908                         /* mov reg, disp(%rsp) */
2909                         save_reg(cfi, op->src.reg, CFI_CFA,
2910                                  op->dest.offset - cfi->cfa.offset);
2911
2912                 } else if (op->dest.reg == CFI_SP) {
2913
2914                         /* mov reg, disp(%rsp) */
2915                         save_reg(cfi, op->src.reg, CFI_CFA,
2916                                  op->dest.offset - cfi->stack_size);
2917
2918                 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
2919
2920                         /* mov %rsp, (%reg); # setup a stack swizzle. */
2921                         cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
2922                         cfi->vals[op->dest.reg].offset = cfa->offset;
2923                 }
2924
2925                 break;
2926
2927         case OP_DEST_MEM:
2928                 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
2929                         WARN_FUNC("unknown stack-related memory operation",
2930                                   insn->sec, insn->offset);
2931                         return -1;
2932                 }
2933
2934                 /* pop mem */
2935                 cfi->stack_size -= 8;
2936                 if (cfa->base == CFI_SP)
2937                         cfa->offset -= 8;
2938
2939                 break;
2940
2941         default:
2942                 WARN_FUNC("unknown stack-related instruction",
2943                           insn->sec, insn->offset);
2944                 return -1;
2945         }
2946
2947         return 0;
2948 }
2949
2950 /*
2951  * The stack layouts of alternatives instructions can sometimes diverge when
2952  * they have stack modifications.  That's fine as long as the potential stack
2953  * layouts don't conflict at any given potential instruction boundary.
2954  *
2955  * Flatten the CFIs of the different alternative code streams (both original
2956  * and replacement) into a single shared CFI array which can be used to detect
2957  * conflicts and nicely feed a linear array of ORC entries to the unwinder.
2958  */
2959 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
2960 {
2961         struct cfi_state **alt_cfi;
2962         int group_off;
2963
2964         if (!insn->alt_group)
2965                 return 0;
2966
2967         if (!insn->cfi) {
2968                 WARN("CFI missing");
2969                 return -1;
2970         }
2971
2972         alt_cfi = insn->alt_group->cfi;
2973         group_off = insn->offset - insn->alt_group->first_insn->offset;
2974
2975         if (!alt_cfi[group_off]) {
2976                 alt_cfi[group_off] = insn->cfi;
2977         } else {
2978                 if (cficmp(alt_cfi[group_off], insn->cfi)) {
2979                         WARN_FUNC("stack layout conflict in alternatives",
2980                                   insn->sec, insn->offset);
2981                         return -1;
2982                 }
2983         }
2984
2985         return 0;
2986 }
2987
2988 static int handle_insn_ops(struct instruction *insn,
2989                            struct instruction *next_insn,
2990                            struct insn_state *state)
2991 {
2992         struct stack_op *op;
2993
2994         list_for_each_entry(op, &insn->stack_ops, list) {
2995
2996                 if (update_cfi_state(insn, next_insn, &state->cfi, op))
2997                         return 1;
2998
2999                 if (!insn->alt_group)
3000                         continue;
3001
3002                 if (op->dest.type == OP_DEST_PUSHF) {
3003                         if (!state->uaccess_stack) {
3004                                 state->uaccess_stack = 1;
3005                         } else if (state->uaccess_stack >> 31) {
3006                                 WARN_FUNC("PUSHF stack exhausted",
3007                                           insn->sec, insn->offset);
3008                                 return 1;
3009                         }
3010                         state->uaccess_stack <<= 1;
3011                         state->uaccess_stack  |= state->uaccess;
3012                 }
3013
3014                 if (op->src.type == OP_SRC_POPF) {
3015                         if (state->uaccess_stack) {
3016                                 state->uaccess = state->uaccess_stack & 1;
3017                                 state->uaccess_stack >>= 1;
3018                                 if (state->uaccess_stack == 1)
3019                                         state->uaccess_stack = 0;
3020                         }
3021                 }
3022         }
3023
3024         return 0;
3025 }
3026
3027 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3028 {
3029         struct cfi_state *cfi1 = insn->cfi;
3030         int i;
3031
3032         if (!cfi1) {
3033                 WARN("CFI missing");
3034                 return false;
3035         }
3036
3037         if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3038
3039                 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3040                           insn->sec, insn->offset,
3041                           cfi1->cfa.base, cfi1->cfa.offset,
3042                           cfi2->cfa.base, cfi2->cfa.offset);
3043
3044         } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3045                 for (i = 0; i < CFI_NUM_REGS; i++) {
3046                         if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3047                                     sizeof(struct cfi_reg)))
3048                                 continue;
3049
3050                         WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3051                                   insn->sec, insn->offset,
3052                                   i, cfi1->regs[i].base, cfi1->regs[i].offset,
3053                                   i, cfi2->regs[i].base, cfi2->regs[i].offset);
3054                         break;
3055                 }
3056
3057         } else if (cfi1->type != cfi2->type) {
3058
3059                 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
3060                           insn->sec, insn->offset, cfi1->type, cfi2->type);
3061
3062         } else if (cfi1->drap != cfi2->drap ||
3063                    (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3064                    (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3065
3066                 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3067                           insn->sec, insn->offset,
3068                           cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3069                           cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3070
3071         } else
3072                 return true;
3073
3074         return false;
3075 }
3076
3077 static inline bool func_uaccess_safe(struct symbol *func)
3078 {
3079         if (func)
3080                 return func->uaccess_safe;
3081
3082         return false;
3083 }
3084
3085 static inline const char *call_dest_name(struct instruction *insn)
3086 {
3087         static char pvname[19];
3088         struct reloc *rel;
3089         int idx;
3090
3091         if (insn->call_dest)
3092                 return insn->call_dest->name;
3093
3094         rel = insn_reloc(NULL, insn);
3095         if (rel && !strcmp(rel->sym->name, "pv_ops")) {
3096                 idx = (rel->addend / sizeof(void *));
3097                 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3098                 return pvname;
3099         }
3100
3101         return "{dynamic}";
3102 }
3103
3104 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3105 {
3106         struct symbol *target;
3107         struct reloc *rel;
3108         int idx;
3109
3110         rel = insn_reloc(file, insn);
3111         if (!rel || strcmp(rel->sym->name, "pv_ops"))
3112                 return false;
3113
3114         idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *));
3115
3116         if (file->pv_ops[idx].clean)
3117                 return true;
3118
3119         file->pv_ops[idx].clean = true;
3120
3121         list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3122                 if (!target->sec->noinstr) {
3123                         WARN("pv_ops[%d]: %s", idx, target->name);
3124                         file->pv_ops[idx].clean = false;
3125                 }
3126         }
3127
3128         return file->pv_ops[idx].clean;
3129 }
3130
3131 static inline bool noinstr_call_dest(struct objtool_file *file,
3132                                      struct instruction *insn,
3133                                      struct symbol *func)
3134 {
3135         /*
3136          * We can't deal with indirect function calls at present;
3137          * assume they're instrumented.
3138          */
3139         if (!func) {
3140                 if (file->pv_ops)
3141                         return pv_call_dest(file, insn);
3142
3143                 return false;
3144         }
3145
3146         /*
3147          * If the symbol is from a noinstr section; we good.
3148          */
3149         if (func->sec->noinstr)
3150                 return true;
3151
3152         /*
3153          * The __ubsan_handle_*() calls are like WARN(), they only happen when
3154          * something 'BAD' happened. At the risk of taking the machine down,
3155          * let them proceed to get the message out.
3156          */
3157         if (!strncmp(func->name, "__ubsan_handle_", 15))
3158                 return true;
3159
3160         return false;
3161 }
3162
3163 static int validate_call(struct objtool_file *file,
3164                          struct instruction *insn,
3165                          struct insn_state *state)
3166 {
3167         if (state->noinstr && state->instr <= 0 &&
3168             !noinstr_call_dest(file, insn, insn->call_dest)) {
3169                 WARN_FUNC("call to %s() leaves .noinstr.text section",
3170                                 insn->sec, insn->offset, call_dest_name(insn));
3171                 return 1;
3172         }
3173
3174         if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
3175                 WARN_FUNC("call to %s() with UACCESS enabled",
3176                                 insn->sec, insn->offset, call_dest_name(insn));
3177                 return 1;
3178         }
3179
3180         if (state->df) {
3181                 WARN_FUNC("call to %s() with DF set",
3182                                 insn->sec, insn->offset, call_dest_name(insn));
3183                 return 1;
3184         }
3185
3186         return 0;
3187 }
3188
3189 static int validate_sibling_call(struct objtool_file *file,
3190                                  struct instruction *insn,
3191                                  struct insn_state *state)
3192 {
3193         if (has_modified_stack_frame(insn, state)) {
3194                 WARN_FUNC("sibling call from callable instruction with modified stack frame",
3195                                 insn->sec, insn->offset);
3196                 return 1;
3197         }
3198
3199         return validate_call(file, insn, state);
3200 }
3201
3202 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3203 {
3204         if (state->noinstr && state->instr > 0) {
3205                 WARN_FUNC("return with instrumentation enabled",
3206                           insn->sec, insn->offset);
3207                 return 1;
3208         }
3209
3210         if (state->uaccess && !func_uaccess_safe(func)) {
3211                 WARN_FUNC("return with UACCESS enabled",
3212                           insn->sec, insn->offset);
3213                 return 1;
3214         }
3215
3216         if (!state->uaccess && func_uaccess_safe(func)) {
3217                 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
3218                           insn->sec, insn->offset);
3219                 return 1;
3220         }
3221
3222         if (state->df) {
3223                 WARN_FUNC("return with DF set",
3224                           insn->sec, insn->offset);
3225                 return 1;
3226         }
3227
3228         if (func && has_modified_stack_frame(insn, state)) {
3229                 WARN_FUNC("return with modified stack frame",
3230                           insn->sec, insn->offset);
3231                 return 1;
3232         }
3233
3234         if (state->cfi.bp_scratch) {
3235                 WARN_FUNC("BP used as a scratch register",
3236                           insn->sec, insn->offset);
3237                 return 1;
3238         }
3239
3240         return 0;
3241 }
3242
3243 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3244                                                  struct instruction *insn)
3245 {
3246         struct alt_group *alt_group = insn->alt_group;
3247
3248         /*
3249          * Simulate the fact that alternatives are patched in-place.  When the
3250          * end of a replacement alt_group is reached, redirect objtool flow to
3251          * the end of the original alt_group.
3252          */
3253         if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
3254                 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3255
3256         return next_insn_same_sec(file, insn);
3257 }
3258
3259 /*
3260  * Follow the branch starting at the given instruction, and recursively follow
3261  * any other branches (jumps).  Meanwhile, track the frame pointer state at
3262  * each instruction and validate all the rules described in
3263  * tools/objtool/Documentation/stack-validation.txt.
3264  */
3265 static int validate_branch(struct objtool_file *file, struct symbol *func,
3266                            struct instruction *insn, struct insn_state state)
3267 {
3268         struct alternative *alt;
3269         struct instruction *next_insn, *prev_insn = NULL;
3270         struct section *sec;
3271         u8 visited;
3272         int ret;
3273
3274         sec = insn->sec;
3275
3276         while (1) {
3277                 next_insn = next_insn_to_validate(file, insn);
3278
3279                 if (func && insn->func && func != insn->func->pfunc) {
3280                         WARN("%s() falls through to next function %s()",
3281                              func->name, insn->func->name);
3282                         return 1;
3283                 }
3284
3285                 if (func && insn->ignore) {
3286                         WARN_FUNC("BUG: why am I validating an ignored function?",
3287                                   sec, insn->offset);
3288                         return 1;
3289                 }
3290
3291                 visited = 1 << state.uaccess;
3292                 if (insn->visited) {
3293                         if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3294                                 return 1;
3295
3296                         if (insn->visited & visited)
3297                                 return 0;
3298                 } else {
3299                         nr_insns_visited++;
3300                 }
3301
3302                 if (state.noinstr)
3303                         state.instr += insn->instr;
3304
3305                 if (insn->hint) {
3306                         state.cfi = *insn->cfi;
3307                 } else {
3308                         /* XXX track if we actually changed state.cfi */
3309
3310                         if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3311                                 insn->cfi = prev_insn->cfi;
3312                                 nr_cfi_reused++;
3313                         } else {
3314                                 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3315                         }
3316                 }
3317
3318                 insn->visited |= visited;
3319
3320                 if (propagate_alt_cfi(file, insn))
3321                         return 1;
3322
3323                 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3324                         bool skip_orig = false;
3325
3326                         list_for_each_entry(alt, &insn->alts, list) {
3327                                 if (alt->skip_orig)
3328                                         skip_orig = true;
3329
3330                                 ret = validate_branch(file, func, alt->insn, state);
3331                                 if (ret) {
3332                                         if (opts.backtrace)
3333                                                 BT_FUNC("(alt)", insn);
3334                                         return ret;
3335                                 }
3336                         }
3337
3338                         if (skip_orig)
3339                                 return 0;
3340                 }
3341
3342                 if (handle_insn_ops(insn, next_insn, &state))
3343                         return 1;
3344
3345                 switch (insn->type) {
3346
3347                 case INSN_RETURN:
3348                         return validate_return(func, insn, &state);
3349
3350                 case INSN_CALL:
3351                 case INSN_CALL_DYNAMIC:
3352                         ret = validate_call(file, insn, &state);
3353                         if (ret)
3354                                 return ret;
3355
3356                         if (opts.stackval && func && !is_fentry_call(insn) &&
3357                             !has_valid_stack_frame(&state)) {
3358                                 WARN_FUNC("call without frame pointer save/setup",
3359                                           sec, insn->offset);
3360                                 return 1;
3361                         }
3362
3363                         if (insn->dead_end)
3364                                 return 0;
3365
3366                         break;
3367
3368                 case INSN_JUMP_CONDITIONAL:
3369                 case INSN_JUMP_UNCONDITIONAL:
3370                         if (is_sibling_call(insn)) {
3371                                 ret = validate_sibling_call(file, insn, &state);
3372                                 if (ret)
3373                                         return ret;
3374
3375                         } else if (insn->jump_dest) {
3376                                 ret = validate_branch(file, func,
3377                                                       insn->jump_dest, state);
3378                                 if (ret) {
3379                                         if (opts.backtrace)
3380                                                 BT_FUNC("(branch)", insn);
3381                                         return ret;
3382                                 }
3383                         }
3384
3385                         if (insn->type == INSN_JUMP_UNCONDITIONAL)
3386                                 return 0;
3387
3388                         break;
3389
3390                 case INSN_JUMP_DYNAMIC:
3391                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3392                         if (is_sibling_call(insn)) {
3393                                 ret = validate_sibling_call(file, insn, &state);
3394                                 if (ret)
3395                                         return ret;
3396                         }
3397
3398                         if (insn->type == INSN_JUMP_DYNAMIC)
3399                                 return 0;
3400
3401                         break;
3402
3403                 case INSN_CONTEXT_SWITCH:
3404                         if (func && (!next_insn || !next_insn->hint)) {
3405                                 WARN_FUNC("unsupported instruction in callable function",
3406                                           sec, insn->offset);
3407                                 return 1;
3408                         }
3409                         return 0;
3410
3411                 case INSN_STAC:
3412                         if (state.uaccess) {
3413                                 WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
3414                                 return 1;
3415                         }
3416
3417                         state.uaccess = true;
3418                         break;
3419
3420                 case INSN_CLAC:
3421                         if (!state.uaccess && func) {
3422                                 WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
3423                                 return 1;
3424                         }
3425
3426                         if (func_uaccess_safe(func) && !state.uaccess_stack) {
3427                                 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
3428                                 return 1;
3429                         }
3430
3431                         state.uaccess = false;
3432                         break;
3433
3434                 case INSN_STD:
3435                         if (state.df) {
3436                                 WARN_FUNC("recursive STD", sec, insn->offset);
3437                                 return 1;
3438                         }
3439
3440                         state.df = true;
3441                         break;
3442
3443                 case INSN_CLD:
3444                         if (!state.df && func) {
3445                                 WARN_FUNC("redundant CLD", sec, insn->offset);
3446                                 return 1;
3447                         }
3448
3449                         state.df = false;
3450                         break;
3451
3452                 default:
3453                         break;
3454                 }
3455
3456                 if (insn->dead_end)
3457                         return 0;
3458
3459                 if (!next_insn) {
3460                         if (state.cfi.cfa.base == CFI_UNDEFINED)
3461                                 return 0;
3462                         WARN("%s: unexpected end of section", sec->name);
3463                         return 1;
3464                 }
3465
3466                 prev_insn = insn;
3467                 insn = next_insn;
3468         }
3469
3470         return 0;
3471 }
3472
3473 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3474 {
3475         struct instruction *insn;
3476         struct insn_state state;
3477         int ret, warnings = 0;
3478
3479         if (!file->hints)
3480                 return 0;
3481
3482         init_insn_state(file, &state, sec);
3483
3484         if (sec) {
3485                 insn = find_insn(file, sec, 0);
3486                 if (!insn)
3487                         return 0;
3488         } else {
3489                 insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3490         }
3491
3492         while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
3493                 if (insn->hint && !insn->visited && !insn->ignore) {
3494                         ret = validate_branch(file, insn->func, insn, state);
3495                         if (ret && opts.backtrace)
3496                                 BT_FUNC("<=== (hint)", insn);
3497                         warnings += ret;
3498                 }
3499
3500                 insn = list_next_entry(insn, list);
3501         }
3502
3503         return warnings;
3504 }
3505
3506 static int validate_retpoline(struct objtool_file *file)
3507 {
3508         struct instruction *insn;
3509         int warnings = 0;
3510
3511         for_each_insn(file, insn) {
3512                 if (insn->type != INSN_JUMP_DYNAMIC &&
3513                     insn->type != INSN_CALL_DYNAMIC)
3514                         continue;
3515
3516                 if (insn->retpoline_safe)
3517                         continue;
3518
3519                 /*
3520                  * .init.text code is ran before userspace and thus doesn't
3521                  * strictly need retpolines, except for modules which are
3522                  * loaded late, they very much do need retpoline in their
3523                  * .init.text
3524                  */
3525                 if (!strcmp(insn->sec->name, ".init.text") && !opts.module)
3526                         continue;
3527
3528                 WARN_FUNC("indirect %s found in RETPOLINE build",
3529                           insn->sec, insn->offset,
3530                           insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3531
3532                 warnings++;
3533         }
3534
3535         return warnings;
3536 }
3537
3538 static bool is_kasan_insn(struct instruction *insn)
3539 {
3540         return (insn->type == INSN_CALL &&
3541                 !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3542 }
3543
3544 static bool is_ubsan_insn(struct instruction *insn)
3545 {
3546         return (insn->type == INSN_CALL &&
3547                 !strcmp(insn->call_dest->name,
3548                         "__ubsan_handle_builtin_unreachable"));
3549 }
3550
3551 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
3552 {
3553         int i;
3554         struct instruction *prev_insn;
3555
3556         if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
3557                 return true;
3558
3559         /*
3560          * Ignore alternative replacement instructions.  This can happen
3561          * when a whitelisted function uses one of the ALTERNATIVE macros.
3562          */
3563         if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
3564             !strcmp(insn->sec->name, ".altinstr_aux"))
3565                 return true;
3566
3567         /*
3568          * Whole archive runs might encounter dead code from weak symbols.
3569          * This is where the linker will have dropped the weak symbol in
3570          * favour of a regular symbol, but leaves the code in place.
3571          *
3572          * In this case we'll find a piece of code (whole function) that is not
3573          * covered by a !section symbol. Ignore them.
3574          */
3575         if (opts.link && !insn->func) {
3576                 int size = find_symbol_hole_containing(insn->sec, insn->offset);
3577                 unsigned long end = insn->offset + size;
3578
3579                 if (!size) /* not a hole */
3580                         return false;
3581
3582                 if (size < 0) /* hole until the end */
3583                         return true;
3584
3585                 sec_for_each_insn_continue(file, insn) {
3586                         /*
3587                          * If we reach a visited instruction at or before the
3588                          * end of the hole, ignore the unreachable.
3589                          */
3590                         if (insn->visited)
3591                                 return true;
3592
3593                         if (insn->offset >= end)
3594                                 break;
3595
3596                         /*
3597                          * If this hole jumps to a .cold function, mark it ignore too.
3598                          */
3599                         if (insn->jump_dest && insn->jump_dest->func &&
3600                             strstr(insn->jump_dest->func->name, ".cold")) {
3601                                 struct instruction *dest = insn->jump_dest;
3602                                 func_for_each_insn(file, dest->func, dest)
3603                                         dest->ignore = true;
3604                         }
3605                 }
3606
3607                 return false;
3608         }
3609
3610         if (!insn->func)
3611                 return false;
3612
3613         if (insn->func->static_call_tramp)
3614                 return true;
3615
3616         /*
3617          * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
3618          * __builtin_unreachable().  The BUG() macro has an unreachable() after
3619          * the UD2, which causes GCC's undefined trap logic to emit another UD2
3620          * (or occasionally a JMP to UD2).
3621          *
3622          * It may also insert a UD2 after calling a __noreturn function.
3623          */
3624         prev_insn = list_prev_entry(insn, list);
3625         if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
3626             (insn->type == INSN_BUG ||
3627              (insn->type == INSN_JUMP_UNCONDITIONAL &&
3628               insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
3629                 return true;
3630
3631         /*
3632          * Check if this (or a subsequent) instruction is related to
3633          * CONFIG_UBSAN or CONFIG_KASAN.
3634          *
3635          * End the search at 5 instructions to avoid going into the weeds.
3636          */
3637         for (i = 0; i < 5; i++) {
3638
3639                 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
3640                         return true;
3641
3642                 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3643                         if (insn->jump_dest &&
3644                             insn->jump_dest->func == insn->func) {
3645                                 insn = insn->jump_dest;
3646                                 continue;
3647                         }
3648
3649                         break;
3650                 }
3651
3652                 if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
3653                         break;
3654
3655                 insn = list_next_entry(insn, list);
3656         }
3657
3658         return false;
3659 }
3660
3661 static int validate_symbol(struct objtool_file *file, struct section *sec,
3662                            struct symbol *sym, struct insn_state *state)
3663 {
3664         struct instruction *insn;
3665         int ret;
3666
3667         if (!sym->len) {
3668                 WARN("%s() is missing an ELF size annotation", sym->name);
3669                 return 1;
3670         }
3671
3672         if (sym->pfunc != sym || sym->alias != sym)
3673                 return 0;
3674
3675         insn = find_insn(file, sec, sym->offset);
3676         if (!insn || insn->ignore || insn->visited)
3677                 return 0;
3678
3679         state->uaccess = sym->uaccess_safe;
3680
3681         ret = validate_branch(file, insn->func, insn, *state);
3682         if (ret && opts.backtrace)
3683                 BT_FUNC("<=== (sym)", insn);
3684         return ret;
3685 }
3686
3687 static int validate_section(struct objtool_file *file, struct section *sec)
3688 {
3689         struct insn_state state;
3690         struct symbol *func;
3691         int warnings = 0;
3692
3693         list_for_each_entry(func, &sec->symbol_list, list) {
3694                 if (func->type != STT_FUNC)
3695                         continue;
3696
3697                 init_insn_state(file, &state, sec);
3698                 set_func_state(&state.cfi);
3699
3700                 warnings += validate_symbol(file, sec, func, &state);
3701         }
3702
3703         return warnings;
3704 }
3705
3706 static int validate_noinstr_sections(struct objtool_file *file)
3707 {
3708         struct section *sec;
3709         int warnings = 0;
3710
3711         sec = find_section_by_name(file->elf, ".noinstr.text");
3712         if (sec) {
3713                 warnings += validate_section(file, sec);
3714                 warnings += validate_unwind_hints(file, sec);
3715         }
3716
3717         sec = find_section_by_name(file->elf, ".entry.text");
3718         if (sec) {
3719                 warnings += validate_section(file, sec);
3720                 warnings += validate_unwind_hints(file, sec);
3721         }
3722
3723         return warnings;
3724 }
3725
3726 static int validate_functions(struct objtool_file *file)
3727 {
3728         struct section *sec;
3729         int warnings = 0;
3730
3731         for_each_sec(file, sec) {
3732                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
3733                         continue;
3734
3735                 warnings += validate_section(file, sec);
3736         }
3737
3738         return warnings;
3739 }
3740
3741 static void mark_endbr_used(struct instruction *insn)
3742 {
3743         if (!list_empty(&insn->call_node))
3744                 list_del_init(&insn->call_node);
3745 }
3746
3747 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
3748 {
3749         struct instruction *dest;
3750         struct reloc *reloc;
3751         unsigned long off;
3752         int warnings = 0;
3753
3754         /*
3755          * Looking for function pointer load relocations.  Ignore
3756          * direct/indirect branches:
3757          */
3758         switch (insn->type) {
3759         case INSN_CALL:
3760         case INSN_CALL_DYNAMIC:
3761         case INSN_JUMP_CONDITIONAL:
3762         case INSN_JUMP_UNCONDITIONAL:
3763         case INSN_JUMP_DYNAMIC:
3764         case INSN_JUMP_DYNAMIC_CONDITIONAL:
3765         case INSN_RETURN:
3766         case INSN_NOP:
3767                 return 0;
3768         default:
3769                 break;
3770         }
3771
3772         for (reloc = insn_reloc(file, insn);
3773              reloc;
3774              reloc = find_reloc_by_dest_range(file->elf, insn->sec,
3775                                               reloc->offset + 1,
3776                                               (insn->offset + insn->len) - (reloc->offset + 1))) {
3777
3778                 /*
3779                  * static_call_update() references the trampoline, which
3780                  * doesn't have (or need) ENDBR.  Skip warning in that case.
3781                  */
3782                 if (reloc->sym->static_call_tramp)
3783                         continue;
3784
3785                 off = reloc->sym->offset;
3786                 if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)
3787                         off += arch_dest_reloc_offset(reloc->addend);
3788                 else
3789                         off += reloc->addend;
3790
3791                 dest = find_insn(file, reloc->sym->sec, off);
3792                 if (!dest)
3793                         continue;
3794
3795                 if (dest->type == INSN_ENDBR) {
3796                         mark_endbr_used(dest);
3797                         continue;
3798                 }
3799
3800                 if (dest->func && dest->func == insn->func) {
3801                         /*
3802                          * Anything from->to self is either _THIS_IP_ or
3803                          * IRET-to-self.
3804                          *
3805                          * There is no sane way to annotate _THIS_IP_ since the
3806                          * compiler treats the relocation as a constant and is
3807                          * happy to fold in offsets, skewing any annotation we
3808                          * do, leading to vast amounts of false-positives.
3809                          *
3810                          * There's also compiler generated _THIS_IP_ through
3811                          * KCOV and such which we have no hope of annotating.
3812                          *
3813                          * As such, blanket accept self-references without
3814                          * issue.
3815                          */
3816                         continue;
3817                 }
3818
3819                 if (dest->noendbr)
3820                         continue;
3821
3822                 WARN_FUNC("relocation to !ENDBR: %s",
3823                           insn->sec, insn->offset,
3824                           offstr(dest->sec, dest->offset));
3825
3826                 warnings++;
3827         }
3828
3829         return warnings;
3830 }
3831
3832 static int validate_ibt_data_reloc(struct objtool_file *file,
3833                                    struct reloc *reloc)
3834 {
3835         struct instruction *dest;
3836
3837         dest = find_insn(file, reloc->sym->sec,
3838                          reloc->sym->offset + reloc->addend);
3839         if (!dest)
3840                 return 0;
3841
3842         if (dest->type == INSN_ENDBR) {
3843                 mark_endbr_used(dest);
3844                 return 0;
3845         }
3846
3847         if (dest->noendbr)
3848                 return 0;
3849
3850         WARN_FUNC("data relocation to !ENDBR: %s",
3851                   reloc->sec->base, reloc->offset,
3852                   offstr(dest->sec, dest->offset));
3853
3854         return 1;
3855 }
3856
3857 /*
3858  * Validate IBT rules and remove used ENDBR instructions from the seal list.
3859  * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
3860  * NOPs) later, in create_ibt_endbr_seal_sections().
3861  */
3862 static int validate_ibt(struct objtool_file *file)
3863 {
3864         struct section *sec;
3865         struct reloc *reloc;
3866         struct instruction *insn;
3867         int warnings = 0;
3868
3869         for_each_insn(file, insn)
3870                 warnings += validate_ibt_insn(file, insn);
3871
3872         for_each_sec(file, sec) {
3873
3874                 /* Already done by validate_ibt_insn() */
3875                 if (sec->sh.sh_flags & SHF_EXECINSTR)
3876                         continue;
3877
3878                 if (!sec->reloc)
3879                         continue;
3880
3881                 /*
3882                  * These sections can reference text addresses, but not with
3883                  * the intent to indirect branch to them.
3884                  */
3885                 if (!strncmp(sec->name, ".discard", 8)                  ||
3886                     !strncmp(sec->name, ".debug", 6)                    ||
3887                     !strcmp(sec->name, ".altinstructions")              ||
3888                     !strcmp(sec->name, ".ibt_endbr_seal")               ||
3889                     !strcmp(sec->name, ".orc_unwind_ip")                ||
3890                     !strcmp(sec->name, ".parainstructions")             ||
3891                     !strcmp(sec->name, ".retpoline_sites")              ||
3892                     !strcmp(sec->name, ".smp_locks")                    ||
3893                     !strcmp(sec->name, ".static_call_sites")            ||
3894                     !strcmp(sec->name, "_error_injection_whitelist")    ||
3895                     !strcmp(sec->name, "_kprobe_blacklist")             ||
3896                     !strcmp(sec->name, "__bug_table")                   ||
3897                     !strcmp(sec->name, "__ex_table")                    ||
3898                     !strcmp(sec->name, "__jump_table")                  ||
3899                     !strcmp(sec->name, "__mcount_loc")                  ||
3900                     !strcmp(sec->name, "__tracepoints"))
3901                         continue;
3902
3903                 list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
3904                         warnings += validate_ibt_data_reloc(file, reloc);
3905         }
3906
3907         return warnings;
3908 }
3909
3910 static int validate_sls(struct objtool_file *file)
3911 {
3912         struct instruction *insn, *next_insn;
3913         int warnings = 0;
3914
3915         for_each_insn(file, insn) {
3916                 next_insn = next_insn_same_sec(file, insn);
3917
3918                 if (insn->retpoline_safe)
3919                         continue;
3920
3921                 switch (insn->type) {
3922                 case INSN_RETURN:
3923                         if (!next_insn || next_insn->type != INSN_TRAP) {
3924                                 WARN_FUNC("missing int3 after ret",
3925                                           insn->sec, insn->offset);
3926                                 warnings++;
3927                         }
3928
3929                         break;
3930                 case INSN_JUMP_DYNAMIC:
3931                         if (!next_insn || next_insn->type != INSN_TRAP) {
3932                                 WARN_FUNC("missing int3 after indirect jump",
3933                                           insn->sec, insn->offset);
3934                                 warnings++;
3935                         }
3936                         break;
3937                 default:
3938                         break;
3939                 }
3940         }
3941
3942         return warnings;
3943 }
3944
3945 static int validate_reachable_instructions(struct objtool_file *file)
3946 {
3947         struct instruction *insn;
3948
3949         if (file->ignore_unreachables)
3950                 return 0;
3951
3952         for_each_insn(file, insn) {
3953                 if (insn->visited || ignore_unreachable_insn(file, insn))
3954                         continue;
3955
3956                 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
3957                 return 1;
3958         }
3959
3960         return 0;
3961 }
3962
3963 int check(struct objtool_file *file)
3964 {
3965         int ret, warnings = 0;
3966
3967         arch_initial_func_cfi_state(&initial_func_cfi);
3968         init_cfi_state(&init_cfi);
3969         init_cfi_state(&func_cfi);
3970         set_func_state(&func_cfi);
3971
3972         if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
3973                 goto out;
3974
3975         cfi_hash_add(&init_cfi);
3976         cfi_hash_add(&func_cfi);
3977
3978         ret = decode_sections(file);
3979         if (ret < 0)
3980                 goto out;
3981
3982         warnings += ret;
3983
3984         if (list_empty(&file->insn_list))
3985                 goto out;
3986
3987         if (opts.retpoline) {
3988                 ret = validate_retpoline(file);
3989                 if (ret < 0)
3990                         return ret;
3991                 warnings += ret;
3992         }
3993
3994         if (opts.stackval || opts.orc || opts.uaccess) {
3995                 ret = validate_functions(file);
3996                 if (ret < 0)
3997                         goto out;
3998                 warnings += ret;
3999
4000                 ret = validate_unwind_hints(file, NULL);
4001                 if (ret < 0)
4002                         goto out;
4003                 warnings += ret;
4004
4005                 if (!warnings) {
4006                         ret = validate_reachable_instructions(file);
4007                         if (ret < 0)
4008                                 goto out;
4009                         warnings += ret;
4010                 }
4011
4012         } else if (opts.noinstr) {
4013                 ret = validate_noinstr_sections(file);
4014                 if (ret < 0)
4015                         goto out;
4016                 warnings += ret;
4017         }
4018
4019         if (opts.ibt) {
4020                 ret = validate_ibt(file);
4021                 if (ret < 0)
4022                         goto out;
4023                 warnings += ret;
4024         }
4025
4026         if (opts.sls) {
4027                 ret = validate_sls(file);
4028                 if (ret < 0)
4029                         goto out;
4030                 warnings += ret;
4031         }
4032
4033         if (opts.static_call) {
4034                 ret = create_static_call_sections(file);
4035                 if (ret < 0)
4036                         goto out;
4037                 warnings += ret;
4038         }
4039
4040         if (opts.retpoline) {
4041                 ret = create_retpoline_sites_sections(file);
4042                 if (ret < 0)
4043                         goto out;
4044                 warnings += ret;
4045
4046                 ret = create_return_sites_sections(file);
4047                 if (ret < 0)
4048                         goto out;
4049                 warnings += ret;
4050         }
4051
4052         if (opts.mcount) {
4053                 ret = create_mcount_loc_sections(file);
4054                 if (ret < 0)
4055                         goto out;
4056                 warnings += ret;
4057         }
4058
4059         if (opts.ibt) {
4060                 ret = create_ibt_endbr_seal_sections(file);
4061                 if (ret < 0)
4062                         goto out;
4063                 warnings += ret;
4064         }
4065
4066         if (opts.orc && !list_empty(&file->insn_list)) {
4067                 ret = orc_create(file);
4068                 if (ret < 0)
4069                         goto out;
4070                 warnings += ret;
4071         }
4072
4073
4074         if (opts.stats) {
4075                 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4076                 printf("nr_cfi: %ld\n", nr_cfi);
4077                 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4078                 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4079         }
4080
4081 out:
4082         /*
4083          *  For now, don't fail the kernel build on fatal warnings.  These
4084          *  errors are still fairly common due to the growing matrix of
4085          *  supported toolchains and their recent pace of change.
4086          */
4087         return 0;
4088 }