tools/objtool/check.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
   4  */
   5
   6 #include <string.h>
   7 #include <stdlib.h>
   8 #include <inttypes.h>
   9 #include <sys/mman.h>
  10
  11 #include <arch/elf.h>
  12 #include <objtool/builtin.h>
  13 #include <objtool/cfi.h>
  14 #include <objtool/arch.h>
  15 #include <objtool/check.h>
  16 #include <objtool/special.h>
  17 #include <objtool/warn.h>
  18 #include <objtool/endianness.h>
  19
  20 #include <linux/objtool.h>
  21 #include <linux/hashtable.h>
  22 #include <linux/kernel.h>
  23 #include <linux/static_call_types.h>
  24
  25 struct alternative {
  26         struct list_head list;
  27         struct instruction *insn;
  28         bool skip_orig;
  29 };
  30
  31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
  32
  33 static struct cfi_init_state initial_func_cfi;
  34 static struct cfi_state init_cfi;
  35 static struct cfi_state func_cfi;
  36
  37 struct instruction *find_insn(struct objtool_file *file,
  38                               struct section *sec, unsigned long offset)
  39 {
  40         struct instruction *insn;
  41
  42         hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
  43                 if (insn->sec == sec && insn->offset == offset)
  44                         return insn;
  45         }
  46
  47         return NULL;
  48 }
  49
  50 static struct instruction *next_insn_same_sec(struct objtool_file *file,
  51                                               struct instruction *insn)
  52 {
  53         struct instruction *next = list_next_entry(insn, list);
  54
  55         if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
  56                 return NULL;
  57
  58         return next;
  59 }
  60
  61 static struct instruction *next_insn_same_func(struct objtool_file *file,
  62                                                struct instruction *insn)
  63 {
  64         struct instruction *next = list_next_entry(insn, list);
  65         struct symbol *func = insn->func;
  66
  67         if (!func)
  68                 return NULL;
  69
  70         if (&next->list != &file->insn_list && next->func == func)
  71                 return next;
  72
  73         /* Check if we're already in the subfunction: */
  74         if (func == func->cfunc)
  75                 return NULL;
  76
  77         /* Move to the subfunction: */
  78         return find_insn(file, func->cfunc->sec, func->cfunc->offset);
  79 }
  80
  81 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
  82                                                struct instruction *insn)
  83 {
  84         struct instruction *prev = list_prev_entry(insn, list);
  85
  86         if (&prev->list != &file->insn_list && prev->func == insn->func)
  87                 return prev;
  88
  89         return NULL;
  90 }
  91
  92 #define func_for_each_insn(file, func, insn)                            \
  93         for (insn = find_insn(file, func->sec, func->offset);           \
  94              insn;                                                      \
  95              insn = next_insn_same_func(file, insn))
  96
  97 #define sym_for_each_insn(file, sym, insn)                              \
  98         for (insn = find_insn(file, sym->sec, sym->offset);             \
  99              insn && &insn->list != &file->insn_list &&                 \
 100                 insn->sec == sym->sec &&                                \
 101                 insn->offset < sym->offset + sym->len;                  \
 102              insn = list_next_entry(insn, list))
 103
 104 #define sym_for_each_insn_continue_reverse(file, sym, insn)             \
 105         for (insn = list_prev_entry(insn, list);                        \
 106              &insn->list != &file->insn_list &&                         \
 107                 insn->sec == sym->sec && insn->offset >= sym->offset;   \
 108              insn = list_prev_entry(insn, list))
 109
 110 #define sec_for_each_insn_from(file, insn)                              \
 111         for (; insn; insn = next_insn_same_sec(file, insn))
 112
 113 #define sec_for_each_insn_continue(file, insn)                          \
 114         for (insn = next_insn_same_sec(file, insn); insn;               \
 115              insn = next_insn_same_sec(file, insn))
 116
 117 static bool is_jump_table_jump(struct instruction *insn)
 118 {
 119         struct alt_group *alt_group = insn->alt_group;
 120
 121         if (insn->jump_table)
 122                 return true;
 123
 124         /* Retpoline alternative for a jump table? */
 125         return alt_group && alt_group->orig_group &&
 126                alt_group->orig_group->first_insn->jump_table;
 127 }
 128
 129 static bool is_sibling_call(struct instruction *insn)
 130 {
 131         /*
 132          * Assume only ELF functions can make sibling calls.  This ensures
 133          * sibling call detection consistency between vmlinux.o and individual
 134          * objects.
 135          */
 136         if (!insn->func)
 137                 return false;
 138
 139         /* An indirect jump is either a sibling call or a jump to a table. */
 140         if (insn->type == INSN_JUMP_DYNAMIC)
 141                 return !is_jump_table_jump(insn);
 142
 143         /* add_jump_destinations() sets insn->call_dest for sibling calls. */
 144         return (is_static_jump(insn) && insn->call_dest);
 145 }
 146
 147 /*
 148  * This checks to see if the given function is a "noreturn" function.
 149  *
 150  * For global functions which are outside the scope of this object file, we
 151  * have to keep a manual list of them.
 152  *
 153  * For local functions, we have to detect them manually by simply looking for
 154  * the lack of a return instruction.
 155  */
 156 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 157                                 int recursion)
 158 {
 159         int i;
 160         struct instruction *insn;
 161         bool empty = true;
 162
 163         /*
 164          * Unfortunately these have to be hard coded because the noreturn
 165          * attribute isn't provided in ELF data.
 166          */
 167         static const char * const global_noreturns[] = {
 168                 "__stack_chk_fail",
 169                 "panic",
 170                 "do_exit",
 171                 "do_task_dead",
 172                 "kthread_exit",
 173                 "make_task_dead",
 174                 "__module_put_and_kthread_exit",
 175                 "kthread_complete_and_exit",
 176                 "__reiserfs_panic",
 177                 "lbug_with_loc",
 178                 "fortify_panic",
 179                 "usercopy_abort",
 180                 "machine_real_restart",
 181                 "rewind_stack_and_make_dead",
 182                 "kunit_try_catch_throw",
 183                 "xen_start_kernel",
 184                 "cpu_bringup_and_idle",
 185                 "do_group_exit",
 186                 "stop_this_cpu",
 187                 "__invalid_creds",
 188                 "cpu_startup_entry",
 189                 "__ubsan_handle_builtin_unreachable",
 190                 "ex_handler_msr_mce",
 191         };
 192
 193         if (!func)
 194                 return false;
 195
 196         if (func->bind == STB_WEAK)
 197                 return false;
 198
 199         if (func->bind == STB_GLOBAL)
 200                 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
 201                         if (!strcmp(func->name, global_noreturns[i]))
 202                                 return true;
 203
 204         if (!func->len)
 205                 return false;
 206
 207         insn = find_insn(file, func->sec, func->offset);
 208         if (!insn->func)
 209                 return false;
 210
 211         func_for_each_insn(file, func, insn) {
 212                 empty = false;
 213
 214                 if (insn->type == INSN_RETURN)
 215                         return false;
 216         }
 217
 218         if (empty)
 219                 return false;
 220
 221         /*
 222          * A function can have a sibling call instead of a return.  In that
 223          * case, the function's dead-end status depends on whether the target
 224          * of the sibling call returns.
 225          */
 226         func_for_each_insn(file, func, insn) {
 227                 if (is_sibling_call(insn)) {
 228                         struct instruction *dest = insn->jump_dest;
 229
 230                         if (!dest)
 231                                 /* sibling call to another file */
 232                                 return false;
 233
 234                         /* local sibling call */
 235                         if (recursion == 5) {
 236                                 /*
 237                                  * Infinite recursion: two functions have
 238                                  * sibling calls to each other.  This is a very
 239                                  * rare case.  It means they aren't dead ends.
 240                                  */
 241                                 return false;
 242                         }
 243
 244                         return __dead_end_function(file, dest->func, recursion+1);
 245                 }
 246         }
 247
 248         return true;
 249 }
 250
 251 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
 252 {
 253         return __dead_end_function(file, func, 0);
 254 }
 255
 256 static void init_cfi_state(struct cfi_state *cfi)
 257 {
 258         int i;
 259
 260         for (i = 0; i < CFI_NUM_REGS; i++) {
 261                 cfi->regs[i].base = CFI_UNDEFINED;
 262                 cfi->vals[i].base = CFI_UNDEFINED;
 263         }
 264         cfi->cfa.base = CFI_UNDEFINED;
 265         cfi->drap_reg = CFI_UNDEFINED;
 266         cfi->drap_offset = -1;
 267 }
 268
 269 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
 270                             struct section *sec)
 271 {
 272         memset(state, 0, sizeof(*state));
 273         init_cfi_state(&state->cfi);
 274
 275         /*
 276          * We need the full vmlinux for noinstr validation, otherwise we can
 277          * not correctly determine insn->call_dest->sec (external symbols do
 278          * not have a section).
 279          */
 280         if (opts.link && opts.noinstr && sec)
 281                 state->noinstr = sec->noinstr;
 282 }
 283
 284 static struct cfi_state *cfi_alloc(void)
 285 {
 286         struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
 287         if (!cfi) {
 288                 WARN("calloc failed");
 289                 exit(1);
 290         }
 291         nr_cfi++;
 292         return cfi;
 293 }
 294
 295 static int cfi_bits;
 296 static struct hlist_head *cfi_hash;
 297
 298 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
 299 {
 300         return memcmp((void *)cfi1 + sizeof(cfi1->hash),
 301                       (void *)cfi2 + sizeof(cfi2->hash),
 302                       sizeof(struct cfi_state) - sizeof(struct hlist_node));
 303 }
 304
 305 static inline u32 cfi_key(struct cfi_state *cfi)
 306 {
 307         return jhash((void *)cfi + sizeof(cfi->hash),
 308                      sizeof(*cfi) - sizeof(cfi->hash), 0);
 309 }
 310
 311 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
 312 {
 313         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 314         struct cfi_state *obj;
 315
 316         hlist_for_each_entry(obj, head, hash) {
 317                 if (!cficmp(cfi, obj)) {
 318                         nr_cfi_cache++;
 319                         return obj;
 320                 }
 321         }
 322
 323         obj = cfi_alloc();
 324         *obj = *cfi;
 325         hlist_add_head(&obj->hash, head);
 326
 327         return obj;
 328 }
 329
 330 static void cfi_hash_add(struct cfi_state *cfi)
 331 {
 332         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 333
 334         hlist_add_head(&cfi->hash, head);
 335 }
 336
 337 static void *cfi_hash_alloc(unsigned long size)
 338 {
 339         cfi_bits = max(10, ilog2(size));
 340         cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
 341                         PROT_READ|PROT_WRITE,
 342                         MAP_PRIVATE|MAP_ANON, -1, 0);
 343         if (cfi_hash == (void *)-1L) {
 344                 WARN("mmap fail cfi_hash");
 345                 cfi_hash = NULL;
 346         }  else if (opts.stats) {
 347                 printf("cfi_bits: %d\n", cfi_bits);
 348         }
 349
 350         return cfi_hash;
 351 }
 352
 353 static unsigned long nr_insns;
 354 static unsigned long nr_insns_visited;
 355
 356 /*
 357  * Call the arch-specific instruction decoder for all the instructions and add
 358  * them to the global instruction list.
 359  */
 360 static int decode_instructions(struct objtool_file *file)
 361 {
 362         struct section *sec;
 363         struct symbol *func;
 364         unsigned long offset;
 365         struct instruction *insn;
 366         int ret;
 367
 368         for_each_sec(file, sec) {
 369
 370                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
 371                         continue;
 372
 373                 if (strcmp(sec->name, ".altinstr_replacement") &&
 374                     strcmp(sec->name, ".altinstr_aux") &&
 375                     strncmp(sec->name, ".discard.", 9))
 376                         sec->text = true;
 377
 378                 if (!strcmp(sec->name, ".noinstr.text") ||
 379                     !strcmp(sec->name, ".entry.text"))
 380                         sec->noinstr = true;
 381
 382                 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
 383                         insn = malloc(sizeof(*insn));
 384                         if (!insn) {
 385                                 WARN("malloc failed");
 386                                 return -1;
 387                         }
 388                         memset(insn, 0, sizeof(*insn));
 389                         INIT_LIST_HEAD(&insn->alts);
 390                         INIT_LIST_HEAD(&insn->stack_ops);
 391                         INIT_LIST_HEAD(&insn->call_node);
 392
 393                         insn->sec = sec;
 394                         insn->offset = offset;
 395
 396                         ret = arch_decode_instruction(file, sec, offset,
 397                                                       sec->sh.sh_size - offset,
 398                                                       &insn->len, &insn->type,
 399                                                       &insn->immediate,
 400                                                       &insn->stack_ops);
 401                         if (ret)
 402                                 goto err;
 403
 404                         /*
 405                          * By default, "ud2" is a dead end unless otherwise
 406                          * annotated, because GCC 7 inserts it for certain
 407                          * divide-by-zero cases.
 408                          */
 409                         if (insn->type == INSN_BUG)
 410                                 insn->dead_end = true;
 411
 412                         hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
 413                         list_add_tail(&insn->list, &file->insn_list);
 414                         nr_insns++;
 415                 }
 416
 417                 list_for_each_entry(func, &sec->symbol_list, list) {
 418                         if (func->type != STT_FUNC || func->alias != func)
 419                                 continue;
 420
 421                         if (!find_insn(file, sec, func->offset)) {
 422                                 WARN("%s(): can't find starting instruction",
 423                                      func->name);
 424                                 return -1;
 425                         }
 426
 427                         sym_for_each_insn(file, func, insn) {
 428                                 insn->func = func;
 429                                 if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) {
 430                                         if (insn->offset == insn->func->offset) {
 431                                                 list_add_tail(&insn->call_node, &file->endbr_list);
 432                                                 file->nr_endbr++;
 433                                         } else {
 434                                                 file->nr_endbr_int++;
 435                                         }
 436                                 }
 437                         }
 438                 }
 439         }
 440
 441         if (opts.stats)
 442                 printf("nr_insns: %lu\n", nr_insns);
 443
 444         return 0;
 445
 446 err:
 447         free(insn);
 448         return ret;
 449 }
 450
 451 /*
 452  * Read the pv_ops[] .data table to find the static initialized values.
 453  */
 454 static int add_pv_ops(struct objtool_file *file, const char *symname)
 455 {
 456         struct symbol *sym, *func;
 457         unsigned long off, end;
 458         struct reloc *rel;
 459         int idx;
 460
 461         sym = find_symbol_by_name(file->elf, symname);
 462         if (!sym)
 463                 return 0;
 464
 465         off = sym->offset;
 466         end = off + sym->len;
 467         for (;;) {
 468                 rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
 469                 if (!rel)
 470                         break;
 471
 472                 func = rel->sym;
 473                 if (func->type == STT_SECTION)
 474                         func = find_symbol_by_offset(rel->sym->sec, rel->addend);
 475
 476                 idx = (rel->offset - sym->offset) / sizeof(unsigned long);
 477
 478                 objtool_pv_add(file, idx, func);
 479
 480                 off = rel->offset + 1;
 481                 if (off > end)
 482                         break;
 483         }
 484
 485         return 0;
 486 }
 487
 488 /*
 489  * Allocate and initialize file->pv_ops[].
 490  */
 491 static int init_pv_ops(struct objtool_file *file)
 492 {
 493         static const char *pv_ops_tables[] = {
 494                 "pv_ops",
 495                 "xen_cpu_ops",
 496                 "xen_irq_ops",
 497                 "xen_mmu_ops",
 498                 NULL,
 499         };
 500         const char *pv_ops;
 501         struct symbol *sym;
 502         int idx, nr;
 503
 504         if (!opts.noinstr)
 505                 return 0;
 506
 507         file->pv_ops = NULL;
 508
 509         sym = find_symbol_by_name(file->elf, "pv_ops");
 510         if (!sym)
 511                 return 0;
 512
 513         nr = sym->len / sizeof(unsigned long);
 514         file->pv_ops = calloc(sizeof(struct pv_state), nr);
 515         if (!file->pv_ops)
 516                 return -1;
 517
 518         for (idx = 0; idx < nr; idx++)
 519                 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
 520
 521         for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
 522                 add_pv_ops(file, pv_ops);
 523
 524         return 0;
 525 }
 526
 527 static struct instruction *find_last_insn(struct objtool_file *file,
 528                                           struct section *sec)
 529 {
 530         struct instruction *insn = NULL;
 531         unsigned int offset;
 532         unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
 533
 534         for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
 535                 insn = find_insn(file, sec, offset);
 536
 537         return insn;
 538 }
 539
 540 /*
 541  * Mark "ud2" instructions and manually annotated dead ends.
 542  */
 543 static int add_dead_ends(struct objtool_file *file)
 544 {
 545         struct section *sec;
 546         struct reloc *reloc;
 547         struct instruction *insn;
 548
 549         /*
 550          * Check for manually annotated dead ends.
 551          */
 552         sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
 553         if (!sec)
 554                 goto reachable;
 555
 556         list_for_each_entry(reloc, &sec->reloc_list, list) {
 557                 if (reloc->sym->type != STT_SECTION) {
 558                         WARN("unexpected relocation symbol type in %s", sec->name);
 559                         return -1;
 560                 }
 561                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 562                 if (insn)
 563                         insn = list_prev_entry(insn, list);
 564                 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 565                         insn = find_last_insn(file, reloc->sym->sec);
 566                         if (!insn) {
 567                                 WARN("can't find unreachable insn at %s+0x%" PRIx64,
 568                                      reloc->sym->sec->name, reloc->addend);
 569                                 return -1;
 570                         }
 571                 } else {
 572                         WARN("can't find unreachable insn at %s+0x%" PRIx64,
 573                              reloc->sym->sec->name, reloc->addend);
 574                         return -1;
 575                 }
 576
 577                 insn->dead_end = true;
 578         }
 579
 580 reachable:
 581         /*
 582          * These manually annotated reachable checks are needed for GCC 4.4,
 583          * where the Linux unreachable() macro isn't supported.  In that case
 584          * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
 585          * not a dead end.
 586          */
 587         sec = find_section_by_name(file->elf, ".rela.discard.reachable");
 588         if (!sec)
 589                 return 0;
 590
 591         list_for_each_entry(reloc, &sec->reloc_list, list) {
 592                 if (reloc->sym->type != STT_SECTION) {
 593                         WARN("unexpected relocation symbol type in %s", sec->name);
 594                         return -1;
 595                 }
 596                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 597                 if (insn)
 598                         insn = list_prev_entry(insn, list);
 599                 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 600                         insn = find_last_insn(file, reloc->sym->sec);
 601                         if (!insn) {
 602                                 WARN("can't find reachable insn at %s+0x%" PRIx64,
 603                                      reloc->sym->sec->name, reloc->addend);
 604                                 return -1;
 605                         }
 606                 } else {
 607                         WARN("can't find reachable insn at %s+0x%" PRIx64,
 608                              reloc->sym->sec->name, reloc->addend);
 609                         return -1;
 610                 }
 611
 612                 insn->dead_end = false;
 613         }
 614
 615         return 0;
 616 }
 617
 618 static int create_static_call_sections(struct objtool_file *file)
 619 {
 620         struct section *sec;
 621         struct static_call_site *site;
 622         struct instruction *insn;
 623         struct symbol *key_sym;
 624         char *key_name, *tmp;
 625         int idx;
 626
 627         sec = find_section_by_name(file->elf, ".static_call_sites");
 628         if (sec) {
 629                 INIT_LIST_HEAD(&file->static_call_list);
 630                 WARN("file already has .static_call_sites section, skipping");
 631                 return 0;
 632         }
 633
 634         if (list_empty(&file->static_call_list))
 635                 return 0;
 636
 637         idx = 0;
 638         list_for_each_entry(insn, &file->static_call_list, call_node)
 639                 idx++;
 640
 641         sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
 642                                  sizeof(struct static_call_site), idx);
 643         if (!sec)
 644                 return -1;
 645
 646         idx = 0;
 647         list_for_each_entry(insn, &file->static_call_list, call_node) {
 648
 649                 site = (struct static_call_site *)sec->data->d_buf + idx;
 650                 memset(site, 0, sizeof(struct static_call_site));
 651
 652                 /* populate reloc for 'addr' */
 653                 if (elf_add_reloc_to_insn(file->elf, sec,
 654                                           idx * sizeof(struct static_call_site),
 655                                           R_X86_64_PC32,
 656                                           insn->sec, insn->offset))
 657                         return -1;
 658
 659                 /* find key symbol */
 660                 key_name = strdup(insn->call_dest->name);
 661                 if (!key_name) {
 662                         perror("strdup");
 663                         return -1;
 664                 }
 665                 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
 666                             STATIC_CALL_TRAMP_PREFIX_LEN)) {
 667                         WARN("static_call: trampoline name malformed: %s", key_name);
 668                         return -1;
 669                 }
 670                 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
 671                 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
 672
 673                 key_sym = find_symbol_by_name(file->elf, tmp);
 674                 if (!key_sym) {
 675                         if (!opts.module) {
 676                                 WARN("static_call: can't find static_call_key symbol: %s", tmp);
 677                                 return -1;
 678                         }
 679
 680                         /*
 681                          * For modules(), the key might not be exported, which
 682                          * means the module can make static calls but isn't
 683                          * allowed to change them.
 684                          *
 685                          * In that case we temporarily set the key to be the
 686                          * trampoline address.  This is fixed up in
 687                          * static_call_add_module().
 688                          */
 689                         key_sym = insn->call_dest;
 690                 }
 691                 free(key_name);
 692
 693                 /* populate reloc for 'key' */
 694                 if (elf_add_reloc(file->elf, sec,
 695                                   idx * sizeof(struct static_call_site) + 4,
 696                                   R_X86_64_PC32, key_sym,
 697                                   is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
 698                         return -1;
 699
 700                 idx++;
 701         }
 702
 703         return 0;
 704 }
 705
 706 static int create_retpoline_sites_sections(struct objtool_file *file)
 707 {
 708         struct instruction *insn;
 709         struct section *sec;
 710         int idx;
 711
 712         sec = find_section_by_name(file->elf, ".retpoline_sites");
 713         if (sec) {
 714                 WARN("file already has .retpoline_sites, skipping");
 715                 return 0;
 716         }
 717
 718         idx = 0;
 719         list_for_each_entry(insn, &file->retpoline_call_list, call_node)
 720                 idx++;
 721
 722         if (!idx)
 723                 return 0;
 724
 725         sec = elf_create_section(file->elf, ".retpoline_sites", 0,
 726                                  sizeof(int), idx);
 727         if (!sec) {
 728                 WARN("elf_create_section: .retpoline_sites");
 729                 return -1;
 730         }
 731
 732         idx = 0;
 733         list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
 734
 735                 int *site = (int *)sec->data->d_buf + idx;
 736                 *site = 0;
 737
 738                 if (elf_add_reloc_to_insn(file->elf, sec,
 739                                           idx * sizeof(int),
 740                                           R_X86_64_PC32,
 741                                           insn->sec, insn->offset)) {
 742                         WARN("elf_add_reloc_to_insn: .retpoline_sites");
 743                         return -1;
 744                 }
 745
 746                 idx++;
 747         }
 748
 749         return 0;
 750 }
 751
 752 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
 753 {
 754         struct instruction *insn;
 755         struct section *sec;
 756         int idx;
 757
 758         sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
 759         if (sec) {
 760                 WARN("file already has .ibt_endbr_seal, skipping");
 761                 return 0;
 762         }
 763
 764         idx = 0;
 765         list_for_each_entry(insn, &file->endbr_list, call_node)
 766                 idx++;
 767
 768         if (opts.stats) {
 769                 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
 770                 printf("ibt: ENDBR inside functions:  %d\n", file->nr_endbr_int);
 771                 printf("ibt: superfluous ENDBR:       %d\n", idx);
 772         }
 773
 774         if (!idx)
 775                 return 0;
 776
 777         sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0,
 778                                  sizeof(int), idx);
 779         if (!sec) {
 780                 WARN("elf_create_section: .ibt_endbr_seal");
 781                 return -1;
 782         }
 783
 784         idx = 0;
 785         list_for_each_entry(insn, &file->endbr_list, call_node) {
 786
 787                 int *site = (int *)sec->data->d_buf + idx;
 788                 *site = 0;
 789
 790                 if (elf_add_reloc_to_insn(file->elf, sec,
 791                                           idx * sizeof(int),
 792                                           R_X86_64_PC32,
 793                                           insn->sec, insn->offset)) {
 794                         WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
 795                         return -1;
 796                 }
 797
 798                 idx++;
 799         }
 800
 801         return 0;
 802 }
 803
 804 static int create_mcount_loc_sections(struct objtool_file *file)
 805 {
 806         struct section *sec;
 807         unsigned long *loc;
 808         struct instruction *insn;
 809         int idx;
 810
 811         sec = find_section_by_name(file->elf, "__mcount_loc");
 812         if (sec) {
 813                 INIT_LIST_HEAD(&file->mcount_loc_list);
 814                 WARN("file already has __mcount_loc section, skipping");
 815                 return 0;
 816         }
 817
 818         if (list_empty(&file->mcount_loc_list))
 819                 return 0;
 820
 821         idx = 0;
 822         list_for_each_entry(insn, &file->mcount_loc_list, call_node)
 823                 idx++;
 824
 825         sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
 826         if (!sec)
 827                 return -1;
 828
 829         idx = 0;
 830         list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
 831
 832                 loc = (unsigned long *)sec->data->d_buf + idx;
 833                 memset(loc, 0, sizeof(unsigned long));
 834
 835                 if (elf_add_reloc_to_insn(file->elf, sec,
 836                                           idx * sizeof(unsigned long),
 837                                           R_X86_64_64,
 838                                           insn->sec, insn->offset))
 839                         return -1;
 840
 841                 idx++;
 842         }
 843
 844         return 0;
 845 }
 846
 847 /*
 848  * Warnings shouldn't be reported for ignored functions.
 849  */
 850 static void add_ignores(struct objtool_file *file)
 851 {
 852         struct instruction *insn;
 853         struct section *sec;
 854         struct symbol *func;
 855         struct reloc *reloc;
 856
 857         sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
 858         if (!sec)
 859                 return;
 860
 861         list_for_each_entry(reloc, &sec->reloc_list, list) {
 862                 switch (reloc->sym->type) {
 863                 case STT_FUNC:
 864                         func = reloc->sym;
 865                         break;
 866
 867                 case STT_SECTION:
 868                         func = find_func_by_offset(reloc->sym->sec, reloc->addend);
 869                         if (!func)
 870                                 continue;
 871                         break;
 872
 873                 default:
 874                         WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
 875                         continue;
 876                 }
 877
 878                 func_for_each_insn(file, func, insn)
 879                         insn->ignore = true;
 880         }
 881 }
 882
 883 /*
 884  * This is a whitelist of functions that is allowed to be called with AC set.
 885  * The list is meant to be minimal and only contains compiler instrumentation
 886  * ABI and a few functions used to implement *_{to,from}_user() functions.
 887  *
 888  * These functions must not directly change AC, but may PUSHF/POPF.
 889  */
 890 static const char *uaccess_safe_builtin[] = {
 891         /* KASAN */
 892         "kasan_report",
 893         "kasan_check_range",
 894         /* KASAN out-of-line */
 895         "__asan_loadN_noabort",
 896         "__asan_load1_noabort",
 897         "__asan_load2_noabort",
 898         "__asan_load4_noabort",
 899         "__asan_load8_noabort",
 900         "__asan_load16_noabort",
 901         "__asan_storeN_noabort",
 902         "__asan_store1_noabort",
 903         "__asan_store2_noabort",
 904         "__asan_store4_noabort",
 905         "__asan_store8_noabort",
 906         "__asan_store16_noabort",
 907         "__kasan_check_read",
 908         "__kasan_check_write",
 909         /* KASAN in-line */
 910         "__asan_report_load_n_noabort",
 911         "__asan_report_load1_noabort",
 912         "__asan_report_load2_noabort",
 913         "__asan_report_load4_noabort",
 914         "__asan_report_load8_noabort",
 915         "__asan_report_load16_noabort",
 916         "__asan_report_store_n_noabort",
 917         "__asan_report_store1_noabort",
 918         "__asan_report_store2_noabort",
 919         "__asan_report_store4_noabort",
 920         "__asan_report_store8_noabort",
 921         "__asan_report_store16_noabort",
 922         /* KCSAN */
 923         "__kcsan_check_access",
 924         "__kcsan_mb",
 925         "__kcsan_wmb",
 926         "__kcsan_rmb",
 927         "__kcsan_release",
 928         "kcsan_found_watchpoint",
 929         "kcsan_setup_watchpoint",
 930         "kcsan_check_scoped_accesses",
 931         "kcsan_disable_current",
 932         "kcsan_enable_current_nowarn",
 933         /* KCSAN/TSAN */
 934         "__tsan_func_entry",
 935         "__tsan_func_exit",
 936         "__tsan_read_range",
 937         "__tsan_write_range",
 938         "__tsan_read1",
 939         "__tsan_read2",
 940         "__tsan_read4",
 941         "__tsan_read8",
 942         "__tsan_read16",
 943         "__tsan_write1",
 944         "__tsan_write2",
 945         "__tsan_write4",
 946         "__tsan_write8",
 947         "__tsan_write16",
 948         "__tsan_read_write1",
 949         "__tsan_read_write2",
 950         "__tsan_read_write4",
 951         "__tsan_read_write8",
 952         "__tsan_read_write16",
 953         "__tsan_atomic8_load",
 954         "__tsan_atomic16_load",
 955         "__tsan_atomic32_load",
 956         "__tsan_atomic64_load",
 957         "__tsan_atomic8_store",
 958         "__tsan_atomic16_store",
 959         "__tsan_atomic32_store",
 960         "__tsan_atomic64_store",
 961         "__tsan_atomic8_exchange",
 962         "__tsan_atomic16_exchange",
 963         "__tsan_atomic32_exchange",
 964         "__tsan_atomic64_exchange",
 965         "__tsan_atomic8_fetch_add",
 966         "__tsan_atomic16_fetch_add",
 967         "__tsan_atomic32_fetch_add",
 968         "__tsan_atomic64_fetch_add",
 969         "__tsan_atomic8_fetch_sub",
 970         "__tsan_atomic16_fetch_sub",
 971         "__tsan_atomic32_fetch_sub",
 972         "__tsan_atomic64_fetch_sub",
 973         "__tsan_atomic8_fetch_and",
 974         "__tsan_atomic16_fetch_and",
 975         "__tsan_atomic32_fetch_and",
 976         "__tsan_atomic64_fetch_and",
 977         "__tsan_atomic8_fetch_or",
 978         "__tsan_atomic16_fetch_or",
 979         "__tsan_atomic32_fetch_or",
 980         "__tsan_atomic64_fetch_or",
 981         "__tsan_atomic8_fetch_xor",
 982         "__tsan_atomic16_fetch_xor",
 983         "__tsan_atomic32_fetch_xor",
 984         "__tsan_atomic64_fetch_xor",
 985         "__tsan_atomic8_fetch_nand",
 986         "__tsan_atomic16_fetch_nand",
 987         "__tsan_atomic32_fetch_nand",
 988         "__tsan_atomic64_fetch_nand",
 989         "__tsan_atomic8_compare_exchange_strong",
 990         "__tsan_atomic16_compare_exchange_strong",
 991         "__tsan_atomic32_compare_exchange_strong",
 992         "__tsan_atomic64_compare_exchange_strong",
 993         "__tsan_atomic8_compare_exchange_weak",
 994         "__tsan_atomic16_compare_exchange_weak",
 995         "__tsan_atomic32_compare_exchange_weak",
 996         "__tsan_atomic64_compare_exchange_weak",
 997         "__tsan_atomic8_compare_exchange_val",
 998         "__tsan_atomic16_compare_exchange_val",
 999         "__tsan_atomic32_compare_exchange_val",
1000         "__tsan_atomic64_compare_exchange_val",
1001         "__tsan_atomic_thread_fence",
1002         "__tsan_atomic_signal_fence",
1003         /* KCOV */
1004         "write_comp_data",
1005         "check_kcov_mode",
1006         "__sanitizer_cov_trace_pc",
1007         "__sanitizer_cov_trace_const_cmp1",
1008         "__sanitizer_cov_trace_const_cmp2",
1009         "__sanitizer_cov_trace_const_cmp4",
1010         "__sanitizer_cov_trace_const_cmp8",
1011         "__sanitizer_cov_trace_cmp1",
1012         "__sanitizer_cov_trace_cmp2",
1013         "__sanitizer_cov_trace_cmp4",
1014         "__sanitizer_cov_trace_cmp8",
1015         "__sanitizer_cov_trace_switch",
1016         /* UBSAN */
1017         "ubsan_type_mismatch_common",
1018         "__ubsan_handle_type_mismatch",
1019         "__ubsan_handle_type_mismatch_v1",
1020         "__ubsan_handle_shift_out_of_bounds",
1021         /* misc */
1022         "csum_partial_copy_generic",
1023         "copy_mc_fragile",
1024         "copy_mc_fragile_handle_tail",
1025         "copy_mc_enhanced_fast_string",
1026         "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1027         NULL
1028 };
1029
1030 static void add_uaccess_safe(struct objtool_file *file)
1031 {
1032         struct symbol *func;
1033         const char **name;
1034
1035         if (!opts.uaccess)
1036                 return;
1037
1038         for (name = uaccess_safe_builtin; *name; name++) {
1039                 func = find_symbol_by_name(file->elf, *name);
1040                 if (!func)
1041                         continue;
1042
1043                 func->uaccess_safe = true;
1044         }
1045 }
1046
1047 /*
1048  * FIXME: For now, just ignore any alternatives which add retpolines.  This is
1049  * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1050  * But it at least allows objtool to understand the control flow *around* the
1051  * retpoline.
1052  */
1053 static int add_ignore_alternatives(struct objtool_file *file)
1054 {
1055         struct section *sec;
1056         struct reloc *reloc;
1057         struct instruction *insn;
1058
1059         sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1060         if (!sec)
1061                 return 0;
1062
1063         list_for_each_entry(reloc, &sec->reloc_list, list) {
1064                 if (reloc->sym->type != STT_SECTION) {
1065                         WARN("unexpected relocation symbol type in %s", sec->name);
1066                         return -1;
1067                 }
1068
1069                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1070                 if (!insn) {
1071                         WARN("bad .discard.ignore_alts entry");
1072                         return -1;
1073                 }
1074
1075                 insn->ignore_alts = true;
1076         }
1077
1078         return 0;
1079 }
1080
1081 __weak bool arch_is_retpoline(struct symbol *sym)
1082 {
1083         return false;
1084 }
1085
1086 #define NEGATIVE_RELOC  ((void *)-1L)
1087
1088 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1089 {
1090         if (insn->reloc == NEGATIVE_RELOC)
1091                 return NULL;
1092
1093         if (!insn->reloc) {
1094                 if (!file)
1095                         return NULL;
1096
1097                 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1098                                                        insn->offset, insn->len);
1099                 if (!insn->reloc) {
1100                         insn->reloc = NEGATIVE_RELOC;
1101                         return NULL;
1102                 }
1103         }
1104
1105         return insn->reloc;
1106 }
1107
1108 static void remove_insn_ops(struct instruction *insn)
1109 {
1110         struct stack_op *op, *tmp;
1111
1112         list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
1113                 list_del(&op->list);
1114                 free(op);
1115         }
1116 }
1117
1118 static void annotate_call_site(struct objtool_file *file,
1119                                struct instruction *insn, bool sibling)
1120 {
1121         struct reloc *reloc = insn_reloc(file, insn);
1122         struct symbol *sym = insn->call_dest;
1123
1124         if (!sym)
1125                 sym = reloc->sym;
1126
1127         /*
1128          * Alternative replacement code is just template code which is
1129          * sometimes copied to the original instruction. For now, don't
1130          * annotate it. (In the future we might consider annotating the
1131          * original instruction if/when it ever makes sense to do so.)
1132          */
1133         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1134                 return;
1135
1136         if (sym->static_call_tramp) {
1137                 list_add_tail(&insn->call_node, &file->static_call_list);
1138                 return;
1139         }
1140
1141         if (sym->retpoline_thunk) {
1142                 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1143                 return;
1144         }
1145
1146         /*
1147          * Many compilers cannot disable KCOV or sanitizer calls with a function
1148          * attribute so they need a little help, NOP out any such calls from
1149          * noinstr text.
1150          */
1151         if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1152                 if (reloc) {
1153                         reloc->type = R_NONE;
1154                         elf_write_reloc(file->elf, reloc);
1155                 }
1156
1157                 elf_write_insn(file->elf, insn->sec,
1158                                insn->offset, insn->len,
1159                                sibling ? arch_ret_insn(insn->len)
1160                                        : arch_nop_insn(insn->len));
1161
1162                 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1163
1164                 if (sibling) {
1165                         /*
1166                          * We've replaced the tail-call JMP insn by two new
1167                          * insn: RET; INT3, except we only have a single struct
1168                          * insn here. Mark it retpoline_safe to avoid the SLS
1169                          * warning, instead of adding another insn.
1170                          */
1171                         insn->retpoline_safe = true;
1172                 }
1173
1174                 return;
1175         }
1176
1177         if (opts.mcount && sym->fentry) {
1178                 if (sibling)
1179                         WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
1180
1181                 if (reloc) {
1182                         reloc->type = R_NONE;
1183                         elf_write_reloc(file->elf, reloc);
1184                 }
1185
1186                 elf_write_insn(file->elf, insn->sec,
1187                                insn->offset, insn->len,
1188                                arch_nop_insn(insn->len));
1189
1190                 insn->type = INSN_NOP;
1191
1192                 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1193                 return;
1194         }
1195
1196         if (!sibling && dead_end_function(file, sym))
1197                 insn->dead_end = true;
1198 }
1199
1200 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1201                           struct symbol *dest, bool sibling)
1202 {
1203         insn->call_dest = dest;
1204         if (!dest)
1205                 return;
1206
1207         /*
1208          * Whatever stack impact regular CALLs have, should be undone
1209          * by the RETURN of the called function.
1210          *
1211          * Annotated intra-function calls retain the stack_ops but
1212          * are converted to JUMP, see read_intra_function_calls().
1213          */
1214         remove_insn_ops(insn);
1215
1216         annotate_call_site(file, insn, sibling);
1217 }
1218
1219 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1220 {
1221         /*
1222          * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1223          * so convert them accordingly.
1224          */
1225         switch (insn->type) {
1226         case INSN_CALL:
1227                 insn->type = INSN_CALL_DYNAMIC;
1228                 break;
1229         case INSN_JUMP_UNCONDITIONAL:
1230                 insn->type = INSN_JUMP_DYNAMIC;
1231                 break;
1232         case INSN_JUMP_CONDITIONAL:
1233                 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1234                 break;
1235         default:
1236                 return;
1237         }
1238
1239         insn->retpoline_safe = true;
1240
1241         /*
1242          * Whatever stack impact regular CALLs have, should be undone
1243          * by the RETURN of the called function.
1244          *
1245          * Annotated intra-function calls retain the stack_ops but
1246          * are converted to JUMP, see read_intra_function_calls().
1247          */
1248         remove_insn_ops(insn);
1249
1250         annotate_call_site(file, insn, false);
1251 }
1252
1253 static bool same_function(struct instruction *insn1, struct instruction *insn2)
1254 {
1255         return insn1->func->pfunc == insn2->func->pfunc;
1256 }
1257
1258 static bool is_first_func_insn(struct objtool_file *file, struct instruction *insn)
1259 {
1260         if (insn->offset == insn->func->offset)
1261                 return true;
1262
1263         if (opts.ibt) {
1264                 struct instruction *prev = prev_insn_same_sym(file, insn);
1265
1266                 if (prev && prev->type == INSN_ENDBR &&
1267                     insn->offset == insn->func->offset + prev->len)
1268                         return true;
1269         }
1270
1271         return false;
1272 }
1273
1274 /*
1275  * Find the destination instructions for all jumps.
1276  */
1277 static int add_jump_destinations(struct objtool_file *file)
1278 {
1279         struct instruction *insn, *jump_dest;
1280         struct reloc *reloc;
1281         struct section *dest_sec;
1282         unsigned long dest_off;
1283
1284         for_each_insn(file, insn) {
1285                 if (insn->jump_dest) {
1286                         /*
1287                          * handle_group_alt() may have previously set
1288                          * 'jump_dest' for some alternatives.
1289                          */
1290                         continue;
1291                 }
1292                 if (!is_static_jump(insn))
1293                         continue;
1294
1295                 reloc = insn_reloc(file, insn);
1296                 if (!reloc) {
1297                         dest_sec = insn->sec;
1298                         dest_off = arch_jump_destination(insn);
1299                 } else if (reloc->sym->type == STT_SECTION) {
1300                         dest_sec = reloc->sym->sec;
1301                         dest_off = arch_dest_reloc_offset(reloc->addend);
1302                 } else if (reloc->sym->retpoline_thunk) {
1303                         add_retpoline_call(file, insn);
1304                         continue;
1305                 } else if (insn->func) {
1306                         /*
1307                          * External sibling call or internal sibling call with
1308                          * STT_FUNC reloc.
1309                          */
1310                         add_call_dest(file, insn, reloc->sym, true);
1311                         continue;
1312                 } else if (reloc->sym->sec->idx) {
1313                         dest_sec = reloc->sym->sec;
1314                         dest_off = reloc->sym->sym.st_value +
1315                                    arch_dest_reloc_offset(reloc->addend);
1316                 } else {
1317                         /* non-func asm code jumping to another file */
1318                         continue;
1319                 }
1320
1321                 jump_dest = find_insn(file, dest_sec, dest_off);
1322                 if (!jump_dest) {
1323                         WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
1324                                   insn->sec, insn->offset, dest_sec->name,
1325                                   dest_off);
1326                         return -1;
1327                 }
1328
1329                 /*
1330                  * Cross-function jump.
1331                  */
1332                 if (insn->func && jump_dest->func &&
1333                     insn->func != jump_dest->func) {
1334
1335                         /*
1336                          * For GCC 8+, create parent/child links for any cold
1337                          * subfunctions.  This is _mostly_ redundant with a
1338                          * similar initialization in read_symbols().
1339                          *
1340                          * If a function has aliases, we want the *first* such
1341                          * function in the symbol table to be the subfunction's
1342                          * parent.  In that case we overwrite the
1343                          * initialization done in read_symbols().
1344                          *
1345                          * However this code can't completely replace the
1346                          * read_symbols() code because this doesn't detect the
1347                          * case where the parent function's only reference to a
1348                          * subfunction is through a jump table.
1349                          */
1350                         if (!strstr(insn->func->name, ".cold") &&
1351                             strstr(jump_dest->func->name, ".cold")) {
1352                                 insn->func->cfunc = jump_dest->func;
1353                                 jump_dest->func->pfunc = insn->func;
1354
1355                         } else if (!same_function(insn, jump_dest) &&
1356                                    is_first_func_insn(file, jump_dest)) {
1357                                 /*
1358                                  * Internal sibling call without reloc or with
1359                                  * STT_SECTION reloc.
1360                                  */
1361                                 add_call_dest(file, insn, jump_dest->func, true);
1362                                 continue;
1363                         }
1364                 }
1365
1366                 insn->jump_dest = jump_dest;
1367         }
1368
1369         return 0;
1370 }
1371
1372 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1373 {
1374         struct symbol *call_dest;
1375
1376         call_dest = find_func_by_offset(sec, offset);
1377         if (!call_dest)
1378                 call_dest = find_symbol_by_offset(sec, offset);
1379
1380         return call_dest;
1381 }
1382
1383 /*
1384  * Find the destination instructions for all calls.
1385  */
1386 static int add_call_destinations(struct objtool_file *file)
1387 {
1388         struct instruction *insn;
1389         unsigned long dest_off;
1390         struct symbol *dest;
1391         struct reloc *reloc;
1392
1393         for_each_insn(file, insn) {
1394                 if (insn->type != INSN_CALL)
1395                         continue;
1396
1397                 reloc = insn_reloc(file, insn);
1398                 if (!reloc) {
1399                         dest_off = arch_jump_destination(insn);
1400                         dest = find_call_destination(insn->sec, dest_off);
1401
1402                         add_call_dest(file, insn, dest, false);
1403
1404                         if (insn->ignore)
1405                                 continue;
1406
1407                         if (!insn->call_dest) {
1408                                 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
1409                                 return -1;
1410                         }
1411
1412                         if (insn->func && insn->call_dest->type != STT_FUNC) {
1413                                 WARN_FUNC("unsupported call to non-function",
1414                                           insn->sec, insn->offset);
1415                                 return -1;
1416                         }
1417
1418                 } else if (reloc->sym->type == STT_SECTION) {
1419                         dest_off = arch_dest_reloc_offset(reloc->addend);
1420                         dest = find_call_destination(reloc->sym->sec, dest_off);
1421                         if (!dest) {
1422                                 WARN_FUNC("can't find call dest symbol at %s+0x%lx",
1423                                           insn->sec, insn->offset,
1424                                           reloc->sym->sec->name,
1425                                           dest_off);
1426                                 return -1;
1427                         }
1428
1429                         add_call_dest(file, insn, dest, false);
1430
1431                 } else if (reloc->sym->retpoline_thunk) {
1432                         add_retpoline_call(file, insn);
1433
1434                 } else
1435                         add_call_dest(file, insn, reloc->sym, false);
1436         }
1437
1438         return 0;
1439 }
1440
1441 /*
1442  * The .alternatives section requires some extra special care over and above
1443  * other special sections because alternatives are patched in place.
1444  */
1445 static int handle_group_alt(struct objtool_file *file,
1446                             struct special_alt *special_alt,
1447                             struct instruction *orig_insn,
1448                             struct instruction **new_insn)
1449 {
1450         struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
1451         struct alt_group *orig_alt_group, *new_alt_group;
1452         unsigned long dest_off;
1453
1454
1455         orig_alt_group = malloc(sizeof(*orig_alt_group));
1456         if (!orig_alt_group) {
1457                 WARN("malloc failed");
1458                 return -1;
1459         }
1460         orig_alt_group->cfi = calloc(special_alt->orig_len,
1461                                      sizeof(struct cfi_state *));
1462         if (!orig_alt_group->cfi) {
1463                 WARN("calloc failed");
1464                 return -1;
1465         }
1466
1467         last_orig_insn = NULL;
1468         insn = orig_insn;
1469         sec_for_each_insn_from(file, insn) {
1470                 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1471                         break;
1472
1473                 insn->alt_group = orig_alt_group;
1474                 last_orig_insn = insn;
1475         }
1476         orig_alt_group->orig_group = NULL;
1477         orig_alt_group->first_insn = orig_insn;
1478         orig_alt_group->last_insn = last_orig_insn;
1479
1480
1481         new_alt_group = malloc(sizeof(*new_alt_group));
1482         if (!new_alt_group) {
1483                 WARN("malloc failed");
1484                 return -1;
1485         }
1486
1487         if (special_alt->new_len < special_alt->orig_len) {
1488                 /*
1489                  * Insert a fake nop at the end to make the replacement
1490                  * alt_group the same size as the original.  This is needed to
1491                  * allow propagate_alt_cfi() to do its magic.  When the last
1492                  * instruction affects the stack, the instruction after it (the
1493                  * nop) will propagate the new state to the shared CFI array.
1494                  */
1495                 nop = malloc(sizeof(*nop));
1496                 if (!nop) {
1497                         WARN("malloc failed");
1498                         return -1;
1499                 }
1500                 memset(nop, 0, sizeof(*nop));
1501                 INIT_LIST_HEAD(&nop->alts);
1502                 INIT_LIST_HEAD(&nop->stack_ops);
1503
1504                 nop->sec = special_alt->new_sec;
1505                 nop->offset = special_alt->new_off + special_alt->new_len;
1506                 nop->len = special_alt->orig_len - special_alt->new_len;
1507                 nop->type = INSN_NOP;
1508                 nop->func = orig_insn->func;
1509                 nop->alt_group = new_alt_group;
1510                 nop->ignore = orig_insn->ignore_alts;
1511         }
1512
1513         if (!special_alt->new_len) {
1514                 *new_insn = nop;
1515                 goto end;
1516         }
1517
1518         insn = *new_insn;
1519         sec_for_each_insn_from(file, insn) {
1520                 struct reloc *alt_reloc;
1521
1522                 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1523                         break;
1524
1525                 last_new_insn = insn;
1526
1527                 insn->ignore = orig_insn->ignore_alts;
1528                 insn->func = orig_insn->func;
1529                 insn->alt_group = new_alt_group;
1530
1531                 /*
1532                  * Since alternative replacement code is copy/pasted by the
1533                  * kernel after applying relocations, generally such code can't
1534                  * have relative-address relocation references to outside the
1535                  * .altinstr_replacement section, unless the arch's
1536                  * alternatives code can adjust the relative offsets
1537                  * accordingly.
1538                  */
1539                 alt_reloc = insn_reloc(file, insn);
1540                 if (alt_reloc &&
1541                     !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1542
1543                         WARN_FUNC("unsupported relocation in alternatives section",
1544                                   insn->sec, insn->offset);
1545                         return -1;
1546                 }
1547
1548                 if (!is_static_jump(insn))
1549                         continue;
1550
1551                 if (!insn->immediate)
1552                         continue;
1553
1554                 dest_off = arch_jump_destination(insn);
1555                 if (dest_off == special_alt->new_off + special_alt->new_len) {
1556                         insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
1557                         if (!insn->jump_dest) {
1558                                 WARN_FUNC("can't find alternative jump destination",
1559                                           insn->sec, insn->offset);
1560                                 return -1;
1561                         }
1562                 }
1563         }
1564
1565         if (!last_new_insn) {
1566                 WARN_FUNC("can't find last new alternative instruction",
1567                           special_alt->new_sec, special_alt->new_off);
1568                 return -1;
1569         }
1570
1571         if (nop)
1572                 list_add(&nop->list, &last_new_insn->list);
1573 end:
1574         new_alt_group->orig_group = orig_alt_group;
1575         new_alt_group->first_insn = *new_insn;
1576         new_alt_group->last_insn = nop ? : last_new_insn;
1577         new_alt_group->cfi = orig_alt_group->cfi;
1578         return 0;
1579 }
1580
1581 /*
1582  * A jump table entry can either convert a nop to a jump or a jump to a nop.
1583  * If the original instruction is a jump, make the alt entry an effective nop
1584  * by just skipping the original instruction.
1585  */
1586 static int handle_jump_alt(struct objtool_file *file,
1587                            struct special_alt *special_alt,
1588                            struct instruction *orig_insn,
1589                            struct instruction **new_insn)
1590 {
1591         if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1592             orig_insn->type != INSN_NOP) {
1593
1594                 WARN_FUNC("unsupported instruction at jump label",
1595                           orig_insn->sec, orig_insn->offset);
1596                 return -1;
1597         }
1598
1599         if (opts.hack_jump_label && special_alt->key_addend & 2) {
1600                 struct reloc *reloc = insn_reloc(file, orig_insn);
1601
1602                 if (reloc) {
1603                         reloc->type = R_NONE;
1604                         elf_write_reloc(file->elf, reloc);
1605                 }
1606                 elf_write_insn(file->elf, orig_insn->sec,
1607                                orig_insn->offset, orig_insn->len,
1608                                arch_nop_insn(orig_insn->len));
1609                 orig_insn->type = INSN_NOP;
1610         }
1611
1612         if (orig_insn->type == INSN_NOP) {
1613                 if (orig_insn->len == 2)
1614                         file->jl_nop_short++;
1615                 else
1616                         file->jl_nop_long++;
1617
1618                 return 0;
1619         }
1620
1621         if (orig_insn->len == 2)
1622                 file->jl_short++;
1623         else
1624                 file->jl_long++;
1625
1626         *new_insn = list_next_entry(orig_insn, list);
1627         return 0;
1628 }
1629
1630 /*
1631  * Read all the special sections which have alternate instructions which can be
1632  * patched in or redirected to at runtime.  Each instruction having alternate
1633  * instruction(s) has them added to its insn->alts list, which will be
1634  * traversed in validate_branch().
1635  */
1636 static int add_special_section_alts(struct objtool_file *file)
1637 {
1638         struct list_head special_alts;
1639         struct instruction *orig_insn, *new_insn;
1640         struct special_alt *special_alt, *tmp;
1641         struct alternative *alt;
1642         int ret;
1643
1644         ret = special_get_alts(file->elf, &special_alts);
1645         if (ret)
1646                 return ret;
1647
1648         list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1649
1650                 orig_insn = find_insn(file, special_alt->orig_sec,
1651                                       special_alt->orig_off);
1652                 if (!orig_insn) {
1653                         WARN_FUNC("special: can't find orig instruction",
1654                                   special_alt->orig_sec, special_alt->orig_off);
1655                         ret = -1;
1656                         goto out;
1657                 }
1658
1659                 new_insn = NULL;
1660                 if (!special_alt->group || special_alt->new_len) {
1661                         new_insn = find_insn(file, special_alt->new_sec,
1662                                              special_alt->new_off);
1663                         if (!new_insn) {
1664                                 WARN_FUNC("special: can't find new instruction",
1665                                           special_alt->new_sec,
1666                                           special_alt->new_off);
1667                                 ret = -1;
1668                                 goto out;
1669                         }
1670                 }
1671
1672                 if (special_alt->group) {
1673                         if (!special_alt->orig_len) {
1674                                 WARN_FUNC("empty alternative entry",
1675                                           orig_insn->sec, orig_insn->offset);
1676                                 continue;
1677                         }
1678
1679                         ret = handle_group_alt(file, special_alt, orig_insn,
1680                                                &new_insn);
1681                         if (ret)
1682                                 goto out;
1683                 } else if (special_alt->jump_or_nop) {
1684                         ret = handle_jump_alt(file, special_alt, orig_insn,
1685                                               &new_insn);
1686                         if (ret)
1687                                 goto out;
1688                 }
1689
1690                 alt = malloc(sizeof(*alt));
1691                 if (!alt) {
1692                         WARN("malloc failed");
1693                         ret = -1;
1694                         goto out;
1695                 }
1696
1697                 alt->insn = new_insn;
1698                 alt->skip_orig = special_alt->skip_orig;
1699                 orig_insn->ignore_alts |= special_alt->skip_alt;
1700                 list_add_tail(&alt->list, &orig_insn->alts);
1701
1702                 list_del(&special_alt->list);
1703                 free(special_alt);
1704         }
1705
1706         if (opts.stats) {
1707                 printf("jl\\\tNOP\tJMP\n");
1708                 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1709                 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1710         }
1711
1712 out:
1713         return ret;
1714 }
1715
1716 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
1717                             struct reloc *table)
1718 {
1719         struct reloc *reloc = table;
1720         struct instruction *dest_insn;
1721         struct alternative *alt;
1722         struct symbol *pfunc = insn->func->pfunc;
1723         unsigned int prev_offset = 0;
1724
1725         /*
1726          * Each @reloc is a switch table relocation which points to the target
1727          * instruction.
1728          */
1729         list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
1730
1731                 /* Check for the end of the table: */
1732                 if (reloc != table && reloc->jump_table_start)
1733                         break;
1734
1735                 /* Make sure the table entries are consecutive: */
1736                 if (prev_offset && reloc->offset != prev_offset + 8)
1737                         break;
1738
1739                 /* Detect function pointers from contiguous objects: */
1740                 if (reloc->sym->sec == pfunc->sec &&
1741                     reloc->addend == pfunc->offset)
1742                         break;
1743
1744                 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
1745                 if (!dest_insn)
1746                         break;
1747
1748                 /* Make sure the destination is in the same function: */
1749                 if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
1750                         break;
1751
1752                 alt = malloc(sizeof(*alt));
1753                 if (!alt) {
1754                         WARN("malloc failed");
1755                         return -1;
1756                 }
1757
1758                 alt->insn = dest_insn;
1759                 list_add_tail(&alt->list, &insn->alts);
1760                 prev_offset = reloc->offset;
1761         }
1762
1763         if (!prev_offset) {
1764                 WARN_FUNC("can't find switch jump table",
1765                           insn->sec, insn->offset);
1766                 return -1;
1767         }
1768
1769         return 0;
1770 }
1771
1772 /*
1773  * find_jump_table() - Given a dynamic jump, find the switch jump table
1774  * associated with it.
1775  */
1776 static struct reloc *find_jump_table(struct objtool_file *file,
1777                                       struct symbol *func,
1778                                       struct instruction *insn)
1779 {
1780         struct reloc *table_reloc;
1781         struct instruction *dest_insn, *orig_insn = insn;
1782
1783         /*
1784          * Backward search using the @first_jump_src links, these help avoid
1785          * much of the 'in between' code. Which avoids us getting confused by
1786          * it.
1787          */
1788         for (;
1789              insn && insn->func && insn->func->pfunc == func;
1790              insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
1791
1792                 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
1793                         break;
1794
1795                 /* allow small jumps within the range */
1796                 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
1797                     insn->jump_dest &&
1798                     (insn->jump_dest->offset <= insn->offset ||
1799                      insn->jump_dest->offset > orig_insn->offset))
1800                     break;
1801
1802                 table_reloc = arch_find_switch_table(file, insn);
1803                 if (!table_reloc)
1804                         continue;
1805                 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
1806                 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
1807                         continue;
1808
1809                 return table_reloc;
1810         }
1811
1812         return NULL;
1813 }
1814
1815 /*
1816  * First pass: Mark the head of each jump table so that in the next pass,
1817  * we know when a given jump table ends and the next one starts.
1818  */
1819 static void mark_func_jump_tables(struct objtool_file *file,
1820                                     struct symbol *func)
1821 {
1822         struct instruction *insn, *last = NULL;
1823         struct reloc *reloc;
1824
1825         func_for_each_insn(file, func, insn) {
1826                 if (!last)
1827                         last = insn;
1828
1829                 /*
1830                  * Store back-pointers for unconditional forward jumps such
1831                  * that find_jump_table() can back-track using those and
1832                  * avoid some potentially confusing code.
1833                  */
1834                 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
1835                     insn->offset > last->offset &&
1836                     insn->jump_dest->offset > insn->offset &&
1837                     !insn->jump_dest->first_jump_src) {
1838
1839                         insn->jump_dest->first_jump_src = insn;
1840                         last = insn->jump_dest;
1841                 }
1842
1843                 if (insn->type != INSN_JUMP_DYNAMIC)
1844                         continue;
1845
1846                 reloc = find_jump_table(file, func, insn);
1847                 if (reloc) {
1848                         reloc->jump_table_start = true;
1849                         insn->jump_table = reloc;
1850                 }
1851         }
1852 }
1853
1854 static int add_func_jump_tables(struct objtool_file *file,
1855                                   struct symbol *func)
1856 {
1857         struct instruction *insn;
1858         int ret;
1859
1860         func_for_each_insn(file, func, insn) {
1861                 if (!insn->jump_table)
1862                         continue;
1863
1864                 ret = add_jump_table(file, insn, insn->jump_table);
1865                 if (ret)
1866                         return ret;
1867         }
1868
1869         return 0;
1870 }
1871
1872 /*
1873  * For some switch statements, gcc generates a jump table in the .rodata
1874  * section which contains a list of addresses within the function to jump to.
1875  * This finds these jump tables and adds them to the insn->alts lists.
1876  */
1877 static int add_jump_table_alts(struct objtool_file *file)
1878 {
1879         struct section *sec;
1880         struct symbol *func;
1881         int ret;
1882
1883         if (!file->rodata)
1884                 return 0;
1885
1886         for_each_sec(file, sec) {
1887                 list_for_each_entry(func, &sec->symbol_list, list) {
1888                         if (func->type != STT_FUNC)
1889                                 continue;
1890
1891                         mark_func_jump_tables(file, func);
1892                         ret = add_func_jump_tables(file, func);
1893                         if (ret)
1894                                 return ret;
1895                 }
1896         }
1897
1898         return 0;
1899 }
1900
1901 static void set_func_state(struct cfi_state *state)
1902 {
1903         state->cfa = initial_func_cfi.cfa;
1904         memcpy(&state->regs, &initial_func_cfi.regs,
1905                CFI_NUM_REGS * sizeof(struct cfi_reg));
1906         state->stack_size = initial_func_cfi.cfa.offset;
1907 }
1908
1909 static int read_unwind_hints(struct objtool_file *file)
1910 {
1911         struct cfi_state cfi = init_cfi;
1912         struct section *sec, *relocsec;
1913         struct unwind_hint *hint;
1914         struct instruction *insn;
1915         struct reloc *reloc;
1916         int i;
1917
1918         sec = find_section_by_name(file->elf, ".discard.unwind_hints");
1919         if (!sec)
1920                 return 0;
1921
1922         relocsec = sec->reloc;
1923         if (!relocsec) {
1924                 WARN("missing .rela.discard.unwind_hints section");
1925                 return -1;
1926         }
1927
1928         if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
1929                 WARN("struct unwind_hint size mismatch");
1930                 return -1;
1931         }
1932
1933         file->hints = true;
1934
1935         for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
1936                 hint = (struct unwind_hint *)sec->data->d_buf + i;
1937
1938                 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
1939                 if (!reloc) {
1940                         WARN("can't find reloc for unwind_hints[%d]", i);
1941                         return -1;
1942                 }
1943
1944                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1945                 if (!insn) {
1946                         WARN("can't find insn for unwind_hints[%d]", i);
1947                         return -1;
1948                 }
1949
1950                 insn->hint = true;
1951
1952                 if (opts.ibt && hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
1953                         struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
1954
1955                         if (sym && sym->bind == STB_GLOBAL &&
1956                             insn->type != INSN_ENDBR && !insn->noendbr) {
1957                                 WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR",
1958                                           insn->sec, insn->offset);
1959                         }
1960                 }
1961
1962                 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
1963                         insn->cfi = &func_cfi;
1964                         continue;
1965                 }
1966
1967                 if (insn->cfi)
1968                         cfi = *(insn->cfi);
1969
1970                 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
1971                         WARN_FUNC("unsupported unwind_hint sp base reg %d",
1972                                   insn->sec, insn->offset, hint->sp_reg);
1973                         return -1;
1974                 }
1975
1976                 cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
1977                 cfi.type = hint->type;
1978                 cfi.end = hint->end;
1979
1980                 insn->cfi = cfi_hash_find_or_add(&cfi);
1981         }
1982
1983         return 0;
1984 }
1985
1986 static int read_noendbr_hints(struct objtool_file *file)
1987 {
1988         struct section *sec;
1989         struct instruction *insn;
1990         struct reloc *reloc;
1991
1992         sec = find_section_by_name(file->elf, ".rela.discard.noendbr");
1993         if (!sec)
1994                 return 0;
1995
1996         list_for_each_entry(reloc, &sec->reloc_list, list) {
1997                 insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend);
1998                 if (!insn) {
1999                         WARN("bad .discard.noendbr entry");
2000                         return -1;
2001                 }
2002
2003                 if (insn->type == INSN_ENDBR)
2004                         WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset);
2005
2006                 insn->noendbr = 1;
2007         }
2008
2009         return 0;
2010 }
2011
2012 static int read_retpoline_hints(struct objtool_file *file)
2013 {
2014         struct section *sec;
2015         struct instruction *insn;
2016         struct reloc *reloc;
2017
2018         sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2019         if (!sec)
2020                 return 0;
2021
2022         list_for_each_entry(reloc, &sec->reloc_list, list) {
2023                 if (reloc->sym->type != STT_SECTION) {
2024                         WARN("unexpected relocation symbol type in %s", sec->name);
2025                         return -1;
2026                 }
2027
2028                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2029                 if (!insn) {
2030                         WARN("bad .discard.retpoline_safe entry");
2031                         return -1;
2032                 }
2033
2034                 if (insn->type != INSN_JUMP_DYNAMIC &&
2035                     insn->type != INSN_CALL_DYNAMIC) {
2036                         WARN_FUNC("retpoline_safe hint not an indirect jump/call",
2037                                   insn->sec, insn->offset);
2038                         return -1;
2039                 }
2040
2041                 insn->retpoline_safe = true;
2042         }
2043
2044         return 0;
2045 }
2046
2047 static int read_instr_hints(struct objtool_file *file)
2048 {
2049         struct section *sec;
2050         struct instruction *insn;
2051         struct reloc *reloc;
2052
2053         sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2054         if (!sec)
2055                 return 0;
2056
2057         list_for_each_entry(reloc, &sec->reloc_list, list) {
2058                 if (reloc->sym->type != STT_SECTION) {
2059                         WARN("unexpected relocation symbol type in %s", sec->name);
2060                         return -1;
2061                 }
2062
2063                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2064                 if (!insn) {
2065                         WARN("bad .discard.instr_end entry");
2066                         return -1;
2067                 }
2068
2069                 insn->instr--;
2070         }
2071
2072         sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2073         if (!sec)
2074                 return 0;
2075
2076         list_for_each_entry(reloc, &sec->reloc_list, list) {
2077                 if (reloc->sym->type != STT_SECTION) {
2078                         WARN("unexpected relocation symbol type in %s", sec->name);
2079                         return -1;
2080                 }
2081
2082                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2083                 if (!insn) {
2084                         WARN("bad .discard.instr_begin entry");
2085                         return -1;
2086                 }
2087
2088                 insn->instr++;
2089         }
2090
2091         return 0;
2092 }
2093
2094 static int read_intra_function_calls(struct objtool_file *file)
2095 {
2096         struct instruction *insn;
2097         struct section *sec;
2098         struct reloc *reloc;
2099
2100         sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2101         if (!sec)
2102                 return 0;
2103
2104         list_for_each_entry(reloc, &sec->reloc_list, list) {
2105                 unsigned long dest_off;
2106
2107                 if (reloc->sym->type != STT_SECTION) {
2108                         WARN("unexpected relocation symbol type in %s",
2109                              sec->name);
2110                         return -1;
2111                 }
2112
2113                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2114                 if (!insn) {
2115                         WARN("bad .discard.intra_function_call entry");
2116                         return -1;
2117                 }
2118
2119                 if (insn->type != INSN_CALL) {
2120                         WARN_FUNC("intra_function_call not a direct call",
2121                                   insn->sec, insn->offset);
2122                         return -1;
2123                 }
2124
2125                 /*
2126                  * Treat intra-function CALLs as JMPs, but with a stack_op.
2127                  * See add_call_destinations(), which strips stack_ops from
2128                  * normal CALLs.
2129                  */
2130                 insn->type = INSN_JUMP_UNCONDITIONAL;
2131
2132                 dest_off = insn->offset + insn->len + insn->immediate;
2133                 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2134                 if (!insn->jump_dest) {
2135                         WARN_FUNC("can't find call dest at %s+0x%lx",
2136                                   insn->sec, insn->offset,
2137                                   insn->sec->name, dest_off);
2138                         return -1;
2139                 }
2140         }
2141
2142         return 0;
2143 }
2144
2145 /*
2146  * Return true if name matches an instrumentation function, where calls to that
2147  * function from noinstr code can safely be removed, but compilers won't do so.
2148  */
2149 static bool is_profiling_func(const char *name)
2150 {
2151         /*
2152          * Many compilers cannot disable KCOV with a function attribute.
2153          */
2154         if (!strncmp(name, "__sanitizer_cov_", 16))
2155                 return true;
2156
2157         /*
2158          * Some compilers currently do not remove __tsan_func_entry/exit nor
2159          * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2160          * the __no_sanitize_thread attribute, remove them. Once the kernel's
2161          * minimum Clang version is 14.0, this can be removed.
2162          */
2163         if (!strncmp(name, "__tsan_func_", 12) ||
2164             !strcmp(name, "__tsan_atomic_signal_fence"))
2165                 return true;
2166
2167         return false;
2168 }
2169
2170 static int classify_symbols(struct objtool_file *file)
2171 {
2172         struct section *sec;
2173         struct symbol *func;
2174
2175         for_each_sec(file, sec) {
2176                 list_for_each_entry(func, &sec->symbol_list, list) {
2177                         if (func->bind != STB_GLOBAL)
2178                                 continue;
2179
2180                         if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2181                                      strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2182                                 func->static_call_tramp = true;
2183
2184                         if (arch_is_retpoline(func))
2185                                 func->retpoline_thunk = true;
2186
2187                         if (!strcmp(func->name, "__fentry__"))
2188                                 func->fentry = true;
2189
2190                         if (is_profiling_func(func->name))
2191                                 func->profiling_func = true;
2192                 }
2193         }
2194
2195         return 0;
2196 }
2197
2198 static void mark_rodata(struct objtool_file *file)
2199 {
2200         struct section *sec;
2201         bool found = false;
2202
2203         /*
2204          * Search for the following rodata sections, each of which can
2205          * potentially contain jump tables:
2206          *
2207          * - .rodata: can contain GCC switch tables
2208          * - .rodata.<func>: same, if -fdata-sections is being used
2209          * - .rodata..c_jump_table: contains C annotated jump tables
2210          *
2211          * .rodata.str1.* sections are ignored; they don't contain jump tables.
2212          */
2213         for_each_sec(file, sec) {
2214                 if (!strncmp(sec->name, ".rodata", 7) &&
2215                     !strstr(sec->name, ".str1.")) {
2216                         sec->rodata = true;
2217                         found = true;
2218                 }
2219         }
2220
2221         file->rodata = found;
2222 }
2223
2224 static int decode_sections(struct objtool_file *file)
2225 {
2226         int ret;
2227
2228         mark_rodata(file);
2229
2230         ret = init_pv_ops(file);
2231         if (ret)
2232                 return ret;
2233
2234         ret = decode_instructions(file);
2235         if (ret)
2236                 return ret;
2237
2238         add_ignores(file);
2239         add_uaccess_safe(file);
2240
2241         ret = add_ignore_alternatives(file);
2242         if (ret)
2243                 return ret;
2244
2245         /*
2246          * Must be before read_unwind_hints() since that needs insn->noendbr.
2247          */
2248         ret = read_noendbr_hints(file);
2249         if (ret)
2250                 return ret;
2251
2252         /*
2253          * Must be before add_{jump_call}_destination.
2254          */
2255         ret = classify_symbols(file);
2256         if (ret)
2257                 return ret;
2258
2259         /*
2260          * Must be before add_jump_destinations(), which depends on 'func'
2261          * being set for alternatives, to enable proper sibling call detection.
2262          */
2263         ret = add_special_section_alts(file);
2264         if (ret)
2265                 return ret;
2266
2267         ret = add_jump_destinations(file);
2268         if (ret)
2269                 return ret;
2270
2271         /*
2272          * Must be before add_call_destination(); it changes INSN_CALL to
2273          * INSN_JUMP.
2274          */
2275         ret = read_intra_function_calls(file);
2276         if (ret)
2277                 return ret;
2278
2279         ret = add_call_destinations(file);
2280         if (ret)
2281                 return ret;
2282
2283         /*
2284          * Must be after add_call_destinations() such that it can override
2285          * dead_end_function() marks.
2286          */
2287         ret = add_dead_ends(file);
2288         if (ret)
2289                 return ret;
2290
2291         ret = add_jump_table_alts(file);
2292         if (ret)
2293                 return ret;
2294
2295         ret = read_unwind_hints(file);
2296         if (ret)
2297                 return ret;
2298
2299         ret = read_retpoline_hints(file);
2300         if (ret)
2301                 return ret;
2302
2303         ret = read_instr_hints(file);
2304         if (ret)
2305                 return ret;
2306
2307         return 0;
2308 }
2309
2310 static bool is_fentry_call(struct instruction *insn)
2311 {
2312         if (insn->type == INSN_CALL &&
2313             insn->call_dest &&
2314             insn->call_dest->fentry)
2315                 return true;
2316
2317         return false;
2318 }
2319
2320 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2321 {
2322         struct cfi_state *cfi = &state->cfi;
2323         int i;
2324
2325         if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2326                 return true;
2327
2328         if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2329                 return true;
2330
2331         if (cfi->stack_size != initial_func_cfi.cfa.offset)
2332                 return true;
2333
2334         for (i = 0; i < CFI_NUM_REGS; i++) {
2335                 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2336                     cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2337                         return true;
2338         }
2339
2340         return false;
2341 }
2342
2343 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2344                                 int expected_offset)
2345 {
2346         return reg->base == CFI_CFA &&
2347                reg->offset == expected_offset;
2348 }
2349
2350 static bool has_valid_stack_frame(struct insn_state *state)
2351 {
2352         struct cfi_state *cfi = &state->cfi;
2353
2354         if (cfi->cfa.base == CFI_BP &&
2355             check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2356             check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2357                 return true;
2358
2359         if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2360                 return true;
2361
2362         return false;
2363 }
2364
2365 static int update_cfi_state_regs(struct instruction *insn,
2366                                   struct cfi_state *cfi,
2367                                   struct stack_op *op)
2368 {
2369         struct cfi_reg *cfa = &cfi->cfa;
2370
2371         if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2372                 return 0;
2373
2374         /* push */
2375         if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2376                 cfa->offset += 8;
2377
2378         /* pop */
2379         if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2380                 cfa->offset -= 8;
2381
2382         /* add immediate to sp */
2383         if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2384             op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2385                 cfa->offset -= op->src.offset;
2386
2387         return 0;
2388 }
2389
2390 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2391 {
2392         if (arch_callee_saved_reg(reg) &&
2393             cfi->regs[reg].base == CFI_UNDEFINED) {
2394                 cfi->regs[reg].base = base;
2395                 cfi->regs[reg].offset = offset;
2396         }
2397 }
2398
2399 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2400 {
2401         cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2402         cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2403 }
2404
2405 /*
2406  * A note about DRAP stack alignment:
2407  *
2408  * GCC has the concept of a DRAP register, which is used to help keep track of
2409  * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2410  * register.  The typical DRAP pattern is:
2411  *
2412  *   4c 8d 54 24 08             lea    0x8(%rsp),%r10
2413  *   48 83 e4 c0                and    $0xffffffffffffffc0,%rsp
2414  *   41 ff 72 f8                pushq  -0x8(%r10)
2415  *   55                         push   %rbp
2416  *   48 89 e5                   mov    %rsp,%rbp
2417  *                              (more pushes)
2418  *   41 52                      push   %r10
2419  *                              ...
2420  *   41 5a                      pop    %r10
2421  *                              (more pops)
2422  *   5d                         pop    %rbp
2423  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2424  *   c3                         retq
2425  *
2426  * There are some variations in the epilogues, like:
2427  *
2428  *   5b                         pop    %rbx
2429  *   41 5a                      pop    %r10
2430  *   41 5c                      pop    %r12
2431  *   41 5d                      pop    %r13
2432  *   41 5e                      pop    %r14
2433  *   c9                         leaveq
2434  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2435  *   c3                         retq
2436  *
2437  * and:
2438  *
2439  *   4c 8b 55 e8                mov    -0x18(%rbp),%r10
2440  *   48 8b 5d e0                mov    -0x20(%rbp),%rbx
2441  *   4c 8b 65 f0                mov    -0x10(%rbp),%r12
2442  *   4c 8b 6d f8                mov    -0x8(%rbp),%r13
2443  *   c9                         leaveq
2444  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2445  *   c3                         retq
2446  *
2447  * Sometimes r13 is used as the DRAP register, in which case it's saved and
2448  * restored beforehand:
2449  *
2450  *   41 55                      push   %r13
2451  *   4c 8d 6c 24 10             lea    0x10(%rsp),%r13
2452  *   48 83 e4 f0                and    $0xfffffffffffffff0,%rsp
2453  *                              ...
2454  *   49 8d 65 f0                lea    -0x10(%r13),%rsp
2455  *   41 5d                      pop    %r13
2456  *   c3                         retq
2457  */
2458 static int update_cfi_state(struct instruction *insn,
2459                             struct instruction *next_insn,
2460                             struct cfi_state *cfi, struct stack_op *op)
2461 {
2462         struct cfi_reg *cfa = &cfi->cfa;
2463         struct cfi_reg *regs = cfi->regs;
2464
2465         /* stack operations don't make sense with an undefined CFA */
2466         if (cfa->base == CFI_UNDEFINED) {
2467                 if (insn->func) {
2468                         WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2469                         return -1;
2470                 }
2471                 return 0;
2472         }
2473
2474         if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2475             cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2476                 return update_cfi_state_regs(insn, cfi, op);
2477
2478         switch (op->dest.type) {
2479
2480         case OP_DEST_REG:
2481                 switch (op->src.type) {
2482
2483                 case OP_SRC_REG:
2484                         if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2485                             cfa->base == CFI_SP &&
2486                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2487
2488                                 /* mov %rsp, %rbp */
2489                                 cfa->base = op->dest.reg;
2490                                 cfi->bp_scratch = false;
2491                         }
2492
2493                         else if (op->src.reg == CFI_SP &&
2494                                  op->dest.reg == CFI_BP && cfi->drap) {
2495
2496                                 /* drap: mov %rsp, %rbp */
2497                                 regs[CFI_BP].base = CFI_BP;
2498                                 regs[CFI_BP].offset = -cfi->stack_size;
2499                                 cfi->bp_scratch = false;
2500                         }
2501
2502                         else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2503
2504                                 /*
2505                                  * mov %rsp, %reg
2506                                  *
2507                                  * This is needed for the rare case where GCC
2508                                  * does:
2509                                  *
2510                                  *   mov    %rsp, %rax
2511                                  *   ...
2512                                  *   mov    %rax, %rsp
2513                                  */
2514                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2515                                 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2516                         }
2517
2518                         else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2519                                  (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2520
2521                                 /*
2522                                  * mov %rbp, %rsp
2523                                  *
2524                                  * Restore the original stack pointer (Clang).
2525                                  */
2526                                 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2527                         }
2528
2529                         else if (op->dest.reg == cfa->base) {
2530
2531                                 /* mov %reg, %rsp */
2532                                 if (cfa->base == CFI_SP &&
2533                                     cfi->vals[op->src.reg].base == CFI_CFA) {
2534
2535                                         /*
2536                                          * This is needed for the rare case
2537                                          * where GCC does something dumb like:
2538                                          *
2539                                          *   lea    0x8(%rsp), %rcx
2540                                          *   ...
2541                                          *   mov    %rcx, %rsp
2542                                          */
2543                                         cfa->offset = -cfi->vals[op->src.reg].offset;
2544                                         cfi->stack_size = cfa->offset;
2545
2546                                 } else if (cfa->base == CFI_SP &&
2547                                            cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2548                                            cfi->vals[op->src.reg].offset == cfa->offset) {
2549
2550                                         /*
2551                                          * Stack swizzle:
2552                                          *
2553                                          * 1: mov %rsp, (%[tos])
2554                                          * 2: mov %[tos], %rsp
2555                                          *    ...
2556                                          * 3: pop %rsp
2557                                          *
2558                                          * Where:
2559                                          *
2560                                          * 1 - places a pointer to the previous
2561                                          *     stack at the Top-of-Stack of the
2562                                          *     new stack.
2563                                          *
2564                                          * 2 - switches to the new stack.
2565                                          *
2566                                          * 3 - pops the Top-of-Stack to restore
2567                                          *     the original stack.
2568                                          *
2569                                          * Note: we set base to SP_INDIRECT
2570                                          * here and preserve offset. Therefore
2571                                          * when the unwinder reaches ToS it
2572                                          * will dereference SP and then add the
2573                                          * offset to find the next frame, IOW:
2574                                          * (%rsp) + offset.
2575                                          */
2576                                         cfa->base = CFI_SP_INDIRECT;
2577
2578                                 } else {
2579                                         cfa->base = CFI_UNDEFINED;
2580                                         cfa->offset = 0;
2581                                 }
2582                         }
2583
2584                         else if (op->dest.reg == CFI_SP &&
2585                                  cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2586                                  cfi->vals[op->src.reg].offset == cfa->offset) {
2587
2588                                 /*
2589                                  * The same stack swizzle case 2) as above. But
2590                                  * because we can't change cfa->base, case 3)
2591                                  * will become a regular POP. Pretend we're a
2592                                  * PUSH so things don't go unbalanced.
2593                                  */
2594                                 cfi->stack_size += 8;
2595                         }
2596
2597
2598                         break;
2599
2600                 case OP_SRC_ADD:
2601                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2602
2603                                 /* add imm, %rsp */
2604                                 cfi->stack_size -= op->src.offset;
2605                                 if (cfa->base == CFI_SP)
2606                                         cfa->offset -= op->src.offset;
2607                                 break;
2608                         }
2609
2610                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2611
2612                                 /* lea disp(%rbp), %rsp */
2613                                 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2614                                 break;
2615                         }
2616
2617                         if (!cfi->drap && op->src.reg == CFI_SP &&
2618                             op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2619                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset + op->src.offset)) {
2620
2621                                 /* lea disp(%rsp), %rbp */
2622                                 cfa->base = CFI_BP;
2623                                 cfa->offset -= op->src.offset;
2624                                 cfi->bp_scratch = false;
2625                                 break;
2626                         }
2627
2628                         if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2629
2630                                 /* drap: lea disp(%rsp), %drap */
2631                                 cfi->drap_reg = op->dest.reg;
2632
2633                                 /*
2634                                  * lea disp(%rsp), %reg
2635                                  *
2636                                  * This is needed for the rare case where GCC
2637                                  * does something dumb like:
2638                                  *
2639                                  *   lea    0x8(%rsp), %rcx
2640                                  *   ...
2641                                  *   mov    %rcx, %rsp
2642                                  */
2643                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2644                                 cfi->vals[op->dest.reg].offset = \
2645                                         -cfi->stack_size + op->src.offset;
2646
2647                                 break;
2648                         }
2649
2650                         if (cfi->drap && op->dest.reg == CFI_SP &&
2651                             op->src.reg == cfi->drap_reg) {
2652
2653                                  /* drap: lea disp(%drap), %rsp */
2654                                 cfa->base = CFI_SP;
2655                                 cfa->offset = cfi->stack_size = -op->src.offset;
2656                                 cfi->drap_reg = CFI_UNDEFINED;
2657                                 cfi->drap = false;
2658                                 break;
2659                         }
2660
2661                         if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2662                                 WARN_FUNC("unsupported stack register modification",
2663                                           insn->sec, insn->offset);
2664                                 return -1;
2665                         }
2666
2667                         break;
2668
2669                 case OP_SRC_AND:
2670                         if (op->dest.reg != CFI_SP ||
2671                             (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2672                             (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2673                                 WARN_FUNC("unsupported stack pointer realignment",
2674                                           insn->sec, insn->offset);
2675                                 return -1;
2676                         }
2677
2678                         if (cfi->drap_reg != CFI_UNDEFINED) {
2679                                 /* drap: and imm, %rsp */
2680                                 cfa->base = cfi->drap_reg;
2681                                 cfa->offset = cfi->stack_size = 0;
2682                                 cfi->drap = true;
2683                         }
2684
2685                         /*
2686                          * Older versions of GCC (4.8ish) realign the stack
2687                          * without DRAP, with a frame pointer.
2688                          */
2689
2690                         break;
2691
2692                 case OP_SRC_POP:
2693                 case OP_SRC_POPF:
2694                         if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2695
2696                                 /* pop %rsp; # restore from a stack swizzle */
2697                                 cfa->base = CFI_SP;
2698                                 break;
2699                         }
2700
2701                         if (!cfi->drap && op->dest.reg == cfa->base) {
2702
2703                                 /* pop %rbp */
2704                                 cfa->base = CFI_SP;
2705                         }
2706
2707                         if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2708                             op->dest.reg == cfi->drap_reg &&
2709                             cfi->drap_offset == -cfi->stack_size) {
2710
2711                                 /* drap: pop %drap */
2712                                 cfa->base = cfi->drap_reg;
2713                                 cfa->offset = 0;
2714                                 cfi->drap_offset = -1;
2715
2716                         } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
2717
2718                                 /* pop %reg */
2719                                 restore_reg(cfi, op->dest.reg);
2720                         }
2721
2722                         cfi->stack_size -= 8;
2723                         if (cfa->base == CFI_SP)
2724                                 cfa->offset -= 8;
2725
2726                         break;
2727
2728                 case OP_SRC_REG_INDIRECT:
2729                         if (!cfi->drap && op->dest.reg == cfa->base &&
2730                             op->dest.reg == CFI_BP) {
2731
2732                                 /* mov disp(%rsp), %rbp */
2733                                 cfa->base = CFI_SP;
2734                                 cfa->offset = cfi->stack_size;
2735                         }
2736
2737                         if (cfi->drap && op->src.reg == CFI_BP &&
2738                             op->src.offset == cfi->drap_offset) {
2739
2740                                 /* drap: mov disp(%rbp), %drap */
2741                                 cfa->base = cfi->drap_reg;
2742                                 cfa->offset = 0;
2743                                 cfi->drap_offset = -1;
2744                         }
2745
2746                         if (cfi->drap && op->src.reg == CFI_BP &&
2747                             op->src.offset == regs[op->dest.reg].offset) {
2748
2749                                 /* drap: mov disp(%rbp), %reg */
2750                                 restore_reg(cfi, op->dest.reg);
2751
2752                         } else if (op->src.reg == cfa->base &&
2753                             op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
2754
2755                                 /* mov disp(%rbp), %reg */
2756                                 /* mov disp(%rsp), %reg */
2757                                 restore_reg(cfi, op->dest.reg);
2758
2759                         } else if (op->src.reg == CFI_SP &&
2760                                    op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
2761
2762                                 /* mov disp(%rsp), %reg */
2763                                 restore_reg(cfi, op->dest.reg);
2764                         }
2765
2766                         break;
2767
2768                 default:
2769                         WARN_FUNC("unknown stack-related instruction",
2770                                   insn->sec, insn->offset);
2771                         return -1;
2772                 }
2773
2774                 break;
2775
2776         case OP_DEST_PUSH:
2777         case OP_DEST_PUSHF:
2778                 cfi->stack_size += 8;
2779                 if (cfa->base == CFI_SP)
2780                         cfa->offset += 8;
2781
2782                 if (op->src.type != OP_SRC_REG)
2783                         break;
2784
2785                 if (cfi->drap) {
2786                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2787
2788                                 /* drap: push %drap */
2789                                 cfa->base = CFI_BP_INDIRECT;
2790                                 cfa->offset = -cfi->stack_size;
2791
2792                                 /* save drap so we know when to restore it */
2793                                 cfi->drap_offset = -cfi->stack_size;
2794
2795                         } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
2796
2797                                 /* drap: push %rbp */
2798                                 cfi->stack_size = 0;
2799
2800                         } else {
2801
2802                                 /* drap: push %reg */
2803                                 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
2804                         }
2805
2806                 } else {
2807
2808                         /* push %reg */
2809                         save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
2810                 }
2811
2812                 /* detect when asm code uses rbp as a scratch register */
2813                 if (opts.stackval && insn->func && op->src.reg == CFI_BP &&
2814                     cfa->base != CFI_BP)
2815                         cfi->bp_scratch = true;
2816                 break;
2817
2818         case OP_DEST_REG_INDIRECT:
2819
2820                 if (cfi->drap) {
2821                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2822
2823                                 /* drap: mov %drap, disp(%rbp) */
2824                                 cfa->base = CFI_BP_INDIRECT;
2825                                 cfa->offset = op->dest.offset;
2826
2827                                 /* save drap offset so we know when to restore it */
2828                                 cfi->drap_offset = op->dest.offset;
2829                         } else {
2830
2831                                 /* drap: mov reg, disp(%rbp) */
2832                                 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
2833                         }
2834
2835                 } else if (op->dest.reg == cfa->base) {
2836
2837                         /* mov reg, disp(%rbp) */
2838                         /* mov reg, disp(%rsp) */
2839                         save_reg(cfi, op->src.reg, CFI_CFA,
2840                                  op->dest.offset - cfi->cfa.offset);
2841
2842                 } else if (op->dest.reg == CFI_SP) {
2843
2844                         /* mov reg, disp(%rsp) */
2845                         save_reg(cfi, op->src.reg, CFI_CFA,
2846                                  op->dest.offset - cfi->stack_size);
2847
2848                 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
2849
2850                         /* mov %rsp, (%reg); # setup a stack swizzle. */
2851                         cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
2852                         cfi->vals[op->dest.reg].offset = cfa->offset;
2853                 }
2854
2855                 break;
2856
2857         case OP_DEST_MEM:
2858                 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
2859                         WARN_FUNC("unknown stack-related memory operation",
2860                                   insn->sec, insn->offset);
2861                         return -1;
2862                 }
2863
2864                 /* pop mem */
2865                 cfi->stack_size -= 8;
2866                 if (cfa->base == CFI_SP)
2867                         cfa->offset -= 8;
2868
2869                 break;
2870
2871         default:
2872                 WARN_FUNC("unknown stack-related instruction",
2873                           insn->sec, insn->offset);
2874                 return -1;
2875         }
2876
2877         return 0;
2878 }
2879
2880 /*
2881  * The stack layouts of alternatives instructions can sometimes diverge when
2882  * they have stack modifications.  That's fine as long as the potential stack
2883  * layouts don't conflict at any given potential instruction boundary.
2884  *
2885  * Flatten the CFIs of the different alternative code streams (both original
2886  * and replacement) into a single shared CFI array which can be used to detect
2887  * conflicts and nicely feed a linear array of ORC entries to the unwinder.
2888  */
2889 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
2890 {
2891         struct cfi_state **alt_cfi;
2892         int group_off;
2893
2894         if (!insn->alt_group)
2895                 return 0;
2896
2897         if (!insn->cfi) {
2898                 WARN("CFI missing");
2899                 return -1;
2900         }
2901
2902         alt_cfi = insn->alt_group->cfi;
2903         group_off = insn->offset - insn->alt_group->first_insn->offset;
2904
2905         if (!alt_cfi[group_off]) {
2906                 alt_cfi[group_off] = insn->cfi;
2907         } else {
2908                 if (cficmp(alt_cfi[group_off], insn->cfi)) {
2909                         WARN_FUNC("stack layout conflict in alternatives",
2910                                   insn->sec, insn->offset);
2911                         return -1;
2912                 }
2913         }
2914
2915         return 0;
2916 }
2917
2918 static int handle_insn_ops(struct instruction *insn,
2919                            struct instruction *next_insn,
2920                            struct insn_state *state)
2921 {
2922         struct stack_op *op;
2923
2924         list_for_each_entry(op, &insn->stack_ops, list) {
2925
2926                 if (update_cfi_state(insn, next_insn, &state->cfi, op))
2927                         return 1;
2928
2929                 if (!insn->alt_group)
2930                         continue;
2931
2932                 if (op->dest.type == OP_DEST_PUSHF) {
2933                         if (!state->uaccess_stack) {
2934                                 state->uaccess_stack = 1;
2935                         } else if (state->uaccess_stack >> 31) {
2936                                 WARN_FUNC("PUSHF stack exhausted",
2937                                           insn->sec, insn->offset);
2938                                 return 1;
2939                         }
2940                         state->uaccess_stack <<= 1;
2941                         state->uaccess_stack  |= state->uaccess;
2942                 }
2943
2944                 if (op->src.type == OP_SRC_POPF) {
2945                         if (state->uaccess_stack) {
2946                                 state->uaccess = state->uaccess_stack & 1;
2947                                 state->uaccess_stack >>= 1;
2948                                 if (state->uaccess_stack == 1)
2949                                         state->uaccess_stack = 0;
2950                         }
2951                 }
2952         }
2953
2954         return 0;
2955 }
2956
2957 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
2958 {
2959         struct cfi_state *cfi1 = insn->cfi;
2960         int i;
2961
2962         if (!cfi1) {
2963                 WARN("CFI missing");
2964                 return false;
2965         }
2966
2967         if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
2968
2969                 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
2970                           insn->sec, insn->offset,
2971                           cfi1->cfa.base, cfi1->cfa.offset,
2972                           cfi2->cfa.base, cfi2->cfa.offset);
2973
2974         } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
2975                 for (i = 0; i < CFI_NUM_REGS; i++) {
2976                         if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
2977                                     sizeof(struct cfi_reg)))
2978                                 continue;
2979
2980                         WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
2981                                   insn->sec, insn->offset,
2982                                   i, cfi1->regs[i].base, cfi1->regs[i].offset,
2983                                   i, cfi2->regs[i].base, cfi2->regs[i].offset);
2984                         break;
2985                 }
2986
2987         } else if (cfi1->type != cfi2->type) {
2988
2989                 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
2990                           insn->sec, insn->offset, cfi1->type, cfi2->type);
2991
2992         } else if (cfi1->drap != cfi2->drap ||
2993                    (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
2994                    (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
2995
2996                 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
2997                           insn->sec, insn->offset,
2998                           cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
2999                           cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3000
3001         } else
3002                 return true;
3003
3004         return false;
3005 }
3006
3007 static inline bool func_uaccess_safe(struct symbol *func)
3008 {
3009         if (func)
3010                 return func->uaccess_safe;
3011
3012         return false;
3013 }
3014
3015 static inline const char *call_dest_name(struct instruction *insn)
3016 {
3017         static char pvname[19];
3018         struct reloc *rel;
3019         int idx;
3020
3021         if (insn->call_dest)
3022                 return insn->call_dest->name;
3023
3024         rel = insn_reloc(NULL, insn);
3025         if (rel && !strcmp(rel->sym->name, "pv_ops")) {
3026                 idx = (rel->addend / sizeof(void *));
3027                 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3028                 return pvname;
3029         }
3030
3031         return "{dynamic}";
3032 }
3033
3034 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3035 {
3036         struct symbol *target;
3037         struct reloc *rel;
3038         int idx;
3039
3040         rel = insn_reloc(file, insn);
3041         if (!rel || strcmp(rel->sym->name, "pv_ops"))
3042                 return false;
3043
3044         idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *));
3045
3046         if (file->pv_ops[idx].clean)
3047                 return true;
3048
3049         file->pv_ops[idx].clean = true;
3050
3051         list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3052                 if (!target->sec->noinstr) {
3053                         WARN("pv_ops[%d]: %s", idx, target->name);
3054                         file->pv_ops[idx].clean = false;
3055                 }
3056         }
3057
3058         return file->pv_ops[idx].clean;
3059 }
3060
3061 static inline bool noinstr_call_dest(struct objtool_file *file,
3062                                      struct instruction *insn,
3063                                      struct symbol *func)
3064 {
3065         /*
3066          * We can't deal with indirect function calls at present;
3067          * assume they're instrumented.
3068          */
3069         if (!func) {
3070                 if (file->pv_ops)
3071                         return pv_call_dest(file, insn);
3072
3073                 return false;
3074         }
3075
3076         /*
3077          * If the symbol is from a noinstr section; we good.
3078          */
3079         if (func->sec->noinstr)
3080                 return true;
3081
3082         /*
3083          * The __ubsan_handle_*() calls are like WARN(), they only happen when
3084          * something 'BAD' happened. At the risk of taking the machine down,
3085          * let them proceed to get the message out.
3086          */
3087         if (!strncmp(func->name, "__ubsan_handle_", 15))
3088                 return true;
3089
3090         return false;
3091 }
3092
3093 static int validate_call(struct objtool_file *file,
3094                          struct instruction *insn,
3095                          struct insn_state *state)
3096 {
3097         if (state->noinstr && state->instr <= 0 &&
3098             !noinstr_call_dest(file, insn, insn->call_dest)) {
3099                 WARN_FUNC("call to %s() leaves .noinstr.text section",
3100                                 insn->sec, insn->offset, call_dest_name(insn));
3101                 return 1;
3102         }
3103
3104         if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
3105                 WARN_FUNC("call to %s() with UACCESS enabled",
3106                                 insn->sec, insn->offset, call_dest_name(insn));
3107                 return 1;
3108         }
3109
3110         if (state->df) {
3111                 WARN_FUNC("call to %s() with DF set",
3112                                 insn->sec, insn->offset, call_dest_name(insn));
3113                 return 1;
3114         }
3115
3116         return 0;
3117 }
3118
3119 static int validate_sibling_call(struct objtool_file *file,
3120                                  struct instruction *insn,
3121                                  struct insn_state *state)
3122 {
3123         if (has_modified_stack_frame(insn, state)) {
3124                 WARN_FUNC("sibling call from callable instruction with modified stack frame",
3125                                 insn->sec, insn->offset);
3126                 return 1;
3127         }
3128
3129         return validate_call(file, insn, state);
3130 }
3131
3132 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3133 {
3134         if (state->noinstr && state->instr > 0) {
3135                 WARN_FUNC("return with instrumentation enabled",
3136                           insn->sec, insn->offset);
3137                 return 1;
3138         }
3139
3140         if (state->uaccess && !func_uaccess_safe(func)) {
3141                 WARN_FUNC("return with UACCESS enabled",
3142                           insn->sec, insn->offset);
3143                 return 1;
3144         }
3145
3146         if (!state->uaccess && func_uaccess_safe(func)) {
3147                 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
3148                           insn->sec, insn->offset);
3149                 return 1;
3150         }
3151
3152         if (state->df) {
3153                 WARN_FUNC("return with DF set",
3154                           insn->sec, insn->offset);
3155                 return 1;
3156         }
3157
3158         if (func && has_modified_stack_frame(insn, state)) {
3159                 WARN_FUNC("return with modified stack frame",
3160                           insn->sec, insn->offset);
3161                 return 1;
3162         }
3163
3164         if (state->cfi.bp_scratch) {
3165                 WARN_FUNC("BP used as a scratch register",
3166                           insn->sec, insn->offset);
3167                 return 1;
3168         }
3169
3170         return 0;
3171 }
3172
3173 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3174                                                  struct instruction *insn)
3175 {
3176         struct alt_group *alt_group = insn->alt_group;
3177
3178         /*
3179          * Simulate the fact that alternatives are patched in-place.  When the
3180          * end of a replacement alt_group is reached, redirect objtool flow to
3181          * the end of the original alt_group.
3182          */
3183         if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
3184                 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3185
3186         return next_insn_same_sec(file, insn);
3187 }
3188
3189 /*
3190  * Follow the branch starting at the given instruction, and recursively follow
3191  * any other branches (jumps).  Meanwhile, track the frame pointer state at
3192  * each instruction and validate all the rules described in
3193  * tools/objtool/Documentation/stack-validation.txt.
3194  */
3195 static int validate_branch(struct objtool_file *file, struct symbol *func,
3196                            struct instruction *insn, struct insn_state state)
3197 {
3198         struct alternative *alt;
3199         struct instruction *next_insn, *prev_insn = NULL;
3200         struct section *sec;
3201         u8 visited;
3202         int ret;
3203
3204         sec = insn->sec;
3205
3206         while (1) {
3207                 next_insn = next_insn_to_validate(file, insn);
3208
3209                 if (func && insn->func && func != insn->func->pfunc) {
3210                         WARN("%s() falls through to next function %s()",
3211                              func->name, insn->func->name);
3212                         return 1;
3213                 }
3214
3215                 if (func && insn->ignore) {
3216                         WARN_FUNC("BUG: why am I validating an ignored function?",
3217                                   sec, insn->offset);
3218                         return 1;
3219                 }
3220
3221                 visited = 1 << state.uaccess;
3222                 if (insn->visited) {
3223                         if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3224                                 return 1;
3225
3226                         if (insn->visited & visited)
3227                                 return 0;
3228                 } else {
3229                         nr_insns_visited++;
3230                 }
3231
3232                 if (state.noinstr)
3233                         state.instr += insn->instr;
3234
3235                 if (insn->hint) {
3236                         state.cfi = *insn->cfi;
3237                 } else {
3238                         /* XXX track if we actually changed state.cfi */
3239
3240                         if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3241                                 insn->cfi = prev_insn->cfi;
3242                                 nr_cfi_reused++;
3243                         } else {
3244                                 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3245                         }
3246                 }
3247
3248                 insn->visited |= visited;
3249
3250                 if (propagate_alt_cfi(file, insn))
3251                         return 1;
3252
3253                 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3254                         bool skip_orig = false;
3255
3256                         list_for_each_entry(alt, &insn->alts, list) {
3257                                 if (alt->skip_orig)
3258                                         skip_orig = true;
3259
3260                                 ret = validate_branch(file, func, alt->insn, state);
3261                                 if (ret) {
3262                                         if (opts.backtrace)
3263                                                 BT_FUNC("(alt)", insn);
3264                                         return ret;
3265                                 }
3266                         }
3267
3268                         if (skip_orig)
3269                                 return 0;
3270                 }
3271
3272                 if (handle_insn_ops(insn, next_insn, &state))
3273                         return 1;
3274
3275                 switch (insn->type) {
3276
3277                 case INSN_RETURN:
3278                         return validate_return(func, insn, &state);
3279
3280                 case INSN_CALL:
3281                 case INSN_CALL_DYNAMIC:
3282                         ret = validate_call(file, insn, &state);
3283                         if (ret)
3284                                 return ret;
3285
3286                         if (opts.stackval && func && !is_fentry_call(insn) &&
3287                             !has_valid_stack_frame(&state)) {
3288                                 WARN_FUNC("call without frame pointer save/setup",
3289                                           sec, insn->offset);
3290                                 return 1;
3291                         }
3292
3293                         if (insn->dead_end)
3294                                 return 0;
3295
3296                         break;
3297
3298                 case INSN_JUMP_CONDITIONAL:
3299                 case INSN_JUMP_UNCONDITIONAL:
3300                         if (is_sibling_call(insn)) {
3301                                 ret = validate_sibling_call(file, insn, &state);
3302                                 if (ret)
3303                                         return ret;
3304
3305                         } else if (insn->jump_dest) {
3306                                 ret = validate_branch(file, func,
3307                                                       insn->jump_dest, state);
3308                                 if (ret) {
3309                                         if (opts.backtrace)
3310                                                 BT_FUNC("(branch)", insn);
3311                                         return ret;
3312                                 }
3313                         }
3314
3315                         if (insn->type == INSN_JUMP_UNCONDITIONAL)
3316                                 return 0;
3317
3318                         break;
3319
3320                 case INSN_JUMP_DYNAMIC:
3321                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3322                         if (is_sibling_call(insn)) {
3323                                 ret = validate_sibling_call(file, insn, &state);
3324                                 if (ret)
3325                                         return ret;
3326                         }
3327
3328                         if (insn->type == INSN_JUMP_DYNAMIC)
3329                                 return 0;
3330
3331                         break;
3332
3333                 case INSN_CONTEXT_SWITCH:
3334                         if (func && (!next_insn || !next_insn->hint)) {
3335                                 WARN_FUNC("unsupported instruction in callable function",
3336                                           sec, insn->offset);
3337                                 return 1;
3338                         }
3339                         return 0;
3340
3341                 case INSN_STAC:
3342                         if (state.uaccess) {
3343                                 WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
3344                                 return 1;
3345                         }
3346
3347                         state.uaccess = true;
3348                         break;
3349
3350                 case INSN_CLAC:
3351                         if (!state.uaccess && func) {
3352                                 WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
3353                                 return 1;
3354                         }
3355
3356                         if (func_uaccess_safe(func) && !state.uaccess_stack) {
3357                                 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
3358                                 return 1;
3359                         }
3360
3361                         state.uaccess = false;
3362                         break;
3363
3364                 case INSN_STD:
3365                         if (state.df) {
3366                                 WARN_FUNC("recursive STD", sec, insn->offset);
3367                                 return 1;
3368                         }
3369
3370                         state.df = true;
3371                         break;
3372
3373                 case INSN_CLD:
3374                         if (!state.df && func) {
3375                                 WARN_FUNC("redundant CLD", sec, insn->offset);
3376                                 return 1;
3377                         }
3378
3379                         state.df = false;
3380                         break;
3381
3382                 default:
3383                         break;
3384                 }
3385
3386                 if (insn->dead_end)
3387                         return 0;
3388
3389                 if (!next_insn) {
3390                         if (state.cfi.cfa.base == CFI_UNDEFINED)
3391                                 return 0;
3392                         WARN("%s: unexpected end of section", sec->name);
3393                         return 1;
3394                 }
3395
3396                 prev_insn = insn;
3397                 insn = next_insn;
3398         }
3399
3400         return 0;
3401 }
3402
3403 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3404 {
3405         struct instruction *insn;
3406         struct insn_state state;
3407         int ret, warnings = 0;
3408
3409         if (!file->hints)
3410                 return 0;
3411
3412         init_insn_state(file, &state, sec);
3413
3414         if (sec) {
3415                 insn = find_insn(file, sec, 0);
3416                 if (!insn)
3417                         return 0;
3418         } else {
3419                 insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3420         }
3421
3422         while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
3423                 if (insn->hint && !insn->visited && !insn->ignore) {
3424                         ret = validate_branch(file, insn->func, insn, state);
3425                         if (ret && opts.backtrace)
3426                                 BT_FUNC("<=== (hint)", insn);
3427                         warnings += ret;
3428                 }
3429
3430                 insn = list_next_entry(insn, list);
3431         }
3432
3433         return warnings;
3434 }
3435
3436 static int validate_retpoline(struct objtool_file *file)
3437 {
3438         struct instruction *insn;
3439         int warnings = 0;
3440
3441         for_each_insn(file, insn) {
3442                 if (insn->type != INSN_JUMP_DYNAMIC &&
3443                     insn->type != INSN_CALL_DYNAMIC)
3444                         continue;
3445
3446                 if (insn->retpoline_safe)
3447                         continue;
3448
3449                 /*
3450                  * .init.text code is ran before userspace and thus doesn't
3451                  * strictly need retpolines, except for modules which are
3452                  * loaded late, they very much do need retpoline in their
3453                  * .init.text
3454                  */
3455                 if (!strcmp(insn->sec->name, ".init.text") && !opts.module)
3456                         continue;
3457
3458                 WARN_FUNC("indirect %s found in RETPOLINE build",
3459                           insn->sec, insn->offset,
3460                           insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3461
3462                 warnings++;
3463         }
3464
3465         return warnings;
3466 }
3467
3468 static bool is_kasan_insn(struct instruction *insn)
3469 {
3470         return (insn->type == INSN_CALL &&
3471                 !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3472 }
3473
3474 static bool is_ubsan_insn(struct instruction *insn)
3475 {
3476         return (insn->type == INSN_CALL &&
3477                 !strcmp(insn->call_dest->name,
3478                         "__ubsan_handle_builtin_unreachable"));
3479 }
3480
3481 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
3482 {
3483         int i;
3484         struct instruction *prev_insn;
3485
3486         if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
3487                 return true;
3488
3489         /*
3490          * Ignore alternative replacement instructions.  This can happen
3491          * when a whitelisted function uses one of the ALTERNATIVE macros.
3492          */
3493         if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
3494             !strcmp(insn->sec->name, ".altinstr_aux"))
3495                 return true;
3496
3497         /*
3498          * Whole archive runs might encounter dead code from weak symbols.
3499          * This is where the linker will have dropped the weak symbol in
3500          * favour of a regular symbol, but leaves the code in place.
3501          *
3502          * In this case we'll find a piece of code (whole function) that is not
3503          * covered by a !section symbol. Ignore them.
3504          */
3505         if (opts.link && !insn->func) {
3506                 int size = find_symbol_hole_containing(insn->sec, insn->offset);
3507                 unsigned long end = insn->offset + size;
3508
3509                 if (!size) /* not a hole */
3510                         return false;
3511
3512                 if (size < 0) /* hole until the end */
3513                         return true;
3514
3515                 sec_for_each_insn_continue(file, insn) {
3516                         /*
3517                          * If we reach a visited instruction at or before the
3518                          * end of the hole, ignore the unreachable.
3519                          */
3520                         if (insn->visited)
3521                                 return true;
3522
3523                         if (insn->offset >= end)
3524                                 break;
3525
3526                         /*
3527                          * If this hole jumps to a .cold function, mark it ignore too.
3528                          */
3529                         if (insn->jump_dest && insn->jump_dest->func &&
3530                             strstr(insn->jump_dest->func->name, ".cold")) {
3531                                 struct instruction *dest = insn->jump_dest;
3532                                 func_for_each_insn(file, dest->func, dest)
3533                                         dest->ignore = true;
3534                         }
3535                 }
3536
3537                 return false;
3538         }
3539
3540         if (!insn->func)
3541                 return false;
3542
3543         if (insn->func->static_call_tramp)
3544                 return true;
3545
3546         /*
3547          * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
3548          * __builtin_unreachable().  The BUG() macro has an unreachable() after
3549          * the UD2, which causes GCC's undefined trap logic to emit another UD2
3550          * (or occasionally a JMP to UD2).
3551          *
3552          * It may also insert a UD2 after calling a __noreturn function.
3553          */
3554         prev_insn = list_prev_entry(insn, list);
3555         if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
3556             (insn->type == INSN_BUG ||
3557              (insn->type == INSN_JUMP_UNCONDITIONAL &&
3558               insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
3559                 return true;
3560
3561         /*
3562          * Check if this (or a subsequent) instruction is related to
3563          * CONFIG_UBSAN or CONFIG_KASAN.
3564          *
3565          * End the search at 5 instructions to avoid going into the weeds.
3566          */
3567         for (i = 0; i < 5; i++) {
3568
3569                 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
3570                         return true;
3571
3572                 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3573                         if (insn->jump_dest &&
3574                             insn->jump_dest->func == insn->func) {
3575                                 insn = insn->jump_dest;
3576                                 continue;
3577                         }
3578
3579                         break;
3580                 }
3581
3582                 if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
3583                         break;
3584
3585                 insn = list_next_entry(insn, list);
3586         }
3587
3588         return false;
3589 }
3590
3591 static int validate_symbol(struct objtool_file *file, struct section *sec,
3592                            struct symbol *sym, struct insn_state *state)
3593 {
3594         struct instruction *insn;
3595         int ret;
3596
3597         if (!sym->len) {
3598                 WARN("%s() is missing an ELF size annotation", sym->name);
3599                 return 1;
3600         }
3601
3602         if (sym->pfunc != sym || sym->alias != sym)
3603                 return 0;
3604
3605         insn = find_insn(file, sec, sym->offset);
3606         if (!insn || insn->ignore || insn->visited)
3607                 return 0;
3608
3609         state->uaccess = sym->uaccess_safe;
3610
3611         ret = validate_branch(file, insn->func, insn, *state);
3612         if (ret && opts.backtrace)
3613                 BT_FUNC("<=== (sym)", insn);
3614         return ret;
3615 }
3616
3617 static int validate_section(struct objtool_file *file, struct section *sec)
3618 {
3619         struct insn_state state;
3620         struct symbol *func;
3621         int warnings = 0;
3622
3623         list_for_each_entry(func, &sec->symbol_list, list) {
3624                 if (func->type != STT_FUNC)
3625                         continue;
3626
3627                 init_insn_state(file, &state, sec);
3628                 set_func_state(&state.cfi);
3629
3630                 warnings += validate_symbol(file, sec, func, &state);
3631         }
3632
3633         return warnings;
3634 }
3635
3636 static int validate_noinstr_sections(struct objtool_file *file)
3637 {
3638         struct section *sec;
3639         int warnings = 0;
3640
3641         sec = find_section_by_name(file->elf, ".noinstr.text");
3642         if (sec) {
3643                 warnings += validate_section(file, sec);
3644                 warnings += validate_unwind_hints(file, sec);
3645         }
3646
3647         sec = find_section_by_name(file->elf, ".entry.text");
3648         if (sec) {
3649                 warnings += validate_section(file, sec);
3650                 warnings += validate_unwind_hints(file, sec);
3651         }
3652
3653         return warnings;
3654 }
3655
3656 static int validate_functions(struct objtool_file *file)
3657 {
3658         struct section *sec;
3659         int warnings = 0;
3660
3661         for_each_sec(file, sec) {
3662                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
3663                         continue;
3664
3665                 warnings += validate_section(file, sec);
3666         }
3667
3668         return warnings;
3669 }
3670
3671 static void mark_endbr_used(struct instruction *insn)
3672 {
3673         if (!list_empty(&insn->call_node))
3674                 list_del_init(&insn->call_node);
3675 }
3676
3677 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
3678 {
3679         struct instruction *dest;
3680         struct reloc *reloc;
3681         unsigned long off;
3682         int warnings = 0;
3683
3684         /*
3685          * Looking for function pointer load relocations.  Ignore
3686          * direct/indirect branches:
3687          */
3688         switch (insn->type) {
3689         case INSN_CALL:
3690         case INSN_CALL_DYNAMIC:
3691         case INSN_JUMP_CONDITIONAL:
3692         case INSN_JUMP_UNCONDITIONAL:
3693         case INSN_JUMP_DYNAMIC:
3694         case INSN_JUMP_DYNAMIC_CONDITIONAL:
3695         case INSN_RETURN:
3696         case INSN_NOP:
3697                 return 0;
3698         default:
3699                 break;
3700         }
3701
3702         for (reloc = insn_reloc(file, insn);
3703              reloc;
3704              reloc = find_reloc_by_dest_range(file->elf, insn->sec,
3705                                               reloc->offset + 1,
3706                                               (insn->offset + insn->len) - (reloc->offset + 1))) {
3707
3708                 /*
3709                  * static_call_update() references the trampoline, which
3710                  * doesn't have (or need) ENDBR.  Skip warning in that case.
3711                  */
3712                 if (reloc->sym->static_call_tramp)
3713                         continue;
3714
3715                 off = reloc->sym->offset;
3716                 if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)
3717                         off += arch_dest_reloc_offset(reloc->addend);
3718                 else
3719                         off += reloc->addend;
3720
3721                 dest = find_insn(file, reloc->sym->sec, off);
3722                 if (!dest)
3723                         continue;
3724
3725                 if (dest->type == INSN_ENDBR) {
3726                         mark_endbr_used(dest);
3727                         continue;
3728                 }
3729
3730                 if (dest->func && dest->func == insn->func) {
3731                         /*
3732                          * Anything from->to self is either _THIS_IP_ or
3733                          * IRET-to-self.
3734                          *
3735                          * There is no sane way to annotate _THIS_IP_ since the
3736                          * compiler treats the relocation as a constant and is
3737                          * happy to fold in offsets, skewing any annotation we
3738                          * do, leading to vast amounts of false-positives.
3739                          *
3740                          * There's also compiler generated _THIS_IP_ through
3741                          * KCOV and such which we have no hope of annotating.
3742                          *
3743                          * As such, blanket accept self-references without
3744                          * issue.
3745                          */
3746                         continue;
3747                 }
3748
3749                 if (dest->noendbr)
3750                         continue;
3751
3752                 WARN_FUNC("relocation to !ENDBR: %s",
3753                           insn->sec, insn->offset,
3754                           offstr(dest->sec, dest->offset));
3755
3756                 warnings++;
3757         }
3758
3759         return warnings;
3760 }
3761
3762 static int validate_ibt_data_reloc(struct objtool_file *file,
3763                                    struct reloc *reloc)
3764 {
3765         struct instruction *dest;
3766
3767         dest = find_insn(file, reloc->sym->sec,
3768                          reloc->sym->offset + reloc->addend);
3769         if (!dest)
3770                 return 0;
3771
3772         if (dest->type == INSN_ENDBR) {
3773                 mark_endbr_used(dest);
3774                 return 0;
3775         }
3776
3777         if (dest->noendbr)
3778                 return 0;
3779
3780         WARN_FUNC("data relocation to !ENDBR: %s",
3781                   reloc->sec->base, reloc->offset,
3782                   offstr(dest->sec, dest->offset));
3783
3784         return 1;
3785 }
3786
3787 /*
3788  * Validate IBT rules and remove used ENDBR instructions from the seal list.
3789  * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
3790  * NOPs) later, in create_ibt_endbr_seal_sections().
3791  */
3792 static int validate_ibt(struct objtool_file *file)
3793 {
3794         struct section *sec;
3795         struct reloc *reloc;
3796         struct instruction *insn;
3797         int warnings = 0;
3798
3799         for_each_insn(file, insn)
3800                 warnings += validate_ibt_insn(file, insn);
3801
3802         for_each_sec(file, sec) {
3803
3804                 /* Already done by validate_ibt_insn() */
3805                 if (sec->sh.sh_flags & SHF_EXECINSTR)
3806                         continue;
3807
3808                 if (!sec->reloc)
3809                         continue;
3810
3811                 /*
3812                  * These sections can reference text addresses, but not with
3813                  * the intent to indirect branch to them.
3814                  */
3815                 if (!strncmp(sec->name, ".discard", 8)                  ||
3816                     !strncmp(sec->name, ".debug", 6)                    ||
3817                     !strcmp(sec->name, ".altinstructions")              ||
3818                     !strcmp(sec->name, ".ibt_endbr_seal")               ||
3819                     !strcmp(sec->name, ".orc_unwind_ip")                ||
3820                     !strcmp(sec->name, ".parainstructions")             ||
3821                     !strcmp(sec->name, ".retpoline_sites")              ||
3822                     !strcmp(sec->name, ".smp_locks")                    ||
3823                     !strcmp(sec->name, ".static_call_sites")            ||
3824                     !strcmp(sec->name, "_error_injection_whitelist")    ||
3825                     !strcmp(sec->name, "_kprobe_blacklist")             ||
3826                     !strcmp(sec->name, "__bug_table")                   ||
3827                     !strcmp(sec->name, "__ex_table")                    ||
3828                     !strcmp(sec->name, "__jump_table")                  ||
3829                     !strcmp(sec->name, "__mcount_loc")                  ||
3830                     !strcmp(sec->name, "__tracepoints"))
3831                         continue;
3832
3833                 list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
3834                         warnings += validate_ibt_data_reloc(file, reloc);
3835         }
3836
3837         return warnings;
3838 }
3839
3840 static int validate_sls(struct objtool_file *file)
3841 {
3842         struct instruction *insn, *next_insn;
3843         int warnings = 0;
3844
3845         for_each_insn(file, insn) {
3846                 next_insn = next_insn_same_sec(file, insn);
3847
3848                 if (insn->retpoline_safe)
3849                         continue;
3850
3851                 switch (insn->type) {
3852                 case INSN_RETURN:
3853                         if (!next_insn || next_insn->type != INSN_TRAP) {
3854                                 WARN_FUNC("missing int3 after ret",
3855                                           insn->sec, insn->offset);
3856                                 warnings++;
3857                         }
3858
3859                         break;
3860                 case INSN_JUMP_DYNAMIC:
3861                         if (!next_insn || next_insn->type != INSN_TRAP) {
3862                                 WARN_FUNC("missing int3 after indirect jump",
3863                                           insn->sec, insn->offset);
3864                                 warnings++;
3865                         }
3866                         break;
3867                 default:
3868                         break;
3869                 }
3870         }
3871
3872         return warnings;
3873 }
3874
3875 static int validate_reachable_instructions(struct objtool_file *file)
3876 {
3877         struct instruction *insn;
3878
3879         if (file->ignore_unreachables)
3880                 return 0;
3881
3882         for_each_insn(file, insn) {
3883                 if (insn->visited || ignore_unreachable_insn(file, insn))
3884                         continue;
3885
3886                 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
3887                 return 1;
3888         }
3889
3890         return 0;
3891 }
3892
3893 int check(struct objtool_file *file)
3894 {
3895         int ret, warnings = 0;
3896
3897         arch_initial_func_cfi_state(&initial_func_cfi);
3898         init_cfi_state(&init_cfi);
3899         init_cfi_state(&func_cfi);
3900         set_func_state(&func_cfi);
3901
3902         if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
3903                 goto out;
3904
3905         cfi_hash_add(&init_cfi);
3906         cfi_hash_add(&func_cfi);
3907
3908         ret = decode_sections(file);
3909         if (ret < 0)
3910                 goto out;
3911
3912         warnings += ret;
3913
3914         if (list_empty(&file->insn_list))
3915                 goto out;
3916
3917         if (opts.retpoline) {
3918                 ret = validate_retpoline(file);
3919                 if (ret < 0)
3920                         return ret;
3921                 warnings += ret;
3922         }
3923
3924         if (opts.stackval || opts.orc || opts.uaccess) {
3925                 ret = validate_functions(file);
3926                 if (ret < 0)
3927                         goto out;
3928                 warnings += ret;
3929
3930                 ret = validate_unwind_hints(file, NULL);
3931                 if (ret < 0)
3932                         goto out;
3933                 warnings += ret;
3934
3935                 if (!warnings) {
3936                         ret = validate_reachable_instructions(file);
3937                         if (ret < 0)
3938                                 goto out;
3939                         warnings += ret;
3940                 }
3941
3942         } else if (opts.noinstr) {
3943                 ret = validate_noinstr_sections(file);
3944                 if (ret < 0)
3945                         goto out;
3946                 warnings += ret;
3947         }
3948
3949         if (opts.ibt) {
3950                 ret = validate_ibt(file);
3951                 if (ret < 0)
3952                         goto out;
3953                 warnings += ret;
3954         }
3955
3956         if (opts.sls) {
3957                 ret = validate_sls(file);
3958                 if (ret < 0)
3959                         goto out;
3960                 warnings += ret;
3961         }
3962
3963         if (opts.static_call) {
3964                 ret = create_static_call_sections(file);
3965                 if (ret < 0)
3966                         goto out;
3967                 warnings += ret;
3968         }
3969
3970         if (opts.retpoline) {
3971                 ret = create_retpoline_sites_sections(file);
3972                 if (ret < 0)
3973                         goto out;
3974                 warnings += ret;
3975         }
3976
3977         if (opts.mcount) {
3978                 ret = create_mcount_loc_sections(file);
3979                 if (ret < 0)
3980                         goto out;
3981                 warnings += ret;
3982         }
3983
3984         if (opts.ibt) {
3985                 ret = create_ibt_endbr_seal_sections(file);
3986                 if (ret < 0)
3987                         goto out;
3988                 warnings += ret;
3989         }
3990
3991         if (opts.orc && !list_empty(&file->insn_list)) {
3992                 ret = orc_create(file);
3993                 if (ret < 0)
3994                         goto out;
3995                 warnings += ret;
3996         }
3997
3998
3999         if (opts.stats) {
4000                 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4001                 printf("nr_cfi: %ld\n", nr_cfi);
4002                 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4003                 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4004         }
4005
4006 out:
4007         /*
4008          *  For now, don't fail the kernel build on fatal warnings.  These
4009          *  errors are still fairly common due to the growing matrix of
4010          *  supported toolchains and their recent pace of change.
4011          */
4012         return 0;
4013 }