tools/objtool/check.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
   4  */
   5
   6 #include <string.h>
   7 #include <stdlib.h>
   8 #include <inttypes.h>
   9
  10 #include <arch/elf.h>
  11 #include <objtool/builtin.h>
  12 #include <objtool/cfi.h>
  13 #include <objtool/arch.h>
  14 #include <objtool/check.h>
  15 #include <objtool/special.h>
  16 #include <objtool/warn.h>
  17 #include <objtool/endianness.h>
  18
  19 #include <linux/objtool.h>
  20 #include <linux/hashtable.h>
  21 #include <linux/kernel.h>
  22 #include <linux/static_call_types.h>
  23
  24 struct alternative {
  25         struct list_head list;
  26         struct instruction *insn;
  27         bool skip_orig;
  28 };
  29
  30 struct cfi_init_state initial_func_cfi;
  31
  32 struct instruction *find_insn(struct objtool_file *file,
  33                               struct section *sec, unsigned long offset)
  34 {
  35         struct instruction *insn;
  36
  37         hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
  38                 if (insn->sec == sec && insn->offset == offset)
  39                         return insn;
  40         }
  41
  42         return NULL;
  43 }
  44
  45 static struct instruction *next_insn_same_sec(struct objtool_file *file,
  46                                               struct instruction *insn)
  47 {
  48         struct instruction *next = list_next_entry(insn, list);
  49
  50         if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
  51                 return NULL;
  52
  53         return next;
  54 }
  55
  56 static struct instruction *next_insn_same_func(struct objtool_file *file,
  57                                                struct instruction *insn)
  58 {
  59         struct instruction *next = list_next_entry(insn, list);
  60         struct symbol *func = insn->func;
  61
  62         if (!func)
  63                 return NULL;
  64
  65         if (&next->list != &file->insn_list && next->func == func)
  66                 return next;
  67
  68         /* Check if we're already in the subfunction: */
  69         if (func == func->cfunc)
  70                 return NULL;
  71
  72         /* Move to the subfunction: */
  73         return find_insn(file, func->cfunc->sec, func->cfunc->offset);
  74 }
  75
  76 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
  77                                                struct instruction *insn)
  78 {
  79         struct instruction *prev = list_prev_entry(insn, list);
  80
  81         if (&prev->list != &file->insn_list && prev->func == insn->func)
  82                 return prev;
  83
  84         return NULL;
  85 }
  86
  87 #define func_for_each_insn(file, func, insn)                            \
  88         for (insn = find_insn(file, func->sec, func->offset);           \
  89              insn;                                                      \
  90              insn = next_insn_same_func(file, insn))
  91
  92 #define sym_for_each_insn(file, sym, insn)                              \
  93         for (insn = find_insn(file, sym->sec, sym->offset);             \
  94              insn && &insn->list != &file->insn_list &&                 \
  95                 insn->sec == sym->sec &&                                \
  96                 insn->offset < sym->offset + sym->len;                  \
  97              insn = list_next_entry(insn, list))
  98
  99 #define sym_for_each_insn_continue_reverse(file, sym, insn)             \
 100         for (insn = list_prev_entry(insn, list);                        \
 101              &insn->list != &file->insn_list &&                         \
 102                 insn->sec == sym->sec && insn->offset >= sym->offset;   \
 103              insn = list_prev_entry(insn, list))
 104
 105 #define sec_for_each_insn_from(file, insn)                              \
 106         for (; insn; insn = next_insn_same_sec(file, insn))
 107
 108 #define sec_for_each_insn_continue(file, insn)                          \
 109         for (insn = next_insn_same_sec(file, insn); insn;               \
 110              insn = next_insn_same_sec(file, insn))
 111
 112 static bool is_jump_table_jump(struct instruction *insn)
 113 {
 114         struct alt_group *alt_group = insn->alt_group;
 115
 116         if (insn->jump_table)
 117                 return true;
 118
 119         /* Retpoline alternative for a jump table? */
 120         return alt_group && alt_group->orig_group &&
 121                alt_group->orig_group->first_insn->jump_table;
 122 }
 123
 124 static bool is_sibling_call(struct instruction *insn)
 125 {
 126         /*
 127          * Assume only ELF functions can make sibling calls.  This ensures
 128          * sibling call detection consistency between vmlinux.o and individual
 129          * objects.
 130          */
 131         if (!insn->func)
 132                 return false;
 133
 134         /* An indirect jump is either a sibling call or a jump to a table. */
 135         if (insn->type == INSN_JUMP_DYNAMIC)
 136                 return !is_jump_table_jump(insn);
 137
 138         /* add_jump_destinations() sets insn->call_dest for sibling calls. */
 139         return (is_static_jump(insn) && insn->call_dest);
 140 }
 141
 142 /*
 143  * This checks to see if the given function is a "noreturn" function.
 144  *
 145  * For global functions which are outside the scope of this object file, we
 146  * have to keep a manual list of them.
 147  *
 148  * For local functions, we have to detect them manually by simply looking for
 149  * the lack of a return instruction.
 150  */
 151 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 152                                 int recursion)
 153 {
 154         int i;
 155         struct instruction *insn;
 156         bool empty = true;
 157
 158         /*
 159          * Unfortunately these have to be hard coded because the noreturn
 160          * attribute isn't provided in ELF data.
 161          */
 162         static const char * const global_noreturns[] = {
 163                 "__stack_chk_fail",
 164                 "panic",
 165                 "do_exit",
 166                 "do_task_dead",
 167                 "__module_put_and_exit",
 168                 "complete_and_exit",
 169                 "__reiserfs_panic",
 170                 "lbug_with_loc",
 171                 "fortify_panic",
 172                 "usercopy_abort",
 173                 "machine_real_restart",
 174                 "rewind_stack_do_exit",
 175                 "kunit_try_catch_throw",
 176                 "xen_start_kernel",
 177                 "cpu_bringup_and_idle",
 178         };
 179
 180         if (!func)
 181                 return false;
 182
 183         if (func->bind == STB_WEAK)
 184                 return false;
 185
 186         if (func->bind == STB_GLOBAL)
 187                 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
 188                         if (!strcmp(func->name, global_noreturns[i]))
 189                                 return true;
 190
 191         if (!func->len)
 192                 return false;
 193
 194         insn = find_insn(file, func->sec, func->offset);
 195         if (!insn->func)
 196                 return false;
 197
 198         func_for_each_insn(file, func, insn) {
 199                 empty = false;
 200
 201                 if (insn->type == INSN_RETURN)
 202                         return false;
 203         }
 204
 205         if (empty)
 206                 return false;
 207
 208         /*
 209          * A function can have a sibling call instead of a return.  In that
 210          * case, the function's dead-end status depends on whether the target
 211          * of the sibling call returns.
 212          */
 213         func_for_each_insn(file, func, insn) {
 214                 if (is_sibling_call(insn)) {
 215                         struct instruction *dest = insn->jump_dest;
 216
 217                         if (!dest)
 218                                 /* sibling call to another file */
 219                                 return false;
 220
 221                         /* local sibling call */
 222                         if (recursion == 5) {
 223                                 /*
 224                                  * Infinite recursion: two functions have
 225                                  * sibling calls to each other.  This is a very
 226                                  * rare case.  It means they aren't dead ends.
 227                                  */
 228                                 return false;
 229                         }
 230
 231                         return __dead_end_function(file, dest->func, recursion+1);
 232                 }
 233         }
 234
 235         return true;
 236 }
 237
 238 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
 239 {
 240         return __dead_end_function(file, func, 0);
 241 }
 242
 243 static void init_cfi_state(struct cfi_state *cfi)
 244 {
 245         int i;
 246
 247         for (i = 0; i < CFI_NUM_REGS; i++) {
 248                 cfi->regs[i].base = CFI_UNDEFINED;
 249                 cfi->vals[i].base = CFI_UNDEFINED;
 250         }
 251         cfi->cfa.base = CFI_UNDEFINED;
 252         cfi->drap_reg = CFI_UNDEFINED;
 253         cfi->drap_offset = -1;
 254 }
 255
 256 static void init_insn_state(struct insn_state *state, struct section *sec)
 257 {
 258         memset(state, 0, sizeof(*state));
 259         init_cfi_state(&state->cfi);
 260
 261         /*
 262          * We need the full vmlinux for noinstr validation, otherwise we can
 263          * not correctly determine insn->call_dest->sec (external symbols do
 264          * not have a section).
 265          */
 266         if (vmlinux && noinstr && sec)
 267                 state->noinstr = sec->noinstr;
 268 }
 269
 270 /*
 271  * Call the arch-specific instruction decoder for all the instructions and add
 272  * them to the global instruction list.
 273  */
 274 static int decode_instructions(struct objtool_file *file)
 275 {
 276         struct section *sec;
 277         struct symbol *func;
 278         unsigned long offset;
 279         struct instruction *insn;
 280         unsigned long nr_insns = 0;
 281         int ret;
 282
 283         for_each_sec(file, sec) {
 284
 285                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
 286                         continue;
 287
 288                 if (strcmp(sec->name, ".altinstr_replacement") &&
 289                     strcmp(sec->name, ".altinstr_aux") &&
 290                     strncmp(sec->name, ".discard.", 9))
 291                         sec->text = true;
 292
 293                 if (!strcmp(sec->name, ".noinstr.text") ||
 294                     !strcmp(sec->name, ".entry.text"))
 295                         sec->noinstr = true;
 296
 297                 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
 298                         insn = malloc(sizeof(*insn));
 299                         if (!insn) {
 300                                 WARN("malloc failed");
 301                                 return -1;
 302                         }
 303                         memset(insn, 0, sizeof(*insn));
 304                         INIT_LIST_HEAD(&insn->alts);
 305                         INIT_LIST_HEAD(&insn->stack_ops);
 306                         init_cfi_state(&insn->cfi);
 307
 308                         insn->sec = sec;
 309                         insn->offset = offset;
 310
 311                         ret = arch_decode_instruction(file->elf, sec, offset,
 312                                                       sec->sh.sh_size - offset,
 313                                                       &insn->len, &insn->type,
 314                                                       &insn->immediate,
 315                                                       &insn->stack_ops);
 316                         if (ret)
 317                                 goto err;
 318
 319                         hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
 320                         list_add_tail(&insn->list, &file->insn_list);
 321                         nr_insns++;
 322                 }
 323
 324                 list_for_each_entry(func, &sec->symbol_list, list) {
 325                         if (func->type != STT_FUNC || func->alias != func)
 326                                 continue;
 327
 328                         if (!find_insn(file, sec, func->offset)) {
 329                                 WARN("%s(): can't find starting instruction",
 330                                      func->name);
 331                                 return -1;
 332                         }
 333
 334                         sym_for_each_insn(file, func, insn)
 335                                 insn->func = func;
 336                 }
 337         }
 338
 339         if (stats)
 340                 printf("nr_insns: %lu\n", nr_insns);
 341
 342         return 0;
 343
 344 err:
 345         free(insn);
 346         return ret;
 347 }
 348
 349 static struct instruction *find_last_insn(struct objtool_file *file,
 350                                           struct section *sec)
 351 {
 352         struct instruction *insn = NULL;
 353         unsigned int offset;
 354         unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
 355
 356         for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
 357                 insn = find_insn(file, sec, offset);
 358
 359         return insn;
 360 }
 361
 362 /*
 363  * Mark "ud2" instructions and manually annotated dead ends.
 364  */
 365 static int add_dead_ends(struct objtool_file *file)
 366 {
 367         struct section *sec;
 368         struct reloc *reloc;
 369         struct instruction *insn;
 370
 371         /*
 372          * By default, "ud2" is a dead end unless otherwise annotated, because
 373          * GCC 7 inserts it for certain divide-by-zero cases.
 374          */
 375         for_each_insn(file, insn)
 376                 if (insn->type == INSN_BUG)
 377                         insn->dead_end = true;
 378
 379         /*
 380          * Check for manually annotated dead ends.
 381          */
 382         sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
 383         if (!sec)
 384                 goto reachable;
 385
 386         list_for_each_entry(reloc, &sec->reloc_list, list) {
 387                 if (reloc->sym->type != STT_SECTION) {
 388                         WARN("unexpected relocation symbol type in %s", sec->name);
 389                         return -1;
 390                 }
 391                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 392                 if (insn)
 393                         insn = list_prev_entry(insn, list);
 394                 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 395                         insn = find_last_insn(file, reloc->sym->sec);
 396                         if (!insn) {
 397                                 WARN("can't find unreachable insn at %s+0x%" PRIx64,
 398                                      reloc->sym->sec->name, reloc->addend);
 399                                 return -1;
 400                         }
 401                 } else {
 402                         WARN("can't find unreachable insn at %s+0x%" PRIx64,
 403                              reloc->sym->sec->name, reloc->addend);
 404                         return -1;
 405                 }
 406
 407                 insn->dead_end = true;
 408         }
 409
 410 reachable:
 411         /*
 412          * These manually annotated reachable checks are needed for GCC 4.4,
 413          * where the Linux unreachable() macro isn't supported.  In that case
 414          * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
 415          * not a dead end.
 416          */
 417         sec = find_section_by_name(file->elf, ".rela.discard.reachable");
 418         if (!sec)
 419                 return 0;
 420
 421         list_for_each_entry(reloc, &sec->reloc_list, list) {
 422                 if (reloc->sym->type != STT_SECTION) {
 423                         WARN("unexpected relocation symbol type in %s", sec->name);
 424                         return -1;
 425                 }
 426                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 427                 if (insn)
 428                         insn = list_prev_entry(insn, list);
 429                 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 430                         insn = find_last_insn(file, reloc->sym->sec);
 431                         if (!insn) {
 432                                 WARN("can't find reachable insn at %s+0x%" PRIx64,
 433                                      reloc->sym->sec->name, reloc->addend);
 434                                 return -1;
 435                         }
 436                 } else {
 437                         WARN("can't find reachable insn at %s+0x%" PRIx64,
 438                              reloc->sym->sec->name, reloc->addend);
 439                         return -1;
 440                 }
 441
 442                 insn->dead_end = false;
 443         }
 444
 445         return 0;
 446 }
 447
 448 static int create_static_call_sections(struct objtool_file *file)
 449 {
 450         struct section *sec;
 451         struct static_call_site *site;
 452         struct instruction *insn;
 453         struct symbol *key_sym;
 454         char *key_name, *tmp;
 455         int idx;
 456
 457         sec = find_section_by_name(file->elf, ".static_call_sites");
 458         if (sec) {
 459                 INIT_LIST_HEAD(&file->static_call_list);
 460                 WARN("file already has .static_call_sites section, skipping");
 461                 return 0;
 462         }
 463
 464         if (list_empty(&file->static_call_list))
 465                 return 0;
 466
 467         idx = 0;
 468         list_for_each_entry(insn, &file->static_call_list, call_node)
 469                 idx++;
 470
 471         sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
 472                                  sizeof(struct static_call_site), idx);
 473         if (!sec)
 474                 return -1;
 475
 476         idx = 0;
 477         list_for_each_entry(insn, &file->static_call_list, call_node) {
 478
 479                 site = (struct static_call_site *)sec->data->d_buf + idx;
 480                 memset(site, 0, sizeof(struct static_call_site));
 481
 482                 /* populate reloc for 'addr' */
 483                 if (elf_add_reloc_to_insn(file->elf, sec,
 484                                           idx * sizeof(struct static_call_site),
 485                                           R_X86_64_PC32,
 486                                           insn->sec, insn->offset))
 487                         return -1;
 488
 489                 /* find key symbol */
 490                 key_name = strdup(insn->call_dest->name);
 491                 if (!key_name) {
 492                         perror("strdup");
 493                         return -1;
 494                 }
 495                 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
 496                             STATIC_CALL_TRAMP_PREFIX_LEN)) {
 497                         WARN("static_call: trampoline name malformed: %s", key_name);
 498                         return -1;
 499                 }
 500                 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
 501                 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
 502
 503                 key_sym = find_symbol_by_name(file->elf, tmp);
 504                 if (!key_sym) {
 505                         if (!module) {
 506                                 WARN("static_call: can't find static_call_key symbol: %s", tmp);
 507                                 return -1;
 508                         }
 509
 510                         /*
 511                          * For modules(), the key might not be exported, which
 512                          * means the module can make static calls but isn't
 513                          * allowed to change them.
 514                          *
 515                          * In that case we temporarily set the key to be the
 516                          * trampoline address.  This is fixed up in
 517                          * static_call_add_module().
 518                          */
 519                         key_sym = insn->call_dest;
 520                 }
 521                 free(key_name);
 522
 523                 /* populate reloc for 'key' */
 524                 if (elf_add_reloc(file->elf, sec,
 525                                   idx * sizeof(struct static_call_site) + 4,
 526                                   R_X86_64_PC32, key_sym,
 527                                   is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
 528                         return -1;
 529
 530                 idx++;
 531         }
 532
 533         return 0;
 534 }
 535
 536 static int create_mcount_loc_sections(struct objtool_file *file)
 537 {
 538         struct section *sec;
 539         unsigned long *loc;
 540         struct instruction *insn;
 541         int idx;
 542
 543         sec = find_section_by_name(file->elf, "__mcount_loc");
 544         if (sec) {
 545                 INIT_LIST_HEAD(&file->mcount_loc_list);
 546                 WARN("file already has __mcount_loc section, skipping");
 547                 return 0;
 548         }
 549
 550         if (list_empty(&file->mcount_loc_list))
 551                 return 0;
 552
 553         idx = 0;
 554         list_for_each_entry(insn, &file->mcount_loc_list, mcount_loc_node)
 555                 idx++;
 556
 557         sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
 558         if (!sec)
 559                 return -1;
 560
 561         idx = 0;
 562         list_for_each_entry(insn, &file->mcount_loc_list, mcount_loc_node) {
 563
 564                 loc = (unsigned long *)sec->data->d_buf + idx;
 565                 memset(loc, 0, sizeof(unsigned long));
 566
 567                 if (elf_add_reloc_to_insn(file->elf, sec,
 568                                           idx * sizeof(unsigned long),
 569                                           R_X86_64_64,
 570                                           insn->sec, insn->offset))
 571                         return -1;
 572
 573                 idx++;
 574         }
 575
 576         return 0;
 577 }
 578
 579 /*
 580  * Warnings shouldn't be reported for ignored functions.
 581  */
 582 static void add_ignores(struct objtool_file *file)
 583 {
 584         struct instruction *insn;
 585         struct section *sec;
 586         struct symbol *func;
 587         struct reloc *reloc;
 588
 589         sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
 590         if (!sec)
 591                 return;
 592
 593         list_for_each_entry(reloc, &sec->reloc_list, list) {
 594                 switch (reloc->sym->type) {
 595                 case STT_FUNC:
 596                         func = reloc->sym;
 597                         break;
 598
 599                 case STT_SECTION:
 600                         func = find_func_by_offset(reloc->sym->sec, reloc->addend);
 601                         if (!func)
 602                                 continue;
 603                         break;
 604
 605                 default:
 606                         WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
 607                         continue;
 608                 }
 609
 610                 func_for_each_insn(file, func, insn)
 611                         insn->ignore = true;
 612         }
 613 }
 614
 615 /*
 616  * This is a whitelist of functions that is allowed to be called with AC set.
 617  * The list is meant to be minimal and only contains compiler instrumentation
 618  * ABI and a few functions used to implement *_{to,from}_user() functions.
 619  *
 620  * These functions must not directly change AC, but may PUSHF/POPF.
 621  */
 622 static const char *uaccess_safe_builtin[] = {
 623         /* KASAN */
 624         "kasan_report",
 625         "kasan_check_range",
 626         /* KASAN out-of-line */
 627         "__asan_loadN_noabort",
 628         "__asan_load1_noabort",
 629         "__asan_load2_noabort",
 630         "__asan_load4_noabort",
 631         "__asan_load8_noabort",
 632         "__asan_load16_noabort",
 633         "__asan_storeN_noabort",
 634         "__asan_store1_noabort",
 635         "__asan_store2_noabort",
 636         "__asan_store4_noabort",
 637         "__asan_store8_noabort",
 638         "__asan_store16_noabort",
 639         "__kasan_check_read",
 640         "__kasan_check_write",
 641         /* KASAN in-line */
 642         "__asan_report_load_n_noabort",
 643         "__asan_report_load1_noabort",
 644         "__asan_report_load2_noabort",
 645         "__asan_report_load4_noabort",
 646         "__asan_report_load8_noabort",
 647         "__asan_report_load16_noabort",
 648         "__asan_report_store_n_noabort",
 649         "__asan_report_store1_noabort",
 650         "__asan_report_store2_noabort",
 651         "__asan_report_store4_noabort",
 652         "__asan_report_store8_noabort",
 653         "__asan_report_store16_noabort",
 654         /* KCSAN */
 655         "__kcsan_check_access",
 656         "kcsan_found_watchpoint",
 657         "kcsan_setup_watchpoint",
 658         "kcsan_check_scoped_accesses",
 659         "kcsan_disable_current",
 660         "kcsan_enable_current_nowarn",
 661         /* KCSAN/TSAN */
 662         "__tsan_func_entry",
 663         "__tsan_func_exit",
 664         "__tsan_read_range",
 665         "__tsan_write_range",
 666         "__tsan_read1",
 667         "__tsan_read2",
 668         "__tsan_read4",
 669         "__tsan_read8",
 670         "__tsan_read16",
 671         "__tsan_write1",
 672         "__tsan_write2",
 673         "__tsan_write4",
 674         "__tsan_write8",
 675         "__tsan_write16",
 676         "__tsan_read_write1",
 677         "__tsan_read_write2",
 678         "__tsan_read_write4",
 679         "__tsan_read_write8",
 680         "__tsan_read_write16",
 681         "__tsan_atomic8_load",
 682         "__tsan_atomic16_load",
 683         "__tsan_atomic32_load",
 684         "__tsan_atomic64_load",
 685         "__tsan_atomic8_store",
 686         "__tsan_atomic16_store",
 687         "__tsan_atomic32_store",
 688         "__tsan_atomic64_store",
 689         "__tsan_atomic8_exchange",
 690         "__tsan_atomic16_exchange",
 691         "__tsan_atomic32_exchange",
 692         "__tsan_atomic64_exchange",
 693         "__tsan_atomic8_fetch_add",
 694         "__tsan_atomic16_fetch_add",
 695         "__tsan_atomic32_fetch_add",
 696         "__tsan_atomic64_fetch_add",
 697         "__tsan_atomic8_fetch_sub",
 698         "__tsan_atomic16_fetch_sub",
 699         "__tsan_atomic32_fetch_sub",
 700         "__tsan_atomic64_fetch_sub",
 701         "__tsan_atomic8_fetch_and",
 702         "__tsan_atomic16_fetch_and",
 703         "__tsan_atomic32_fetch_and",
 704         "__tsan_atomic64_fetch_and",
 705         "__tsan_atomic8_fetch_or",
 706         "__tsan_atomic16_fetch_or",
 707         "__tsan_atomic32_fetch_or",
 708         "__tsan_atomic64_fetch_or",
 709         "__tsan_atomic8_fetch_xor",
 710         "__tsan_atomic16_fetch_xor",
 711         "__tsan_atomic32_fetch_xor",
 712         "__tsan_atomic64_fetch_xor",
 713         "__tsan_atomic8_fetch_nand",
 714         "__tsan_atomic16_fetch_nand",
 715         "__tsan_atomic32_fetch_nand",
 716         "__tsan_atomic64_fetch_nand",
 717         "__tsan_atomic8_compare_exchange_strong",
 718         "__tsan_atomic16_compare_exchange_strong",
 719         "__tsan_atomic32_compare_exchange_strong",
 720         "__tsan_atomic64_compare_exchange_strong",
 721         "__tsan_atomic8_compare_exchange_weak",
 722         "__tsan_atomic16_compare_exchange_weak",
 723         "__tsan_atomic32_compare_exchange_weak",
 724         "__tsan_atomic64_compare_exchange_weak",
 725         "__tsan_atomic8_compare_exchange_val",
 726         "__tsan_atomic16_compare_exchange_val",
 727         "__tsan_atomic32_compare_exchange_val",
 728         "__tsan_atomic64_compare_exchange_val",
 729         "__tsan_atomic_thread_fence",
 730         "__tsan_atomic_signal_fence",
 731         /* KCOV */
 732         "write_comp_data",
 733         "check_kcov_mode",
 734         "__sanitizer_cov_trace_pc",
 735         "__sanitizer_cov_trace_const_cmp1",
 736         "__sanitizer_cov_trace_const_cmp2",
 737         "__sanitizer_cov_trace_const_cmp4",
 738         "__sanitizer_cov_trace_const_cmp8",
 739         "__sanitizer_cov_trace_cmp1",
 740         "__sanitizer_cov_trace_cmp2",
 741         "__sanitizer_cov_trace_cmp4",
 742         "__sanitizer_cov_trace_cmp8",
 743         "__sanitizer_cov_trace_switch",
 744         /* UBSAN */
 745         "ubsan_type_mismatch_common",
 746         "__ubsan_handle_type_mismatch",
 747         "__ubsan_handle_type_mismatch_v1",
 748         "__ubsan_handle_shift_out_of_bounds",
 749         /* misc */
 750         "csum_partial_copy_generic",
 751         "copy_mc_fragile",
 752         "copy_mc_fragile_handle_tail",
 753         "copy_mc_enhanced_fast_string",
 754         "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
 755         NULL
 756 };
 757
 758 static void add_uaccess_safe(struct objtool_file *file)
 759 {
 760         struct symbol *func;
 761         const char **name;
 762
 763         if (!uaccess)
 764                 return;
 765
 766         for (name = uaccess_safe_builtin; *name; name++) {
 767                 func = find_symbol_by_name(file->elf, *name);
 768                 if (!func)
 769                         continue;
 770
 771                 func->uaccess_safe = true;
 772         }
 773 }
 774
 775 /*
 776  * FIXME: For now, just ignore any alternatives which add retpolines.  This is
 777  * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
 778  * But it at least allows objtool to understand the control flow *around* the
 779  * retpoline.
 780  */
 781 static int add_ignore_alternatives(struct objtool_file *file)
 782 {
 783         struct section *sec;
 784         struct reloc *reloc;
 785         struct instruction *insn;
 786
 787         sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
 788         if (!sec)
 789                 return 0;
 790
 791         list_for_each_entry(reloc, &sec->reloc_list, list) {
 792                 if (reloc->sym->type != STT_SECTION) {
 793                         WARN("unexpected relocation symbol type in %s", sec->name);
 794                         return -1;
 795                 }
 796
 797                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 798                 if (!insn) {
 799                         WARN("bad .discard.ignore_alts entry");
 800                         return -1;
 801                 }
 802
 803                 insn->ignore_alts = true;
 804         }
 805
 806         return 0;
 807 }
 808
 809 __weak bool arch_is_retpoline(struct symbol *sym)
 810 {
 811         return false;
 812 }
 813
 814 #define NEGATIVE_RELOC  ((void *)-1L)
 815
 816 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
 817 {
 818         if (insn->reloc == NEGATIVE_RELOC)
 819                 return NULL;
 820
 821         if (!insn->reloc) {
 822                 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
 823                                                        insn->offset, insn->len);
 824                 if (!insn->reloc) {
 825                         insn->reloc = NEGATIVE_RELOC;
 826                         return NULL;
 827                 }
 828         }
 829
 830         return insn->reloc;
 831 }
 832
 833 static void remove_insn_ops(struct instruction *insn)
 834 {
 835         struct stack_op *op, *tmp;
 836
 837         list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
 838                 list_del(&op->list);
 839                 free(op);
 840         }
 841 }
 842
 843 static void annotate_call_site(struct objtool_file *file,
 844                                struct instruction *insn, bool sibling)
 845 {
 846         struct reloc *reloc = insn_reloc(file, insn);
 847         struct symbol *sym = insn->call_dest;
 848
 849         if (!sym)
 850                 sym = reloc->sym;
 851
 852         /*
 853          * Alternative replacement code is just template code which is
 854          * sometimes copied to the original instruction. For now, don't
 855          * annotate it. (In the future we might consider annotating the
 856          * original instruction if/when it ever makes sense to do so.)
 857          */
 858         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
 859                 return;
 860
 861         if (sym->static_call_tramp) {
 862                 list_add_tail(&insn->call_node, &file->static_call_list);
 863                 return;
 864         }
 865
 866         /*
 867          * Many compilers cannot disable KCOV with a function attribute
 868          * so they need a little help, NOP out any KCOV calls from noinstr
 869          * text.
 870          */
 871         if (insn->sec->noinstr && sym->kcov) {
 872                 if (reloc) {
 873                         reloc->type = R_NONE;
 874                         elf_write_reloc(file->elf, reloc);
 875                 }
 876
 877                 elf_write_insn(file->elf, insn->sec,
 878                                insn->offset, insn->len,
 879                                sibling ? arch_ret_insn(insn->len)
 880                                        : arch_nop_insn(insn->len));
 881
 882                 insn->type = sibling ? INSN_RETURN : INSN_NOP;
 883
 884                 if (sibling) {
 885                         /*
 886                          * We've replaced the tail-call JMP insn by two new
 887                          * insn: RET; INT3, except we only have a single struct
 888                          * insn here. Mark it retpoline_safe to avoid the SLS
 889                          * warning, instead of adding another insn.
 890                          */
 891                         insn->retpoline_safe = true;
 892                 }
 893
 894                 return;
 895         }
 896
 897         if (mcount && sym->fentry) {
 898                 if (sibling)
 899                         WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
 900
 901                 if (reloc) {
 902                         reloc->type = R_NONE;
 903                         elf_write_reloc(file->elf, reloc);
 904                 }
 905
 906                 elf_write_insn(file->elf, insn->sec,
 907                                insn->offset, insn->len,
 908                                arch_nop_insn(insn->len));
 909
 910                 insn->type = INSN_NOP;
 911
 912                 list_add_tail(&insn->mcount_loc_node, &file->mcount_loc_list);
 913                 return;
 914         }
 915 }
 916
 917 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
 918                           struct symbol *dest, bool sibling)
 919 {
 920         insn->call_dest = dest;
 921         if (!dest)
 922                 return;
 923
 924         /*
 925          * Whatever stack impact regular CALLs have, should be undone
 926          * by the RETURN of the called function.
 927          *
 928          * Annotated intra-function calls retain the stack_ops but
 929          * are converted to JUMP, see read_intra_function_calls().
 930          */
 931         remove_insn_ops(insn);
 932
 933         annotate_call_site(file, insn, sibling);
 934 }
 935
 936 /*
 937  * Find the destination instructions for all jumps.
 938  */
 939 static int add_jump_destinations(struct objtool_file *file)
 940 {
 941         struct instruction *insn;
 942         struct reloc *reloc;
 943         struct section *dest_sec;
 944         unsigned long dest_off;
 945
 946         for_each_insn(file, insn) {
 947                 if (!is_static_jump(insn))
 948                         continue;
 949
 950                 reloc = insn_reloc(file, insn);
 951                 if (!reloc) {
 952                         dest_sec = insn->sec;
 953                         dest_off = arch_jump_destination(insn);
 954                 } else if (reloc->sym->type == STT_SECTION) {
 955                         dest_sec = reloc->sym->sec;
 956                         dest_off = arch_dest_reloc_offset(reloc->addend);
 957                 } else if (reloc->sym->retpoline_thunk) {
 958                         /*
 959                          * Retpoline jumps are really dynamic jumps in
 960                          * disguise, so convert them accordingly.
 961                          */
 962                         if (insn->type == INSN_JUMP_UNCONDITIONAL)
 963                                 insn->type = INSN_JUMP_DYNAMIC;
 964                         else
 965                                 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
 966
 967                         list_add_tail(&insn->call_node,
 968                                       &file->retpoline_call_list);
 969
 970                         insn->retpoline_safe = true;
 971                         continue;
 972                 } else if (insn->func) {
 973                         /* internal or external sibling call (with reloc) */
 974                         add_call_dest(file, insn, reloc->sym, true);
 975                         continue;
 976                 } else if (reloc->sym->sec->idx) {
 977                         dest_sec = reloc->sym->sec;
 978                         dest_off = reloc->sym->sym.st_value +
 979                                    arch_dest_reloc_offset(reloc->addend);
 980                 } else {
 981                         /* non-func asm code jumping to another file */
 982                         continue;
 983                 }
 984
 985                 insn->jump_dest = find_insn(file, dest_sec, dest_off);
 986                 if (!insn->jump_dest) {
 987
 988                         /*
 989                          * This is a special case where an alt instruction
 990                          * jumps past the end of the section.  These are
 991                          * handled later in handle_group_alt().
 992                          */
 993                         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
 994                                 continue;
 995
 996                         WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
 997                                   insn->sec, insn->offset, dest_sec->name,
 998                                   dest_off);
 999                         return -1;
1000                 }
1001
1002                 /*
1003                  * Cross-function jump.
1004                  */
1005                 if (insn->func && insn->jump_dest->func &&
1006                     insn->func != insn->jump_dest->func) {
1007
1008                         /*
1009                          * For GCC 8+, create parent/child links for any cold
1010                          * subfunctions.  This is _mostly_ redundant with a
1011                          * similar initialization in read_symbols().
1012                          *
1013                          * If a function has aliases, we want the *first* such
1014                          * function in the symbol table to be the subfunction's
1015                          * parent.  In that case we overwrite the
1016                          * initialization done in read_symbols().
1017                          *
1018                          * However this code can't completely replace the
1019                          * read_symbols() code because this doesn't detect the
1020                          * case where the parent function's only reference to a
1021                          * subfunction is through a jump table.
1022                          */
1023                         if (!strstr(insn->func->name, ".cold") &&
1024                             strstr(insn->jump_dest->func->name, ".cold")) {
1025                                 insn->func->cfunc = insn->jump_dest->func;
1026                                 insn->jump_dest->func->pfunc = insn->func;
1027
1028                         } else if (insn->jump_dest->func->pfunc != insn->func->pfunc &&
1029                                    insn->jump_dest->offset == insn->jump_dest->func->offset) {
1030                                 /* internal sibling call (without reloc) */
1031                                 add_call_dest(file, insn, insn->jump_dest->func, true);
1032                         }
1033                 }
1034         }
1035
1036         return 0;
1037 }
1038
1039 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1040 {
1041         struct symbol *call_dest;
1042
1043         call_dest = find_func_by_offset(sec, offset);
1044         if (!call_dest)
1045                 call_dest = find_symbol_by_offset(sec, offset);
1046
1047         return call_dest;
1048 }
1049
1050 /*
1051  * Find the destination instructions for all calls.
1052  */
1053 static int add_call_destinations(struct objtool_file *file)
1054 {
1055         struct instruction *insn;
1056         unsigned long dest_off;
1057         struct symbol *dest;
1058         struct reloc *reloc;
1059
1060         for_each_insn(file, insn) {
1061                 if (insn->type != INSN_CALL)
1062                         continue;
1063
1064                 reloc = insn_reloc(file, insn);
1065                 if (!reloc) {
1066                         dest_off = arch_jump_destination(insn);
1067                         dest = find_call_destination(insn->sec, dest_off);
1068
1069                         add_call_dest(file, insn, dest, false);
1070
1071                         if (insn->ignore)
1072                                 continue;
1073
1074                         if (!insn->call_dest) {
1075                                 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
1076                                 return -1;
1077                         }
1078
1079                         if (insn->func && insn->call_dest->type != STT_FUNC) {
1080                                 WARN_FUNC("unsupported call to non-function",
1081                                           insn->sec, insn->offset);
1082                                 return -1;
1083                         }
1084
1085                 } else if (reloc->sym->type == STT_SECTION) {
1086                         dest_off = arch_dest_reloc_offset(reloc->addend);
1087                         dest = find_call_destination(reloc->sym->sec, dest_off);
1088                         if (!dest) {
1089                                 WARN_FUNC("can't find call dest symbol at %s+0x%lx",
1090                                           insn->sec, insn->offset,
1091                                           reloc->sym->sec->name,
1092                                           dest_off);
1093                                 return -1;
1094                         }
1095
1096                         add_call_dest(file, insn, dest, false);
1097
1098                 } else if (reloc->sym->retpoline_thunk) {
1099                         /*
1100                          * Retpoline calls are really dynamic calls in
1101                          * disguise, so convert them accordingly.
1102                          */
1103                         insn->type = INSN_CALL_DYNAMIC;
1104                         insn->retpoline_safe = true;
1105
1106                         list_add_tail(&insn->call_node,
1107                                       &file->retpoline_call_list);
1108
1109                         remove_insn_ops(insn);
1110                         continue;
1111
1112                 } else
1113                         add_call_dest(file, insn, reloc->sym, false);
1114         }
1115
1116         return 0;
1117 }
1118
1119 /*
1120  * The .alternatives section requires some extra special care over and above
1121  * other special sections because alternatives are patched in place.
1122  */
1123 static int handle_group_alt(struct objtool_file *file,
1124                             struct special_alt *special_alt,
1125                             struct instruction *orig_insn,
1126                             struct instruction **new_insn)
1127 {
1128         struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
1129         struct alt_group *orig_alt_group, *new_alt_group;
1130         unsigned long dest_off;
1131
1132
1133         orig_alt_group = malloc(sizeof(*orig_alt_group));
1134         if (!orig_alt_group) {
1135                 WARN("malloc failed");
1136                 return -1;
1137         }
1138         orig_alt_group->cfi = calloc(special_alt->orig_len,
1139                                      sizeof(struct cfi_state *));
1140         if (!orig_alt_group->cfi) {
1141                 WARN("calloc failed");
1142                 return -1;
1143         }
1144
1145         last_orig_insn = NULL;
1146         insn = orig_insn;
1147         sec_for_each_insn_from(file, insn) {
1148                 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1149                         break;
1150
1151                 insn->alt_group = orig_alt_group;
1152                 last_orig_insn = insn;
1153         }
1154         orig_alt_group->orig_group = NULL;
1155         orig_alt_group->first_insn = orig_insn;
1156         orig_alt_group->last_insn = last_orig_insn;
1157
1158
1159         new_alt_group = malloc(sizeof(*new_alt_group));
1160         if (!new_alt_group) {
1161                 WARN("malloc failed");
1162                 return -1;
1163         }
1164
1165         if (special_alt->new_len < special_alt->orig_len) {
1166                 /*
1167                  * Insert a fake nop at the end to make the replacement
1168                  * alt_group the same size as the original.  This is needed to
1169                  * allow propagate_alt_cfi() to do its magic.  When the last
1170                  * instruction affects the stack, the instruction after it (the
1171                  * nop) will propagate the new state to the shared CFI array.
1172                  */
1173                 nop = malloc(sizeof(*nop));
1174                 if (!nop) {
1175                         WARN("malloc failed");
1176                         return -1;
1177                 }
1178                 memset(nop, 0, sizeof(*nop));
1179                 INIT_LIST_HEAD(&nop->alts);
1180                 INIT_LIST_HEAD(&nop->stack_ops);
1181                 init_cfi_state(&nop->cfi);
1182
1183                 nop->sec = special_alt->new_sec;
1184                 nop->offset = special_alt->new_off + special_alt->new_len;
1185                 nop->len = special_alt->orig_len - special_alt->new_len;
1186                 nop->type = INSN_NOP;
1187                 nop->func = orig_insn->func;
1188                 nop->alt_group = new_alt_group;
1189                 nop->ignore = orig_insn->ignore_alts;
1190         }
1191
1192         if (!special_alt->new_len) {
1193                 *new_insn = nop;
1194                 goto end;
1195         }
1196
1197         insn = *new_insn;
1198         sec_for_each_insn_from(file, insn) {
1199                 struct reloc *alt_reloc;
1200
1201                 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1202                         break;
1203
1204                 last_new_insn = insn;
1205
1206                 insn->ignore = orig_insn->ignore_alts;
1207                 insn->func = orig_insn->func;
1208                 insn->alt_group = new_alt_group;
1209
1210                 /*
1211                  * Since alternative replacement code is copy/pasted by the
1212                  * kernel after applying relocations, generally such code can't
1213                  * have relative-address relocation references to outside the
1214                  * .altinstr_replacement section, unless the arch's
1215                  * alternatives code can adjust the relative offsets
1216                  * accordingly.
1217                  */
1218                 alt_reloc = insn_reloc(file, insn);
1219                 if (alt_reloc &&
1220                     !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1221
1222                         WARN_FUNC("unsupported relocation in alternatives section",
1223                                   insn->sec, insn->offset);
1224                         return -1;
1225                 }
1226
1227                 if (!is_static_jump(insn))
1228                         continue;
1229
1230                 if (!insn->immediate)
1231                         continue;
1232
1233                 dest_off = arch_jump_destination(insn);
1234                 if (dest_off == special_alt->new_off + special_alt->new_len)
1235                         insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
1236
1237                 if (!insn->jump_dest) {
1238                         WARN_FUNC("can't find alternative jump destination",
1239                                   insn->sec, insn->offset);
1240                         return -1;
1241                 }
1242         }
1243
1244         if (!last_new_insn) {
1245                 WARN_FUNC("can't find last new alternative instruction",
1246                           special_alt->new_sec, special_alt->new_off);
1247                 return -1;
1248         }
1249
1250         if (nop)
1251                 list_add(&nop->list, &last_new_insn->list);
1252 end:
1253         new_alt_group->orig_group = orig_alt_group;
1254         new_alt_group->first_insn = *new_insn;
1255         new_alt_group->last_insn = nop ? : last_new_insn;
1256         new_alt_group->cfi = orig_alt_group->cfi;
1257         return 0;
1258 }
1259
1260 /*
1261  * A jump table entry can either convert a nop to a jump or a jump to a nop.
1262  * If the original instruction is a jump, make the alt entry an effective nop
1263  * by just skipping the original instruction.
1264  */
1265 static int handle_jump_alt(struct objtool_file *file,
1266                            struct special_alt *special_alt,
1267                            struct instruction *orig_insn,
1268                            struct instruction **new_insn)
1269 {
1270         if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1271             orig_insn->type != INSN_NOP) {
1272
1273                 WARN_FUNC("unsupported instruction at jump label",
1274                           orig_insn->sec, orig_insn->offset);
1275                 return -1;
1276         }
1277
1278         if (special_alt->key_addend & 2) {
1279                 struct reloc *reloc = insn_reloc(file, orig_insn);
1280
1281                 if (reloc) {
1282                         reloc->type = R_NONE;
1283                         elf_write_reloc(file->elf, reloc);
1284                 }
1285                 elf_write_insn(file->elf, orig_insn->sec,
1286                                orig_insn->offset, orig_insn->len,
1287                                arch_nop_insn(orig_insn->len));
1288                 orig_insn->type = INSN_NOP;
1289         }
1290
1291         if (orig_insn->type == INSN_NOP) {
1292                 if (orig_insn->len == 2)
1293                         file->jl_nop_short++;
1294                 else
1295                         file->jl_nop_long++;
1296
1297                 return 0;
1298         }
1299
1300         if (orig_insn->len == 2)
1301                 file->jl_short++;
1302         else
1303                 file->jl_long++;
1304
1305         *new_insn = list_next_entry(orig_insn, list);
1306         return 0;
1307 }
1308
1309 /*
1310  * Read all the special sections which have alternate instructions which can be
1311  * patched in or redirected to at runtime.  Each instruction having alternate
1312  * instruction(s) has them added to its insn->alts list, which will be
1313  * traversed in validate_branch().
1314  */
1315 static int add_special_section_alts(struct objtool_file *file)
1316 {
1317         struct list_head special_alts;
1318         struct instruction *orig_insn, *new_insn;
1319         struct special_alt *special_alt, *tmp;
1320         struct alternative *alt;
1321         int ret;
1322
1323         ret = special_get_alts(file->elf, &special_alts);
1324         if (ret)
1325                 return ret;
1326
1327         list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1328
1329                 orig_insn = find_insn(file, special_alt->orig_sec,
1330                                       special_alt->orig_off);
1331                 if (!orig_insn) {
1332                         WARN_FUNC("special: can't find orig instruction",
1333                                   special_alt->orig_sec, special_alt->orig_off);
1334                         ret = -1;
1335                         goto out;
1336                 }
1337
1338                 new_insn = NULL;
1339                 if (!special_alt->group || special_alt->new_len) {
1340                         new_insn = find_insn(file, special_alt->new_sec,
1341                                              special_alt->new_off);
1342                         if (!new_insn) {
1343                                 WARN_FUNC("special: can't find new instruction",
1344                                           special_alt->new_sec,
1345                                           special_alt->new_off);
1346                                 ret = -1;
1347                                 goto out;
1348                         }
1349                 }
1350
1351                 if (special_alt->group) {
1352                         if (!special_alt->orig_len) {
1353                                 WARN_FUNC("empty alternative entry",
1354                                           orig_insn->sec, orig_insn->offset);
1355                                 continue;
1356                         }
1357
1358                         ret = handle_group_alt(file, special_alt, orig_insn,
1359                                                &new_insn);
1360                         if (ret)
1361                                 goto out;
1362                 } else if (special_alt->jump_or_nop) {
1363                         ret = handle_jump_alt(file, special_alt, orig_insn,
1364                                               &new_insn);
1365                         if (ret)
1366                                 goto out;
1367                 }
1368
1369                 alt = malloc(sizeof(*alt));
1370                 if (!alt) {
1371                         WARN("malloc failed");
1372                         ret = -1;
1373                         goto out;
1374                 }
1375
1376                 alt->insn = new_insn;
1377                 alt->skip_orig = special_alt->skip_orig;
1378                 orig_insn->ignore_alts |= special_alt->skip_alt;
1379                 list_add_tail(&alt->list, &orig_insn->alts);
1380
1381                 list_del(&special_alt->list);
1382                 free(special_alt);
1383         }
1384
1385         if (stats) {
1386                 printf("jl\\\tNOP\tJMP\n");
1387                 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1388                 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1389         }
1390
1391 out:
1392         return ret;
1393 }
1394
1395 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
1396                             struct reloc *table)
1397 {
1398         struct reloc *reloc = table;
1399         struct instruction *dest_insn;
1400         struct alternative *alt;
1401         struct symbol *pfunc = insn->func->pfunc;
1402         unsigned int prev_offset = 0;
1403
1404         /*
1405          * Each @reloc is a switch table relocation which points to the target
1406          * instruction.
1407          */
1408         list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
1409
1410                 /* Check for the end of the table: */
1411                 if (reloc != table && reloc->jump_table_start)
1412                         break;
1413
1414                 /* Make sure the table entries are consecutive: */
1415                 if (prev_offset && reloc->offset != prev_offset + 8)
1416                         break;
1417
1418                 /* Detect function pointers from contiguous objects: */
1419                 if (reloc->sym->sec == pfunc->sec &&
1420                     reloc->addend == pfunc->offset)
1421                         break;
1422
1423                 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
1424                 if (!dest_insn)
1425                         break;
1426
1427                 /* Make sure the destination is in the same function: */
1428                 if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
1429                         break;
1430
1431                 alt = malloc(sizeof(*alt));
1432                 if (!alt) {
1433                         WARN("malloc failed");
1434                         return -1;
1435                 }
1436
1437                 alt->insn = dest_insn;
1438                 list_add_tail(&alt->list, &insn->alts);
1439                 prev_offset = reloc->offset;
1440         }
1441
1442         if (!prev_offset) {
1443                 WARN_FUNC("can't find switch jump table",
1444                           insn->sec, insn->offset);
1445                 return -1;
1446         }
1447
1448         return 0;
1449 }
1450
1451 /*
1452  * find_jump_table() - Given a dynamic jump, find the switch jump table
1453  * associated with it.
1454  */
1455 static struct reloc *find_jump_table(struct objtool_file *file,
1456                                       struct symbol *func,
1457                                       struct instruction *insn)
1458 {
1459         struct reloc *table_reloc;
1460         struct instruction *dest_insn, *orig_insn = insn;
1461
1462         /*
1463          * Backward search using the @first_jump_src links, these help avoid
1464          * much of the 'in between' code. Which avoids us getting confused by
1465          * it.
1466          */
1467         for (;
1468              insn && insn->func && insn->func->pfunc == func;
1469              insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
1470
1471                 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
1472                         break;
1473
1474                 /* allow small jumps within the range */
1475                 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
1476                     insn->jump_dest &&
1477                     (insn->jump_dest->offset <= insn->offset ||
1478                      insn->jump_dest->offset > orig_insn->offset))
1479                     break;
1480
1481                 table_reloc = arch_find_switch_table(file, insn);
1482                 if (!table_reloc)
1483                         continue;
1484                 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
1485                 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
1486                         continue;
1487
1488                 return table_reloc;
1489         }
1490
1491         return NULL;
1492 }
1493
1494 /*
1495  * First pass: Mark the head of each jump table so that in the next pass,
1496  * we know when a given jump table ends and the next one starts.
1497  */
1498 static void mark_func_jump_tables(struct objtool_file *file,
1499                                     struct symbol *func)
1500 {
1501         struct instruction *insn, *last = NULL;
1502         struct reloc *reloc;
1503
1504         func_for_each_insn(file, func, insn) {
1505                 if (!last)
1506                         last = insn;
1507
1508                 /*
1509                  * Store back-pointers for unconditional forward jumps such
1510                  * that find_jump_table() can back-track using those and
1511                  * avoid some potentially confusing code.
1512                  */
1513                 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
1514                     insn->offset > last->offset &&
1515                     insn->jump_dest->offset > insn->offset &&
1516                     !insn->jump_dest->first_jump_src) {
1517
1518                         insn->jump_dest->first_jump_src = insn;
1519                         last = insn->jump_dest;
1520                 }
1521
1522                 if (insn->type != INSN_JUMP_DYNAMIC)
1523                         continue;
1524
1525                 reloc = find_jump_table(file, func, insn);
1526                 if (reloc) {
1527                         reloc->jump_table_start = true;
1528                         insn->jump_table = reloc;
1529                 }
1530         }
1531 }
1532
1533 static int add_func_jump_tables(struct objtool_file *file,
1534                                   struct symbol *func)
1535 {
1536         struct instruction *insn;
1537         int ret;
1538
1539         func_for_each_insn(file, func, insn) {
1540                 if (!insn->jump_table)
1541                         continue;
1542
1543                 ret = add_jump_table(file, insn, insn->jump_table);
1544                 if (ret)
1545                         return ret;
1546         }
1547
1548         return 0;
1549 }
1550
1551 /*
1552  * For some switch statements, gcc generates a jump table in the .rodata
1553  * section which contains a list of addresses within the function to jump to.
1554  * This finds these jump tables and adds them to the insn->alts lists.
1555  */
1556 static int add_jump_table_alts(struct objtool_file *file)
1557 {
1558         struct section *sec;
1559         struct symbol *func;
1560         int ret;
1561
1562         if (!file->rodata)
1563                 return 0;
1564
1565         for_each_sec(file, sec) {
1566                 list_for_each_entry(func, &sec->symbol_list, list) {
1567                         if (func->type != STT_FUNC)
1568                                 continue;
1569
1570                         mark_func_jump_tables(file, func);
1571                         ret = add_func_jump_tables(file, func);
1572                         if (ret)
1573                                 return ret;
1574                 }
1575         }
1576
1577         return 0;
1578 }
1579
1580 static void set_func_state(struct cfi_state *state)
1581 {
1582         state->cfa = initial_func_cfi.cfa;
1583         memcpy(&state->regs, &initial_func_cfi.regs,
1584                CFI_NUM_REGS * sizeof(struct cfi_reg));
1585         state->stack_size = initial_func_cfi.cfa.offset;
1586 }
1587
1588 static int read_unwind_hints(struct objtool_file *file)
1589 {
1590         struct section *sec, *relocsec;
1591         struct reloc *reloc;
1592         struct unwind_hint *hint;
1593         struct instruction *insn;
1594         int i;
1595
1596         sec = find_section_by_name(file->elf, ".discard.unwind_hints");
1597         if (!sec)
1598                 return 0;
1599
1600         relocsec = sec->reloc;
1601         if (!relocsec) {
1602                 WARN("missing .rela.discard.unwind_hints section");
1603                 return -1;
1604         }
1605
1606         if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
1607                 WARN("struct unwind_hint size mismatch");
1608                 return -1;
1609         }
1610
1611         file->hints = true;
1612
1613         for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
1614                 hint = (struct unwind_hint *)sec->data->d_buf + i;
1615
1616                 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
1617                 if (!reloc) {
1618                         WARN("can't find reloc for unwind_hints[%d]", i);
1619                         return -1;
1620                 }
1621
1622                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1623                 if (!insn) {
1624                         WARN("can't find insn for unwind_hints[%d]", i);
1625                         return -1;
1626                 }
1627
1628                 insn->hint = true;
1629
1630                 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
1631                         set_func_state(&insn->cfi);
1632                         continue;
1633                 }
1634
1635                 if (arch_decode_hint_reg(insn, hint->sp_reg)) {
1636                         WARN_FUNC("unsupported unwind_hint sp base reg %d",
1637                                   insn->sec, insn->offset, hint->sp_reg);
1638                         return -1;
1639                 }
1640
1641                 insn->cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
1642                 insn->cfi.type = hint->type;
1643                 insn->cfi.end = hint->end;
1644         }
1645
1646         return 0;
1647 }
1648
1649 static int read_retpoline_hints(struct objtool_file *file)
1650 {
1651         struct section *sec;
1652         struct instruction *insn;
1653         struct reloc *reloc;
1654
1655         sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
1656         if (!sec)
1657                 return 0;
1658
1659         list_for_each_entry(reloc, &sec->reloc_list, list) {
1660                 if (reloc->sym->type != STT_SECTION) {
1661                         WARN("unexpected relocation symbol type in %s", sec->name);
1662                         return -1;
1663                 }
1664
1665                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1666                 if (!insn) {
1667                         WARN("bad .discard.retpoline_safe entry");
1668                         return -1;
1669                 }
1670
1671                 if (insn->type != INSN_JUMP_DYNAMIC &&
1672                     insn->type != INSN_CALL_DYNAMIC) {
1673                         WARN_FUNC("retpoline_safe hint not an indirect jump/call",
1674                                   insn->sec, insn->offset);
1675                         return -1;
1676                 }
1677
1678                 insn->retpoline_safe = true;
1679         }
1680
1681         return 0;
1682 }
1683
1684 static int read_instr_hints(struct objtool_file *file)
1685 {
1686         struct section *sec;
1687         struct instruction *insn;
1688         struct reloc *reloc;
1689
1690         sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
1691         if (!sec)
1692                 return 0;
1693
1694         list_for_each_entry(reloc, &sec->reloc_list, list) {
1695                 if (reloc->sym->type != STT_SECTION) {
1696                         WARN("unexpected relocation symbol type in %s", sec->name);
1697                         return -1;
1698                 }
1699
1700                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1701                 if (!insn) {
1702                         WARN("bad .discard.instr_end entry");
1703                         return -1;
1704                 }
1705
1706                 insn->instr--;
1707         }
1708
1709         sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
1710         if (!sec)
1711                 return 0;
1712
1713         list_for_each_entry(reloc, &sec->reloc_list, list) {
1714                 if (reloc->sym->type != STT_SECTION) {
1715                         WARN("unexpected relocation symbol type in %s", sec->name);
1716                         return -1;
1717                 }
1718
1719                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1720                 if (!insn) {
1721                         WARN("bad .discard.instr_begin entry");
1722                         return -1;
1723                 }
1724
1725                 insn->instr++;
1726         }
1727
1728         return 0;
1729 }
1730
1731 static int read_intra_function_calls(struct objtool_file *file)
1732 {
1733         struct instruction *insn;
1734         struct section *sec;
1735         struct reloc *reloc;
1736
1737         sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
1738         if (!sec)
1739                 return 0;
1740
1741         list_for_each_entry(reloc, &sec->reloc_list, list) {
1742                 unsigned long dest_off;
1743
1744                 if (reloc->sym->type != STT_SECTION) {
1745                         WARN("unexpected relocation symbol type in %s",
1746                              sec->name);
1747                         return -1;
1748                 }
1749
1750                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1751                 if (!insn) {
1752                         WARN("bad .discard.intra_function_call entry");
1753                         return -1;
1754                 }
1755
1756                 if (insn->type != INSN_CALL) {
1757                         WARN_FUNC("intra_function_call not a direct call",
1758                                   insn->sec, insn->offset);
1759                         return -1;
1760                 }
1761
1762                 /*
1763                  * Treat intra-function CALLs as JMPs, but with a stack_op.
1764                  * See add_call_destinations(), which strips stack_ops from
1765                  * normal CALLs.
1766                  */
1767                 insn->type = INSN_JUMP_UNCONDITIONAL;
1768
1769                 dest_off = insn->offset + insn->len + insn->immediate;
1770                 insn->jump_dest = find_insn(file, insn->sec, dest_off);
1771                 if (!insn->jump_dest) {
1772                         WARN_FUNC("can't find call dest at %s+0x%lx",
1773                                   insn->sec, insn->offset,
1774                                   insn->sec->name, dest_off);
1775                         return -1;
1776                 }
1777         }
1778
1779         return 0;
1780 }
1781
1782 static int classify_symbols(struct objtool_file *file)
1783 {
1784         struct section *sec;
1785         struct symbol *func;
1786
1787         for_each_sec(file, sec) {
1788                 list_for_each_entry(func, &sec->symbol_list, list) {
1789                         if (func->bind != STB_GLOBAL)
1790                                 continue;
1791
1792                         if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
1793                                      strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
1794                                 func->static_call_tramp = true;
1795
1796                         if (arch_is_retpoline(func))
1797                                 func->retpoline_thunk = true;
1798
1799                         if (!strcmp(func->name, "__fentry__"))
1800                                 func->fentry = true;
1801
1802                         if (!strncmp(func->name, "__sanitizer_cov_", 16))
1803                                 func->kcov = true;
1804                 }
1805         }
1806
1807         return 0;
1808 }
1809
1810 static void mark_rodata(struct objtool_file *file)
1811 {
1812         struct section *sec;
1813         bool found = false;
1814
1815         /*
1816          * Search for the following rodata sections, each of which can
1817          * potentially contain jump tables:
1818          *
1819          * - .rodata: can contain GCC switch tables
1820          * - .rodata.<func>: same, if -fdata-sections is being used
1821          * - .rodata..c_jump_table: contains C annotated jump tables
1822          *
1823          * .rodata.str1.* sections are ignored; they don't contain jump tables.
1824          */
1825         for_each_sec(file, sec) {
1826                 if (!strncmp(sec->name, ".rodata", 7) &&
1827                     !strstr(sec->name, ".str1.")) {
1828                         sec->rodata = true;
1829                         found = true;
1830                 }
1831         }
1832
1833         file->rodata = found;
1834 }
1835
1836 __weak int arch_rewrite_retpolines(struct objtool_file *file)
1837 {
1838         return 0;
1839 }
1840
1841 static int decode_sections(struct objtool_file *file)
1842 {
1843         int ret;
1844
1845         mark_rodata(file);
1846
1847         ret = decode_instructions(file);
1848         if (ret)
1849                 return ret;
1850
1851         ret = add_dead_ends(file);
1852         if (ret)
1853                 return ret;
1854
1855         add_ignores(file);
1856         add_uaccess_safe(file);
1857
1858         ret = add_ignore_alternatives(file);
1859         if (ret)
1860                 return ret;
1861
1862         /*
1863          * Must be before add_{jump_call}_destination.
1864          */
1865         ret = classify_symbols(file);
1866         if (ret)
1867                 return ret;
1868
1869         /*
1870          * Must be before add_special_section_alts() as that depends on
1871          * jump_dest being set.
1872          */
1873         ret = add_jump_destinations(file);
1874         if (ret)
1875                 return ret;
1876
1877         ret = add_special_section_alts(file);
1878         if (ret)
1879                 return ret;
1880
1881         /*
1882          * Must be before add_call_destination(); it changes INSN_CALL to
1883          * INSN_JUMP.
1884          */
1885         ret = read_intra_function_calls(file);
1886         if (ret)
1887                 return ret;
1888
1889         ret = add_call_destinations(file);
1890         if (ret)
1891                 return ret;
1892
1893         ret = add_jump_table_alts(file);
1894         if (ret)
1895                 return ret;
1896
1897         ret = read_unwind_hints(file);
1898         if (ret)
1899                 return ret;
1900
1901         ret = read_retpoline_hints(file);
1902         if (ret)
1903                 return ret;
1904
1905         ret = read_instr_hints(file);
1906         if (ret)
1907                 return ret;
1908
1909         /*
1910          * Must be after add_special_section_alts(), since this will emit
1911          * alternatives. Must be after add_{jump,call}_destination(), since
1912          * those create the call insn lists.
1913          */
1914         ret = arch_rewrite_retpolines(file);
1915         if (ret)
1916                 return ret;
1917
1918         return 0;
1919 }
1920
1921 static bool is_fentry_call(struct instruction *insn)
1922 {
1923         if (insn->type == INSN_CALL &&
1924             insn->call_dest &&
1925             insn->call_dest->fentry)
1926                 return true;
1927
1928         return false;
1929 }
1930
1931 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
1932 {
1933         struct cfi_state *cfi = &state->cfi;
1934         int i;
1935
1936         if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
1937                 return true;
1938
1939         if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
1940                 return true;
1941
1942         if (cfi->stack_size != initial_func_cfi.cfa.offset)
1943                 return true;
1944
1945         for (i = 0; i < CFI_NUM_REGS; i++) {
1946                 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
1947                     cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
1948                         return true;
1949         }
1950
1951         return false;
1952 }
1953
1954 static bool check_reg_frame_pos(const struct cfi_reg *reg,
1955                                 int expected_offset)
1956 {
1957         return reg->base == CFI_CFA &&
1958                reg->offset == expected_offset;
1959 }
1960
1961 static bool has_valid_stack_frame(struct insn_state *state)
1962 {
1963         struct cfi_state *cfi = &state->cfi;
1964
1965         if (cfi->cfa.base == CFI_BP &&
1966             check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
1967             check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
1968                 return true;
1969
1970         if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
1971                 return true;
1972
1973         return false;
1974 }
1975
1976 static int update_cfi_state_regs(struct instruction *insn,
1977                                   struct cfi_state *cfi,
1978                                   struct stack_op *op)
1979 {
1980         struct cfi_reg *cfa = &cfi->cfa;
1981
1982         if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
1983                 return 0;
1984
1985         /* push */
1986         if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
1987                 cfa->offset += 8;
1988
1989         /* pop */
1990         if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
1991                 cfa->offset -= 8;
1992
1993         /* add immediate to sp */
1994         if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
1995             op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
1996                 cfa->offset -= op->src.offset;
1997
1998         return 0;
1999 }
2000
2001 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2002 {
2003         if (arch_callee_saved_reg(reg) &&
2004             cfi->regs[reg].base == CFI_UNDEFINED) {
2005                 cfi->regs[reg].base = base;
2006                 cfi->regs[reg].offset = offset;
2007         }
2008 }
2009
2010 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2011 {
2012         cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2013         cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2014 }
2015
2016 /*
2017  * A note about DRAP stack alignment:
2018  *
2019  * GCC has the concept of a DRAP register, which is used to help keep track of
2020  * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2021  * register.  The typical DRAP pattern is:
2022  *
2023  *   4c 8d 54 24 08             lea    0x8(%rsp),%r10
2024  *   48 83 e4 c0                and    $0xffffffffffffffc0,%rsp
2025  *   41 ff 72 f8                pushq  -0x8(%r10)
2026  *   55                         push   %rbp
2027  *   48 89 e5                   mov    %rsp,%rbp
2028  *                              (more pushes)
2029  *   41 52                      push   %r10
2030  *                              ...
2031  *   41 5a                      pop    %r10
2032  *                              (more pops)
2033  *   5d                         pop    %rbp
2034  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2035  *   c3                         retq
2036  *
2037  * There are some variations in the epilogues, like:
2038  *
2039  *   5b                         pop    %rbx
2040  *   41 5a                      pop    %r10
2041  *   41 5c                      pop    %r12
2042  *   41 5d                      pop    %r13
2043  *   41 5e                      pop    %r14
2044  *   c9                         leaveq
2045  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2046  *   c3                         retq
2047  *
2048  * and:
2049  *
2050  *   4c 8b 55 e8                mov    -0x18(%rbp),%r10
2051  *   48 8b 5d e0                mov    -0x20(%rbp),%rbx
2052  *   4c 8b 65 f0                mov    -0x10(%rbp),%r12
2053  *   4c 8b 6d f8                mov    -0x8(%rbp),%r13
2054  *   c9                         leaveq
2055  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2056  *   c3                         retq
2057  *
2058  * Sometimes r13 is used as the DRAP register, in which case it's saved and
2059  * restored beforehand:
2060  *
2061  *   41 55                      push   %r13
2062  *   4c 8d 6c 24 10             lea    0x10(%rsp),%r13
2063  *   48 83 e4 f0                and    $0xfffffffffffffff0,%rsp
2064  *                              ...
2065  *   49 8d 65 f0                lea    -0x10(%r13),%rsp
2066  *   41 5d                      pop    %r13
2067  *   c3                         retq
2068  */
2069 static int update_cfi_state(struct instruction *insn,
2070                             struct instruction *next_insn,
2071                             struct cfi_state *cfi, struct stack_op *op)
2072 {
2073         struct cfi_reg *cfa = &cfi->cfa;
2074         struct cfi_reg *regs = cfi->regs;
2075
2076         /* stack operations don't make sense with an undefined CFA */
2077         if (cfa->base == CFI_UNDEFINED) {
2078                 if (insn->func) {
2079                         WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2080                         return -1;
2081                 }
2082                 return 0;
2083         }
2084
2085         if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2086             cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2087                 return update_cfi_state_regs(insn, cfi, op);
2088
2089         switch (op->dest.type) {
2090
2091         case OP_DEST_REG:
2092                 switch (op->src.type) {
2093
2094                 case OP_SRC_REG:
2095                         if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2096                             cfa->base == CFI_SP &&
2097                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2098
2099                                 /* mov %rsp, %rbp */
2100                                 cfa->base = op->dest.reg;
2101                                 cfi->bp_scratch = false;
2102                         }
2103
2104                         else if (op->src.reg == CFI_SP &&
2105                                  op->dest.reg == CFI_BP && cfi->drap) {
2106
2107                                 /* drap: mov %rsp, %rbp */
2108                                 regs[CFI_BP].base = CFI_BP;
2109                                 regs[CFI_BP].offset = -cfi->stack_size;
2110                                 cfi->bp_scratch = false;
2111                         }
2112
2113                         else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2114
2115                                 /*
2116                                  * mov %rsp, %reg
2117                                  *
2118                                  * This is needed for the rare case where GCC
2119                                  * does:
2120                                  *
2121                                  *   mov    %rsp, %rax
2122                                  *   ...
2123                                  *   mov    %rax, %rsp
2124                                  */
2125                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2126                                 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2127                         }
2128
2129                         else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2130                                  (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2131
2132                                 /*
2133                                  * mov %rbp, %rsp
2134                                  *
2135                                  * Restore the original stack pointer (Clang).
2136                                  */
2137                                 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2138                         }
2139
2140                         else if (op->dest.reg == cfa->base) {
2141
2142                                 /* mov %reg, %rsp */
2143                                 if (cfa->base == CFI_SP &&
2144                                     cfi->vals[op->src.reg].base == CFI_CFA) {
2145
2146                                         /*
2147                                          * This is needed for the rare case
2148                                          * where GCC does something dumb like:
2149                                          *
2150                                          *   lea    0x8(%rsp), %rcx
2151                                          *   ...
2152                                          *   mov    %rcx, %rsp
2153                                          */
2154                                         cfa->offset = -cfi->vals[op->src.reg].offset;
2155                                         cfi->stack_size = cfa->offset;
2156
2157                                 } else if (cfa->base == CFI_SP &&
2158                                            cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2159                                            cfi->vals[op->src.reg].offset == cfa->offset) {
2160
2161                                         /*
2162                                          * Stack swizzle:
2163                                          *
2164                                          * 1: mov %rsp, (%[tos])
2165                                          * 2: mov %[tos], %rsp
2166                                          *    ...
2167                                          * 3: pop %rsp
2168                                          *
2169                                          * Where:
2170                                          *
2171                                          * 1 - places a pointer to the previous
2172                                          *     stack at the Top-of-Stack of the
2173                                          *     new stack.
2174                                          *
2175                                          * 2 - switches to the new stack.
2176                                          *
2177                                          * 3 - pops the Top-of-Stack to restore
2178                                          *     the original stack.
2179                                          *
2180                                          * Note: we set base to SP_INDIRECT
2181                                          * here and preserve offset. Therefore
2182                                          * when the unwinder reaches ToS it
2183                                          * will dereference SP and then add the
2184                                          * offset to find the next frame, IOW:
2185                                          * (%rsp) + offset.
2186                                          */
2187                                         cfa->base = CFI_SP_INDIRECT;
2188
2189                                 } else {
2190                                         cfa->base = CFI_UNDEFINED;
2191                                         cfa->offset = 0;
2192                                 }
2193                         }
2194
2195                         else if (op->dest.reg == CFI_SP &&
2196                                  cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2197                                  cfi->vals[op->src.reg].offset == cfa->offset) {
2198
2199                                 /*
2200                                  * The same stack swizzle case 2) as above. But
2201                                  * because we can't change cfa->base, case 3)
2202                                  * will become a regular POP. Pretend we're a
2203                                  * PUSH so things don't go unbalanced.
2204                                  */
2205                                 cfi->stack_size += 8;
2206                         }
2207
2208
2209                         break;
2210
2211                 case OP_SRC_ADD:
2212                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2213
2214                                 /* add imm, %rsp */
2215                                 cfi->stack_size -= op->src.offset;
2216                                 if (cfa->base == CFI_SP)
2217                                         cfa->offset -= op->src.offset;
2218                                 break;
2219                         }
2220
2221                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2222
2223                                 /* lea disp(%rbp), %rsp */
2224                                 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2225                                 break;
2226                         }
2227
2228                         if (!cfi->drap && op->src.reg == CFI_SP &&
2229                             op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2230                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset + op->src.offset)) {
2231
2232                                 /* lea disp(%rsp), %rbp */
2233                                 cfa->base = CFI_BP;
2234                                 cfa->offset -= op->src.offset;
2235                                 cfi->bp_scratch = false;
2236                                 break;
2237                         }
2238
2239                         if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2240
2241                                 /* drap: lea disp(%rsp), %drap */
2242                                 cfi->drap_reg = op->dest.reg;
2243
2244                                 /*
2245                                  * lea disp(%rsp), %reg
2246                                  *
2247                                  * This is needed for the rare case where GCC
2248                                  * does something dumb like:
2249                                  *
2250                                  *   lea    0x8(%rsp), %rcx
2251                                  *   ...
2252                                  *   mov    %rcx, %rsp
2253                                  */
2254                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2255                                 cfi->vals[op->dest.reg].offset = \
2256                                         -cfi->stack_size + op->src.offset;
2257
2258                                 break;
2259                         }
2260
2261                         if (cfi->drap && op->dest.reg == CFI_SP &&
2262                             op->src.reg == cfi->drap_reg) {
2263
2264                                  /* drap: lea disp(%drap), %rsp */
2265                                 cfa->base = CFI_SP;
2266                                 cfa->offset = cfi->stack_size = -op->src.offset;
2267                                 cfi->drap_reg = CFI_UNDEFINED;
2268                                 cfi->drap = false;
2269                                 break;
2270                         }
2271
2272                         if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2273                                 WARN_FUNC("unsupported stack register modification",
2274                                           insn->sec, insn->offset);
2275                                 return -1;
2276                         }
2277
2278                         break;
2279
2280                 case OP_SRC_AND:
2281                         if (op->dest.reg != CFI_SP ||
2282                             (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2283                             (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2284                                 WARN_FUNC("unsupported stack pointer realignment",
2285                                           insn->sec, insn->offset);
2286                                 return -1;
2287                         }
2288
2289                         if (cfi->drap_reg != CFI_UNDEFINED) {
2290                                 /* drap: and imm, %rsp */
2291                                 cfa->base = cfi->drap_reg;
2292                                 cfa->offset = cfi->stack_size = 0;
2293                                 cfi->drap = true;
2294                         }
2295
2296                         /*
2297                          * Older versions of GCC (4.8ish) realign the stack
2298                          * without DRAP, with a frame pointer.
2299                          */
2300
2301                         break;
2302
2303                 case OP_SRC_POP:
2304                 case OP_SRC_POPF:
2305                         if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2306
2307                                 /* pop %rsp; # restore from a stack swizzle */
2308                                 cfa->base = CFI_SP;
2309                                 break;
2310                         }
2311
2312                         if (!cfi->drap && op->dest.reg == cfa->base) {
2313
2314                                 /* pop %rbp */
2315                                 cfa->base = CFI_SP;
2316                         }
2317
2318                         if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2319                             op->dest.reg == cfi->drap_reg &&
2320                             cfi->drap_offset == -cfi->stack_size) {
2321
2322                                 /* drap: pop %drap */
2323                                 cfa->base = cfi->drap_reg;
2324                                 cfa->offset = 0;
2325                                 cfi->drap_offset = -1;
2326
2327                         } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
2328
2329                                 /* pop %reg */
2330                                 restore_reg(cfi, op->dest.reg);
2331                         }
2332
2333                         cfi->stack_size -= 8;
2334                         if (cfa->base == CFI_SP)
2335                                 cfa->offset -= 8;
2336
2337                         break;
2338
2339                 case OP_SRC_REG_INDIRECT:
2340                         if (!cfi->drap && op->dest.reg == cfa->base &&
2341                             op->dest.reg == CFI_BP) {
2342
2343                                 /* mov disp(%rsp), %rbp */
2344                                 cfa->base = CFI_SP;
2345                                 cfa->offset = cfi->stack_size;
2346                         }
2347
2348                         if (cfi->drap && op->src.reg == CFI_BP &&
2349                             op->src.offset == cfi->drap_offset) {
2350
2351                                 /* drap: mov disp(%rbp), %drap */
2352                                 cfa->base = cfi->drap_reg;
2353                                 cfa->offset = 0;
2354                                 cfi->drap_offset = -1;
2355                         }
2356
2357                         if (cfi->drap && op->src.reg == CFI_BP &&
2358                             op->src.offset == regs[op->dest.reg].offset) {
2359
2360                                 /* drap: mov disp(%rbp), %reg */
2361                                 restore_reg(cfi, op->dest.reg);
2362
2363                         } else if (op->src.reg == cfa->base &&
2364                             op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
2365
2366                                 /* mov disp(%rbp), %reg */
2367                                 /* mov disp(%rsp), %reg */
2368                                 restore_reg(cfi, op->dest.reg);
2369
2370                         } else if (op->src.reg == CFI_SP &&
2371                                    op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
2372
2373                                 /* mov disp(%rsp), %reg */
2374                                 restore_reg(cfi, op->dest.reg);
2375                         }
2376
2377                         break;
2378
2379                 default:
2380                         WARN_FUNC("unknown stack-related instruction",
2381                                   insn->sec, insn->offset);
2382                         return -1;
2383                 }
2384
2385                 break;
2386
2387         case OP_DEST_PUSH:
2388         case OP_DEST_PUSHF:
2389                 cfi->stack_size += 8;
2390                 if (cfa->base == CFI_SP)
2391                         cfa->offset += 8;
2392
2393                 if (op->src.type != OP_SRC_REG)
2394                         break;
2395
2396                 if (cfi->drap) {
2397                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2398
2399                                 /* drap: push %drap */
2400                                 cfa->base = CFI_BP_INDIRECT;
2401                                 cfa->offset = -cfi->stack_size;
2402
2403                                 /* save drap so we know when to restore it */
2404                                 cfi->drap_offset = -cfi->stack_size;
2405
2406                         } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
2407
2408                                 /* drap: push %rbp */
2409                                 cfi->stack_size = 0;
2410
2411                         } else {
2412
2413                                 /* drap: push %reg */
2414                                 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
2415                         }
2416
2417                 } else {
2418
2419                         /* push %reg */
2420                         save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
2421                 }
2422
2423                 /* detect when asm code uses rbp as a scratch register */
2424                 if (!no_fp && insn->func && op->src.reg == CFI_BP &&
2425                     cfa->base != CFI_BP)
2426                         cfi->bp_scratch = true;
2427                 break;
2428
2429         case OP_DEST_REG_INDIRECT:
2430
2431                 if (cfi->drap) {
2432                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2433
2434                                 /* drap: mov %drap, disp(%rbp) */
2435                                 cfa->base = CFI_BP_INDIRECT;
2436                                 cfa->offset = op->dest.offset;
2437
2438                                 /* save drap offset so we know when to restore it */
2439                                 cfi->drap_offset = op->dest.offset;
2440                         } else {
2441
2442                                 /* drap: mov reg, disp(%rbp) */
2443                                 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
2444                         }
2445
2446                 } else if (op->dest.reg == cfa->base) {
2447
2448                         /* mov reg, disp(%rbp) */
2449                         /* mov reg, disp(%rsp) */
2450                         save_reg(cfi, op->src.reg, CFI_CFA,
2451                                  op->dest.offset - cfi->cfa.offset);
2452
2453                 } else if (op->dest.reg == CFI_SP) {
2454
2455                         /* mov reg, disp(%rsp) */
2456                         save_reg(cfi, op->src.reg, CFI_CFA,
2457                                  op->dest.offset - cfi->stack_size);
2458
2459                 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
2460
2461                         /* mov %rsp, (%reg); # setup a stack swizzle. */
2462                         cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
2463                         cfi->vals[op->dest.reg].offset = cfa->offset;
2464                 }
2465
2466                 break;
2467
2468         case OP_DEST_MEM:
2469                 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
2470                         WARN_FUNC("unknown stack-related memory operation",
2471                                   insn->sec, insn->offset);
2472                         return -1;
2473                 }
2474
2475                 /* pop mem */
2476                 cfi->stack_size -= 8;
2477                 if (cfa->base == CFI_SP)
2478                         cfa->offset -= 8;
2479
2480                 break;
2481
2482         default:
2483                 WARN_FUNC("unknown stack-related instruction",
2484                           insn->sec, insn->offset);
2485                 return -1;
2486         }
2487
2488         return 0;
2489 }
2490
2491 /*
2492  * The stack layouts of alternatives instructions can sometimes diverge when
2493  * they have stack modifications.  That's fine as long as the potential stack
2494  * layouts don't conflict at any given potential instruction boundary.
2495  *
2496  * Flatten the CFIs of the different alternative code streams (both original
2497  * and replacement) into a single shared CFI array which can be used to detect
2498  * conflicts and nicely feed a linear array of ORC entries to the unwinder.
2499  */
2500 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
2501 {
2502         struct cfi_state **alt_cfi;
2503         int group_off;
2504
2505         if (!insn->alt_group)
2506                 return 0;
2507
2508         alt_cfi = insn->alt_group->cfi;
2509         group_off = insn->offset - insn->alt_group->first_insn->offset;
2510
2511         if (!alt_cfi[group_off]) {
2512                 alt_cfi[group_off] = &insn->cfi;
2513         } else {
2514                 if (memcmp(alt_cfi[group_off], &insn->cfi, sizeof(struct cfi_state))) {
2515                         WARN_FUNC("stack layout conflict in alternatives",
2516                                   insn->sec, insn->offset);
2517                         return -1;
2518                 }
2519         }
2520
2521         return 0;
2522 }
2523
2524 static int handle_insn_ops(struct instruction *insn,
2525                            struct instruction *next_insn,
2526                            struct insn_state *state)
2527 {
2528         struct stack_op *op;
2529
2530         list_for_each_entry(op, &insn->stack_ops, list) {
2531
2532                 if (update_cfi_state(insn, next_insn, &state->cfi, op))
2533                         return 1;
2534
2535                 if (!insn->alt_group)
2536                         continue;
2537
2538                 if (op->dest.type == OP_DEST_PUSHF) {
2539                         if (!state->uaccess_stack) {
2540                                 state->uaccess_stack = 1;
2541                         } else if (state->uaccess_stack >> 31) {
2542                                 WARN_FUNC("PUSHF stack exhausted",
2543                                           insn->sec, insn->offset);
2544                                 return 1;
2545                         }
2546                         state->uaccess_stack <<= 1;
2547                         state->uaccess_stack  |= state->uaccess;
2548                 }
2549
2550                 if (op->src.type == OP_SRC_POPF) {
2551                         if (state->uaccess_stack) {
2552                                 state->uaccess = state->uaccess_stack & 1;
2553                                 state->uaccess_stack >>= 1;
2554                                 if (state->uaccess_stack == 1)
2555                                         state->uaccess_stack = 0;
2556                         }
2557                 }
2558         }
2559
2560         return 0;
2561 }
2562
2563 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
2564 {
2565         struct cfi_state *cfi1 = &insn->cfi;
2566         int i;
2567
2568         if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
2569
2570                 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
2571                           insn->sec, insn->offset,
2572                           cfi1->cfa.base, cfi1->cfa.offset,
2573                           cfi2->cfa.base, cfi2->cfa.offset);
2574
2575         } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
2576                 for (i = 0; i < CFI_NUM_REGS; i++) {
2577                         if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
2578                                     sizeof(struct cfi_reg)))
2579                                 continue;
2580
2581                         WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
2582                                   insn->sec, insn->offset,
2583                                   i, cfi1->regs[i].base, cfi1->regs[i].offset,
2584                                   i, cfi2->regs[i].base, cfi2->regs[i].offset);
2585                         break;
2586                 }
2587
2588         } else if (cfi1->type != cfi2->type) {
2589
2590                 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
2591                           insn->sec, insn->offset, cfi1->type, cfi2->type);
2592
2593         } else if (cfi1->drap != cfi2->drap ||
2594                    (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
2595                    (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
2596
2597                 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
2598                           insn->sec, insn->offset,
2599                           cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
2600                           cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
2601
2602         } else
2603                 return true;
2604
2605         return false;
2606 }
2607
2608 static inline bool func_uaccess_safe(struct symbol *func)
2609 {
2610         if (func)
2611                 return func->uaccess_safe;
2612
2613         return false;
2614 }
2615
2616 static inline const char *call_dest_name(struct instruction *insn)
2617 {
2618         if (insn->call_dest)
2619                 return insn->call_dest->name;
2620
2621         return "{dynamic}";
2622 }
2623
2624 static inline bool noinstr_call_dest(struct symbol *func)
2625 {
2626         /*
2627          * We can't deal with indirect function calls at present;
2628          * assume they're instrumented.
2629          */
2630         if (!func)
2631                 return false;
2632
2633         /*
2634          * If the symbol is from a noinstr section; we good.
2635          */
2636         if (func->sec->noinstr)
2637                 return true;
2638
2639         /*
2640          * The __ubsan_handle_*() calls are like WARN(), they only happen when
2641          * something 'BAD' happened. At the risk of taking the machine down,
2642          * let them proceed to get the message out.
2643          */
2644         if (!strncmp(func->name, "__ubsan_handle_", 15))
2645                 return true;
2646
2647         return false;
2648 }
2649
2650 static int validate_call(struct instruction *insn, struct insn_state *state)
2651 {
2652         if (state->noinstr && state->instr <= 0 &&
2653             !noinstr_call_dest(insn->call_dest)) {
2654                 WARN_FUNC("call to %s() leaves .noinstr.text section",
2655                                 insn->sec, insn->offset, call_dest_name(insn));
2656                 return 1;
2657         }
2658
2659         if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
2660                 WARN_FUNC("call to %s() with UACCESS enabled",
2661                                 insn->sec, insn->offset, call_dest_name(insn));
2662                 return 1;
2663         }
2664
2665         if (state->df) {
2666                 WARN_FUNC("call to %s() with DF set",
2667                                 insn->sec, insn->offset, call_dest_name(insn));
2668                 return 1;
2669         }
2670
2671         return 0;
2672 }
2673
2674 static int validate_sibling_call(struct instruction *insn, struct insn_state *state)
2675 {
2676         if (has_modified_stack_frame(insn, state)) {
2677                 WARN_FUNC("sibling call from callable instruction with modified stack frame",
2678                                 insn->sec, insn->offset);
2679                 return 1;
2680         }
2681
2682         return validate_call(insn, state);
2683 }
2684
2685 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
2686 {
2687         if (state->noinstr && state->instr > 0) {
2688                 WARN_FUNC("return with instrumentation enabled",
2689                           insn->sec, insn->offset);
2690                 return 1;
2691         }
2692
2693         if (state->uaccess && !func_uaccess_safe(func)) {
2694                 WARN_FUNC("return with UACCESS enabled",
2695                           insn->sec, insn->offset);
2696                 return 1;
2697         }
2698
2699         if (!state->uaccess && func_uaccess_safe(func)) {
2700                 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
2701                           insn->sec, insn->offset);
2702                 return 1;
2703         }
2704
2705         if (state->df) {
2706                 WARN_FUNC("return with DF set",
2707                           insn->sec, insn->offset);
2708                 return 1;
2709         }
2710
2711         if (func && has_modified_stack_frame(insn, state)) {
2712                 WARN_FUNC("return with modified stack frame",
2713                           insn->sec, insn->offset);
2714                 return 1;
2715         }
2716
2717         if (state->cfi.bp_scratch) {
2718                 WARN_FUNC("BP used as a scratch register",
2719                           insn->sec, insn->offset);
2720                 return 1;
2721         }
2722
2723         return 0;
2724 }
2725
2726 static struct instruction *next_insn_to_validate(struct objtool_file *file,
2727                                                  struct instruction *insn)
2728 {
2729         struct alt_group *alt_group = insn->alt_group;
2730
2731         /*
2732          * Simulate the fact that alternatives are patched in-place.  When the
2733          * end of a replacement alt_group is reached, redirect objtool flow to
2734          * the end of the original alt_group.
2735          */
2736         if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
2737                 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
2738
2739         return next_insn_same_sec(file, insn);
2740 }
2741
2742 /*
2743  * Follow the branch starting at the given instruction, and recursively follow
2744  * any other branches (jumps).  Meanwhile, track the frame pointer state at
2745  * each instruction and validate all the rules described in
2746  * tools/objtool/Documentation/stack-validation.txt.
2747  */
2748 static int validate_branch(struct objtool_file *file, struct symbol *func,
2749                            struct instruction *insn, struct insn_state state)
2750 {
2751         struct alternative *alt;
2752         struct instruction *next_insn;
2753         struct section *sec;
2754         u8 visited;
2755         int ret;
2756
2757         sec = insn->sec;
2758
2759         while (1) {
2760                 next_insn = next_insn_to_validate(file, insn);
2761
2762                 if (file->c_file && func && insn->func && func != insn->func->pfunc) {
2763                         WARN("%s() falls through to next function %s()",
2764                              func->name, insn->func->name);
2765                         return 1;
2766                 }
2767
2768                 if (func && insn->ignore) {
2769                         WARN_FUNC("BUG: why am I validating an ignored function?",
2770                                   sec, insn->offset);
2771                         return 1;
2772                 }
2773
2774                 visited = 1 << state.uaccess;
2775                 if (insn->visited) {
2776                         if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
2777                                 return 1;
2778
2779                         if (insn->visited & visited)
2780                                 return 0;
2781                 }
2782
2783                 if (state.noinstr)
2784                         state.instr += insn->instr;
2785
2786                 if (insn->hint)
2787                         state.cfi = insn->cfi;
2788                 else
2789                         insn->cfi = state.cfi;
2790
2791                 insn->visited |= visited;
2792
2793                 if (propagate_alt_cfi(file, insn))
2794                         return 1;
2795
2796                 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
2797                         bool skip_orig = false;
2798
2799                         list_for_each_entry(alt, &insn->alts, list) {
2800                                 if (alt->skip_orig)
2801                                         skip_orig = true;
2802
2803                                 ret = validate_branch(file, func, alt->insn, state);
2804                                 if (ret) {
2805                                         if (backtrace)
2806                                                 BT_FUNC("(alt)", insn);
2807                                         return ret;
2808                                 }
2809                         }
2810
2811                         if (skip_orig)
2812                                 return 0;
2813                 }
2814
2815                 if (handle_insn_ops(insn, next_insn, &state))
2816                         return 1;
2817
2818                 switch (insn->type) {
2819
2820                 case INSN_RETURN:
2821                         if (next_insn && next_insn->type == INSN_TRAP) {
2822                                 next_insn->ignore = true;
2823                         } else if (sls && !insn->retpoline_safe) {
2824                                 WARN_FUNC("missing int3 after ret",
2825                                           insn->sec, insn->offset);
2826                         }
2827                         return validate_return(func, insn, &state);
2828
2829                 case INSN_CALL:
2830                 case INSN_CALL_DYNAMIC:
2831                         ret = validate_call(insn, &state);
2832                         if (ret)
2833                                 return ret;
2834
2835                         if (!no_fp && func && !is_fentry_call(insn) &&
2836                             !has_valid_stack_frame(&state)) {
2837                                 WARN_FUNC("call without frame pointer save/setup",
2838                                           sec, insn->offset);
2839                                 return 1;
2840                         }
2841
2842                         if (dead_end_function(file, insn->call_dest))
2843                                 return 0;
2844
2845                         break;
2846
2847                 case INSN_JUMP_CONDITIONAL:
2848                 case INSN_JUMP_UNCONDITIONAL:
2849                         if (is_sibling_call(insn)) {
2850                                 ret = validate_sibling_call(insn, &state);
2851                                 if (ret)
2852                                         return ret;
2853
2854                         } else if (insn->jump_dest) {
2855                                 ret = validate_branch(file, func,
2856                                                       insn->jump_dest, state);
2857                                 if (ret) {
2858                                         if (backtrace)
2859                                                 BT_FUNC("(branch)", insn);
2860                                         return ret;
2861                                 }
2862                         }
2863
2864                         if (insn->type == INSN_JUMP_UNCONDITIONAL)
2865                                 return 0;
2866
2867                         break;
2868
2869                 case INSN_JUMP_DYNAMIC:
2870                         if (next_insn && next_insn->type == INSN_TRAP) {
2871                                 next_insn->ignore = true;
2872                         } else if (sls && !insn->retpoline_safe) {
2873                                 WARN_FUNC("missing int3 after indirect jump",
2874                                           insn->sec, insn->offset);
2875                         }
2876
2877                         /* fallthrough */
2878                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
2879                         if (is_sibling_call(insn)) {
2880                                 ret = validate_sibling_call(insn, &state);
2881                                 if (ret)
2882                                         return ret;
2883                         }
2884
2885                         if (insn->type == INSN_JUMP_DYNAMIC)
2886                                 return 0;
2887
2888                         break;
2889
2890                 case INSN_CONTEXT_SWITCH:
2891                         if (func && (!next_insn || !next_insn->hint)) {
2892                                 WARN_FUNC("unsupported instruction in callable function",
2893                                           sec, insn->offset);
2894                                 return 1;
2895                         }
2896                         return 0;
2897
2898                 case INSN_STAC:
2899                         if (state.uaccess) {
2900                                 WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
2901                                 return 1;
2902                         }
2903
2904                         state.uaccess = true;
2905                         break;
2906
2907                 case INSN_CLAC:
2908                         if (!state.uaccess && func) {
2909                                 WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
2910                                 return 1;
2911                         }
2912
2913                         if (func_uaccess_safe(func) && !state.uaccess_stack) {
2914                                 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
2915                                 return 1;
2916                         }
2917
2918                         state.uaccess = false;
2919                         break;
2920
2921                 case INSN_STD:
2922                         if (state.df) {
2923                                 WARN_FUNC("recursive STD", sec, insn->offset);
2924                                 return 1;
2925                         }
2926
2927                         state.df = true;
2928                         break;
2929
2930                 case INSN_CLD:
2931                         if (!state.df && func) {
2932                                 WARN_FUNC("redundant CLD", sec, insn->offset);
2933                                 return 1;
2934                         }
2935
2936                         state.df = false;
2937                         break;
2938
2939                 default:
2940                         break;
2941                 }
2942
2943                 if (insn->dead_end)
2944                         return 0;
2945
2946                 if (!next_insn) {
2947                         if (state.cfi.cfa.base == CFI_UNDEFINED)
2948                                 return 0;
2949                         WARN("%s: unexpected end of section", sec->name);
2950                         return 1;
2951                 }
2952
2953                 insn = next_insn;
2954         }
2955
2956         return 0;
2957 }
2958
2959 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
2960 {
2961         struct instruction *insn;
2962         struct insn_state state;
2963         int ret, warnings = 0;
2964
2965         if (!file->hints)
2966                 return 0;
2967
2968         init_insn_state(&state, sec);
2969
2970         if (sec) {
2971                 insn = find_insn(file, sec, 0);
2972                 if (!insn)
2973                         return 0;
2974         } else {
2975                 insn = list_first_entry(&file->insn_list, typeof(*insn), list);
2976         }
2977
2978         while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
2979                 if (insn->hint && !insn->visited) {
2980                         ret = validate_branch(file, insn->func, insn, state);
2981                         if (ret && backtrace)
2982                                 BT_FUNC("<=== (hint)", insn);
2983                         warnings += ret;
2984                 }
2985
2986                 insn = list_next_entry(insn, list);
2987         }
2988
2989         return warnings;
2990 }
2991
2992 static int validate_retpoline(struct objtool_file *file)
2993 {
2994         struct instruction *insn;
2995         int warnings = 0;
2996
2997         for_each_insn(file, insn) {
2998                 if (insn->type != INSN_JUMP_DYNAMIC &&
2999                     insn->type != INSN_CALL_DYNAMIC)
3000                         continue;
3001
3002                 if (insn->retpoline_safe)
3003                         continue;
3004
3005                 /*
3006                  * .init.text code is ran before userspace and thus doesn't
3007                  * strictly need retpolines, except for modules which are
3008                  * loaded late, they very much do need retpoline in their
3009                  * .init.text
3010                  */
3011                 if (!strcmp(insn->sec->name, ".init.text") && !module)
3012                         continue;
3013
3014                 WARN_FUNC("indirect %s found in RETPOLINE build",
3015                           insn->sec, insn->offset,
3016                           insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3017
3018                 warnings++;
3019         }
3020
3021         return warnings;
3022 }
3023
3024 static bool is_kasan_insn(struct instruction *insn)
3025 {
3026         return (insn->type == INSN_CALL &&
3027                 !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3028 }
3029
3030 static bool is_ubsan_insn(struct instruction *insn)
3031 {
3032         return (insn->type == INSN_CALL &&
3033                 !strcmp(insn->call_dest->name,
3034                         "__ubsan_handle_builtin_unreachable"));
3035 }
3036
3037 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
3038 {
3039         int i;
3040         struct instruction *prev_insn;
3041
3042         if (insn->ignore || insn->type == INSN_NOP)
3043                 return true;
3044
3045         /*
3046          * Ignore any unused exceptions.  This can happen when a whitelisted
3047          * function has an exception table entry.
3048          *
3049          * Also ignore alternative replacement instructions.  This can happen
3050          * when a whitelisted function uses one of the ALTERNATIVE macros.
3051          */
3052         if (!strcmp(insn->sec->name, ".fixup") ||
3053             !strcmp(insn->sec->name, ".altinstr_replacement") ||
3054             !strcmp(insn->sec->name, ".altinstr_aux"))
3055                 return true;
3056
3057         if (!insn->func)
3058                 return false;
3059
3060         /*
3061          * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
3062          * __builtin_unreachable().  The BUG() macro has an unreachable() after
3063          * the UD2, which causes GCC's undefined trap logic to emit another UD2
3064          * (or occasionally a JMP to UD2).
3065          *
3066          * It may also insert a UD2 after calling a __noreturn function.
3067          */
3068         prev_insn = list_prev_entry(insn, list);
3069         if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
3070             (insn->type == INSN_BUG ||
3071              (insn->type == INSN_JUMP_UNCONDITIONAL &&
3072               insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
3073                 return true;
3074
3075         /*
3076          * Check if this (or a subsequent) instruction is related to
3077          * CONFIG_UBSAN or CONFIG_KASAN.
3078          *
3079          * End the search at 5 instructions to avoid going into the weeds.
3080          */
3081         for (i = 0; i < 5; i++) {
3082
3083                 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
3084                         return true;
3085
3086                 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3087                         if (insn->jump_dest &&
3088                             insn->jump_dest->func == insn->func) {
3089                                 insn = insn->jump_dest;
3090                                 continue;
3091                         }
3092
3093                         break;
3094                 }
3095
3096                 if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
3097                         break;
3098
3099                 insn = list_next_entry(insn, list);
3100         }
3101
3102         return false;
3103 }
3104
3105 static int validate_symbol(struct objtool_file *file, struct section *sec,
3106                            struct symbol *sym, struct insn_state *state)
3107 {
3108         struct instruction *insn;
3109         int ret;
3110
3111         if (!sym->len) {
3112                 WARN("%s() is missing an ELF size annotation", sym->name);
3113                 return 1;
3114         }
3115
3116         if (sym->pfunc != sym || sym->alias != sym)
3117                 return 0;
3118
3119         insn = find_insn(file, sec, sym->offset);
3120         if (!insn || insn->ignore || insn->visited)
3121                 return 0;
3122
3123         state->uaccess = sym->uaccess_safe;
3124
3125         ret = validate_branch(file, insn->func, insn, *state);
3126         if (ret && backtrace)
3127                 BT_FUNC("<=== (sym)", insn);
3128         return ret;
3129 }
3130
3131 static int validate_section(struct objtool_file *file, struct section *sec)
3132 {
3133         struct insn_state state;
3134         struct symbol *func;
3135         int warnings = 0;
3136
3137         list_for_each_entry(func, &sec->symbol_list, list) {
3138                 if (func->type != STT_FUNC)
3139                         continue;
3140
3141                 init_insn_state(&state, sec);
3142                 set_func_state(&state.cfi);
3143
3144                 warnings += validate_symbol(file, sec, func, &state);
3145         }
3146
3147         return warnings;
3148 }
3149
3150 static int validate_vmlinux_functions(struct objtool_file *file)
3151 {
3152         struct section *sec;
3153         int warnings = 0;
3154
3155         sec = find_section_by_name(file->elf, ".noinstr.text");
3156         if (sec) {
3157                 warnings += validate_section(file, sec);
3158                 warnings += validate_unwind_hints(file, sec);
3159         }
3160
3161         sec = find_section_by_name(file->elf, ".entry.text");
3162         if (sec) {
3163                 warnings += validate_section(file, sec);
3164                 warnings += validate_unwind_hints(file, sec);
3165         }
3166
3167         return warnings;
3168 }
3169
3170 static int validate_functions(struct objtool_file *file)
3171 {
3172         struct section *sec;
3173         int warnings = 0;
3174
3175         for_each_sec(file, sec) {
3176                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
3177                         continue;
3178
3179                 warnings += validate_section(file, sec);
3180         }
3181
3182         return warnings;
3183 }
3184
3185 static int validate_reachable_instructions(struct objtool_file *file)
3186 {
3187         struct instruction *insn;
3188
3189         if (file->ignore_unreachables)
3190                 return 0;
3191
3192         for_each_insn(file, insn) {
3193                 if (insn->visited || ignore_unreachable_insn(file, insn))
3194                         continue;
3195
3196                 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
3197                 return 1;
3198         }
3199
3200         return 0;
3201 }
3202
3203 int check(struct objtool_file *file)
3204 {
3205         int ret, warnings = 0;
3206
3207         arch_initial_func_cfi_state(&initial_func_cfi);
3208
3209         ret = decode_sections(file);
3210         if (ret < 0)
3211                 goto out;
3212         warnings += ret;
3213
3214         if (list_empty(&file->insn_list))
3215                 goto out;
3216
3217         if (vmlinux && !validate_dup) {
3218                 ret = validate_vmlinux_functions(file);
3219                 if (ret < 0)
3220                         goto out;
3221
3222                 warnings += ret;
3223                 goto out;
3224         }
3225
3226         if (retpoline) {
3227                 ret = validate_retpoline(file);
3228                 if (ret < 0)
3229                         return ret;
3230                 warnings += ret;
3231         }
3232
3233         ret = validate_functions(file);
3234         if (ret < 0)
3235                 goto out;
3236         warnings += ret;
3237
3238         ret = validate_unwind_hints(file, NULL);
3239         if (ret < 0)
3240                 goto out;
3241         warnings += ret;
3242
3243         if (!warnings) {
3244                 ret = validate_reachable_instructions(file);
3245                 if (ret < 0)
3246                         goto out;
3247                 warnings += ret;
3248         }
3249
3250         ret = create_static_call_sections(file);
3251         if (ret < 0)
3252                 goto out;
3253         warnings += ret;
3254
3255         if (mcount) {
3256                 ret = create_mcount_loc_sections(file);
3257                 if (ret < 0)
3258                         goto out;
3259                 warnings += ret;
3260         }
3261
3262 out:
3263         /*
3264          *  For now, don't fail the kernel build on fatal warnings.  These
3265          *  errors are still fairly common due to the growing matrix of
3266          *  supported toolchains and their recent pace of change.
3267          */
3268         return 0;
3269 }