tools/objtool/check.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
   4  */
   5
   6 #include <string.h>
   7 #include <stdlib.h>
   8 #include <inttypes.h>
   9 #include <sys/mman.h>
  10
  11 #include <arch/elf.h>
  12 #include <objtool/builtin.h>
  13 #include <objtool/cfi.h>
  14 #include <objtool/arch.h>
  15 #include <objtool/check.h>
  16 #include <objtool/special.h>
  17 #include <objtool/warn.h>
  18 #include <objtool/endianness.h>
  19
  20 #include <linux/objtool.h>
  21 #include <linux/hashtable.h>
  22 #include <linux/kernel.h>
  23 #include <linux/static_call_types.h>
  24
  25 struct alternative {
  26         struct list_head list;
  27         struct instruction *insn;
  28         bool skip_orig;
  29 };
  30
  31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
  32
  33 static struct cfi_init_state initial_func_cfi;
  34 static struct cfi_state init_cfi;
  35 static struct cfi_state func_cfi;
  36
  37 struct instruction *find_insn(struct objtool_file *file,
  38                               struct section *sec, unsigned long offset)
  39 {
  40         struct instruction *insn;
  41
  42         hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
  43                 if (insn->sec == sec && insn->offset == offset)
  44                         return insn;
  45         }
  46
  47         return NULL;
  48 }
  49
  50 static struct instruction *next_insn_same_sec(struct objtool_file *file,
  51                                               struct instruction *insn)
  52 {
  53         struct instruction *next = list_next_entry(insn, list);
  54
  55         if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
  56                 return NULL;
  57
  58         return next;
  59 }
  60
  61 static struct instruction *next_insn_same_func(struct objtool_file *file,
  62                                                struct instruction *insn)
  63 {
  64         struct instruction *next = list_next_entry(insn, list);
  65         struct symbol *func = insn->func;
  66
  67         if (!func)
  68                 return NULL;
  69
  70         if (&next->list != &file->insn_list && next->func == func)
  71                 return next;
  72
  73         /* Check if we're already in the subfunction: */
  74         if (func == func->cfunc)
  75                 return NULL;
  76
  77         /* Move to the subfunction: */
  78         return find_insn(file, func->cfunc->sec, func->cfunc->offset);
  79 }
  80
  81 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
  82                                                struct instruction *insn)
  83 {
  84         struct instruction *prev = list_prev_entry(insn, list);
  85
  86         if (&prev->list != &file->insn_list && prev->func == insn->func)
  87                 return prev;
  88
  89         return NULL;
  90 }
  91
  92 #define func_for_each_insn(file, func, insn)                            \
  93         for (insn = find_insn(file, func->sec, func->offset);           \
  94              insn;                                                      \
  95              insn = next_insn_same_func(file, insn))
  96
  97 #define sym_for_each_insn(file, sym, insn)                              \
  98         for (insn = find_insn(file, sym->sec, sym->offset);             \
  99              insn && &insn->list != &file->insn_list &&                 \
 100                 insn->sec == sym->sec &&                                \
 101                 insn->offset < sym->offset + sym->len;                  \
 102              insn = list_next_entry(insn, list))
 103
 104 #define sym_for_each_insn_continue_reverse(file, sym, insn)             \
 105         for (insn = list_prev_entry(insn, list);                        \
 106              &insn->list != &file->insn_list &&                         \
 107                 insn->sec == sym->sec && insn->offset >= sym->offset;   \
 108              insn = list_prev_entry(insn, list))
 109
 110 #define sec_for_each_insn_from(file, insn)                              \
 111         for (; insn; insn = next_insn_same_sec(file, insn))
 112
 113 #define sec_for_each_insn_continue(file, insn)                          \
 114         for (insn = next_insn_same_sec(file, insn); insn;               \
 115              insn = next_insn_same_sec(file, insn))
 116
 117 static bool is_jump_table_jump(struct instruction *insn)
 118 {
 119         struct alt_group *alt_group = insn->alt_group;
 120
 121         if (insn->jump_table)
 122                 return true;
 123
 124         /* Retpoline alternative for a jump table? */
 125         return alt_group && alt_group->orig_group &&
 126                alt_group->orig_group->first_insn->jump_table;
 127 }
 128
 129 static bool is_sibling_call(struct instruction *insn)
 130 {
 131         /*
 132          * Assume only ELF functions can make sibling calls.  This ensures
 133          * sibling call detection consistency between vmlinux.o and individual
 134          * objects.
 135          */
 136         if (!insn->func)
 137                 return false;
 138
 139         /* An indirect jump is either a sibling call or a jump to a table. */
 140         if (insn->type == INSN_JUMP_DYNAMIC)
 141                 return !is_jump_table_jump(insn);
 142
 143         /* add_jump_destinations() sets insn->call_dest for sibling calls. */
 144         return (is_static_jump(insn) && insn->call_dest);
 145 }
 146
 147 /*
 148  * This checks to see if the given function is a "noreturn" function.
 149  *
 150  * For global functions which are outside the scope of this object file, we
 151  * have to keep a manual list of them.
 152  *
 153  * For local functions, we have to detect them manually by simply looking for
 154  * the lack of a return instruction.
 155  */
 156 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 157                                 int recursion)
 158 {
 159         int i;
 160         struct instruction *insn;
 161         bool empty = true;
 162
 163         /*
 164          * Unfortunately these have to be hard coded because the noreturn
 165          * attribute isn't provided in ELF data.
 166          */
 167         static const char * const global_noreturns[] = {
 168                 "__stack_chk_fail",
 169                 "panic",
 170                 "do_exit",
 171                 "do_task_dead",
 172                 "__module_put_and_exit",
 173                 "complete_and_exit",
 174                 "__reiserfs_panic",
 175                 "lbug_with_loc",
 176                 "fortify_panic",
 177                 "usercopy_abort",
 178                 "machine_real_restart",
 179                 "rewind_stack_do_exit",
 180                 "kunit_try_catch_throw",
 181                 "xen_start_kernel",
 182                 "cpu_bringup_and_idle",
 183         };
 184
 185         if (!func)
 186                 return false;
 187
 188         if (func->bind == STB_WEAK)
 189                 return false;
 190
 191         if (func->bind == STB_GLOBAL)
 192                 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
 193                         if (!strcmp(func->name, global_noreturns[i]))
 194                                 return true;
 195
 196         if (!func->len)
 197                 return false;
 198
 199         insn = find_insn(file, func->sec, func->offset);
 200         if (!insn->func)
 201                 return false;
 202
 203         func_for_each_insn(file, func, insn) {
 204                 empty = false;
 205
 206                 if (insn->type == INSN_RETURN)
 207                         return false;
 208         }
 209
 210         if (empty)
 211                 return false;
 212
 213         /*
 214          * A function can have a sibling call instead of a return.  In that
 215          * case, the function's dead-end status depends on whether the target
 216          * of the sibling call returns.
 217          */
 218         func_for_each_insn(file, func, insn) {
 219                 if (is_sibling_call(insn)) {
 220                         struct instruction *dest = insn->jump_dest;
 221
 222                         if (!dest)
 223                                 /* sibling call to another file */
 224                                 return false;
 225
 226                         /* local sibling call */
 227                         if (recursion == 5) {
 228                                 /*
 229                                  * Infinite recursion: two functions have
 230                                  * sibling calls to each other.  This is a very
 231                                  * rare case.  It means they aren't dead ends.
 232                                  */
 233                                 return false;
 234                         }
 235
 236                         return __dead_end_function(file, dest->func, recursion+1);
 237                 }
 238         }
 239
 240         return true;
 241 }
 242
 243 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
 244 {
 245         return __dead_end_function(file, func, 0);
 246 }
 247
 248 static void init_cfi_state(struct cfi_state *cfi)
 249 {
 250         int i;
 251
 252         for (i = 0; i < CFI_NUM_REGS; i++) {
 253                 cfi->regs[i].base = CFI_UNDEFINED;
 254                 cfi->vals[i].base = CFI_UNDEFINED;
 255         }
 256         cfi->cfa.base = CFI_UNDEFINED;
 257         cfi->drap_reg = CFI_UNDEFINED;
 258         cfi->drap_offset = -1;
 259 }
 260
 261 static void init_insn_state(struct insn_state *state, struct section *sec)
 262 {
 263         memset(state, 0, sizeof(*state));
 264         init_cfi_state(&state->cfi);
 265
 266         /*
 267          * We need the full vmlinux for noinstr validation, otherwise we can
 268          * not correctly determine insn->call_dest->sec (external symbols do
 269          * not have a section).
 270          */
 271         if (vmlinux && noinstr && sec)
 272                 state->noinstr = sec->noinstr;
 273 }
 274
 275 static struct cfi_state *cfi_alloc(void)
 276 {
 277         struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
 278         if (!cfi) {
 279                 WARN("calloc failed");
 280                 exit(1);
 281         }
 282         nr_cfi++;
 283         return cfi;
 284 }
 285
 286 static int cfi_bits;
 287 static struct hlist_head *cfi_hash;
 288
 289 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
 290 {
 291         return memcmp((void *)cfi1 + sizeof(cfi1->hash),
 292                       (void *)cfi2 + sizeof(cfi2->hash),
 293                       sizeof(struct cfi_state) - sizeof(struct hlist_node));
 294 }
 295
 296 static inline u32 cfi_key(struct cfi_state *cfi)
 297 {
 298         return jhash((void *)cfi + sizeof(cfi->hash),
 299                      sizeof(*cfi) - sizeof(cfi->hash), 0);
 300 }
 301
 302 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
 303 {
 304         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 305         struct cfi_state *obj;
 306
 307         hlist_for_each_entry(obj, head, hash) {
 308                 if (!cficmp(cfi, obj)) {
 309                         nr_cfi_cache++;
 310                         return obj;
 311                 }
 312         }
 313
 314         obj = cfi_alloc();
 315         *obj = *cfi;
 316         hlist_add_head(&obj->hash, head);
 317
 318         return obj;
 319 }
 320
 321 static void cfi_hash_add(struct cfi_state *cfi)
 322 {
 323         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 324
 325         hlist_add_head(&cfi->hash, head);
 326 }
 327
 328 static void *cfi_hash_alloc(unsigned long size)
 329 {
 330         cfi_bits = max(10, ilog2(size));
 331         cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
 332                         PROT_READ|PROT_WRITE,
 333                         MAP_PRIVATE|MAP_ANON, -1, 0);
 334         if (cfi_hash == (void *)-1L) {
 335                 WARN("mmap fail cfi_hash");
 336                 cfi_hash = NULL;
 337         }  else if (stats) {
 338                 printf("cfi_bits: %d\n", cfi_bits);
 339         }
 340
 341         return cfi_hash;
 342 }
 343
 344 static unsigned long nr_insns;
 345 static unsigned long nr_insns_visited;
 346
 347 /*
 348  * Call the arch-specific instruction decoder for all the instructions and add
 349  * them to the global instruction list.
 350  */
 351 static int decode_instructions(struct objtool_file *file)
 352 {
 353         struct section *sec;
 354         struct symbol *func;
 355         unsigned long offset;
 356         struct instruction *insn;
 357         int ret;
 358
 359         for_each_sec(file, sec) {
 360
 361                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
 362                         continue;
 363
 364                 if (strcmp(sec->name, ".altinstr_replacement") &&
 365                     strcmp(sec->name, ".altinstr_aux") &&
 366                     strncmp(sec->name, ".discard.", 9))
 367                         sec->text = true;
 368
 369                 if (!strcmp(sec->name, ".noinstr.text") ||
 370                     !strcmp(sec->name, ".entry.text") ||
 371                     !strncmp(sec->name, ".text.__x86.", 12))
 372                         sec->noinstr = true;
 373
 374                 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
 375                         insn = malloc(sizeof(*insn));
 376                         if (!insn) {
 377                                 WARN("malloc failed");
 378                                 return -1;
 379                         }
 380                         memset(insn, 0, sizeof(*insn));
 381                         INIT_LIST_HEAD(&insn->alts);
 382                         INIT_LIST_HEAD(&insn->stack_ops);
 383
 384                         insn->sec = sec;
 385                         insn->offset = offset;
 386
 387                         ret = arch_decode_instruction(file->elf, sec, offset,
 388                                                       sec->sh.sh_size - offset,
 389                                                       &insn->len, &insn->type,
 390                                                       &insn->immediate,
 391                                                       &insn->stack_ops);
 392                         if (ret)
 393                                 goto err;
 394
 395                         hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
 396                         list_add_tail(&insn->list, &file->insn_list);
 397                         nr_insns++;
 398                 }
 399
 400                 list_for_each_entry(func, &sec->symbol_list, list) {
 401                         if (func->type != STT_FUNC || func->alias != func)
 402                                 continue;
 403
 404                         if (!find_insn(file, sec, func->offset)) {
 405                                 WARN("%s(): can't find starting instruction",
 406                                      func->name);
 407                                 return -1;
 408                         }
 409
 410                         sym_for_each_insn(file, func, insn)
 411                                 insn->func = func;
 412                 }
 413         }
 414
 415         if (stats)
 416                 printf("nr_insns: %lu\n", nr_insns);
 417
 418         return 0;
 419
 420 err:
 421         free(insn);
 422         return ret;
 423 }
 424
 425 static struct instruction *find_last_insn(struct objtool_file *file,
 426                                           struct section *sec)
 427 {
 428         struct instruction *insn = NULL;
 429         unsigned int offset;
 430         unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
 431
 432         for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
 433                 insn = find_insn(file, sec, offset);
 434
 435         return insn;
 436 }
 437
 438 /*
 439  * Mark "ud2" instructions and manually annotated dead ends.
 440  */
 441 static int add_dead_ends(struct objtool_file *file)
 442 {
 443         struct section *sec;
 444         struct reloc *reloc;
 445         struct instruction *insn;
 446
 447         /*
 448          * By default, "ud2" is a dead end unless otherwise annotated, because
 449          * GCC 7 inserts it for certain divide-by-zero cases.
 450          */
 451         for_each_insn(file, insn)
 452                 if (insn->type == INSN_BUG)
 453                         insn->dead_end = true;
 454
 455         /*
 456          * Check for manually annotated dead ends.
 457          */
 458         sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
 459         if (!sec)
 460                 goto reachable;
 461
 462         list_for_each_entry(reloc, &sec->reloc_list, list) {
 463                 if (reloc->sym->type != STT_SECTION) {
 464                         WARN("unexpected relocation symbol type in %s", sec->name);
 465                         return -1;
 466                 }
 467                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 468                 if (insn)
 469                         insn = list_prev_entry(insn, list);
 470                 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 471                         insn = find_last_insn(file, reloc->sym->sec);
 472                         if (!insn) {
 473                                 WARN("can't find unreachable insn at %s+0x%" PRIx64,
 474                                      reloc->sym->sec->name, reloc->addend);
 475                                 return -1;
 476                         }
 477                 } else {
 478                         WARN("can't find unreachable insn at %s+0x%" PRIx64,
 479                              reloc->sym->sec->name, reloc->addend);
 480                         return -1;
 481                 }
 482
 483                 insn->dead_end = true;
 484         }
 485
 486 reachable:
 487         /*
 488          * These manually annotated reachable checks are needed for GCC 4.4,
 489          * where the Linux unreachable() macro isn't supported.  In that case
 490          * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
 491          * not a dead end.
 492          */
 493         sec = find_section_by_name(file->elf, ".rela.discard.reachable");
 494         if (!sec)
 495                 return 0;
 496
 497         list_for_each_entry(reloc, &sec->reloc_list, list) {
 498                 if (reloc->sym->type != STT_SECTION) {
 499                         WARN("unexpected relocation symbol type in %s", sec->name);
 500                         return -1;
 501                 }
 502                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 503                 if (insn)
 504                         insn = list_prev_entry(insn, list);
 505                 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 506                         insn = find_last_insn(file, reloc->sym->sec);
 507                         if (!insn) {
 508                                 WARN("can't find reachable insn at %s+0x%" PRIx64,
 509                                      reloc->sym->sec->name, reloc->addend);
 510                                 return -1;
 511                         }
 512                 } else {
 513                         WARN("can't find reachable insn at %s+0x%" PRIx64,
 514                              reloc->sym->sec->name, reloc->addend);
 515                         return -1;
 516                 }
 517
 518                 insn->dead_end = false;
 519         }
 520
 521         return 0;
 522 }
 523
 524 static int create_static_call_sections(struct objtool_file *file)
 525 {
 526         struct section *sec;
 527         struct static_call_site *site;
 528         struct instruction *insn;
 529         struct symbol *key_sym;
 530         char *key_name, *tmp;
 531         int idx;
 532
 533         sec = find_section_by_name(file->elf, ".static_call_sites");
 534         if (sec) {
 535                 INIT_LIST_HEAD(&file->static_call_list);
 536                 WARN("file already has .static_call_sites section, skipping");
 537                 return 0;
 538         }
 539
 540         if (list_empty(&file->static_call_list))
 541                 return 0;
 542
 543         idx = 0;
 544         list_for_each_entry(insn, &file->static_call_list, call_node)
 545                 idx++;
 546
 547         sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
 548                                  sizeof(struct static_call_site), idx);
 549         if (!sec)
 550                 return -1;
 551
 552         idx = 0;
 553         list_for_each_entry(insn, &file->static_call_list, call_node) {
 554
 555                 site = (struct static_call_site *)sec->data->d_buf + idx;
 556                 memset(site, 0, sizeof(struct static_call_site));
 557
 558                 /* populate reloc for 'addr' */
 559                 if (elf_add_reloc_to_insn(file->elf, sec,
 560                                           idx * sizeof(struct static_call_site),
 561                                           R_X86_64_PC32,
 562                                           insn->sec, insn->offset))
 563                         return -1;
 564
 565                 /* find key symbol */
 566                 key_name = strdup(insn->call_dest->name);
 567                 if (!key_name) {
 568                         perror("strdup");
 569                         return -1;
 570                 }
 571                 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
 572                             STATIC_CALL_TRAMP_PREFIX_LEN)) {
 573                         WARN("static_call: trampoline name malformed: %s", key_name);
 574                         return -1;
 575                 }
 576                 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
 577                 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
 578
 579                 key_sym = find_symbol_by_name(file->elf, tmp);
 580                 if (!key_sym) {
 581                         if (!module) {
 582                                 WARN("static_call: can't find static_call_key symbol: %s", tmp);
 583                                 return -1;
 584                         }
 585
 586                         /*
 587                          * For modules(), the key might not be exported, which
 588                          * means the module can make static calls but isn't
 589                          * allowed to change them.
 590                          *
 591                          * In that case we temporarily set the key to be the
 592                          * trampoline address.  This is fixed up in
 593                          * static_call_add_module().
 594                          */
 595                         key_sym = insn->call_dest;
 596                 }
 597                 free(key_name);
 598
 599                 /* populate reloc for 'key' */
 600                 if (elf_add_reloc(file->elf, sec,
 601                                   idx * sizeof(struct static_call_site) + 4,
 602                                   R_X86_64_PC32, key_sym,
 603                                   is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
 604                         return -1;
 605
 606                 idx++;
 607         }
 608
 609         return 0;
 610 }
 611
 612 static int create_retpoline_sites_sections(struct objtool_file *file)
 613 {
 614         struct instruction *insn;
 615         struct section *sec;
 616         int idx;
 617
 618         sec = find_section_by_name(file->elf, ".retpoline_sites");
 619         if (sec) {
 620                 WARN("file already has .retpoline_sites, skipping");
 621                 return 0;
 622         }
 623
 624         idx = 0;
 625         list_for_each_entry(insn, &file->retpoline_call_list, call_node)
 626                 idx++;
 627
 628         if (!idx)
 629                 return 0;
 630
 631         sec = elf_create_section(file->elf, ".retpoline_sites", 0,
 632                                  sizeof(int), idx);
 633         if (!sec) {
 634                 WARN("elf_create_section: .retpoline_sites");
 635                 return -1;
 636         }
 637
 638         idx = 0;
 639         list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
 640
 641                 int *site = (int *)sec->data->d_buf + idx;
 642                 *site = 0;
 643
 644                 if (elf_add_reloc_to_insn(file->elf, sec,
 645                                           idx * sizeof(int),
 646                                           R_X86_64_PC32,
 647                                           insn->sec, insn->offset)) {
 648                         WARN("elf_add_reloc_to_insn: .retpoline_sites");
 649                         return -1;
 650                 }
 651
 652                 idx++;
 653         }
 654
 655         return 0;
 656 }
 657
 658 static int create_return_sites_sections(struct objtool_file *file)
 659 {
 660         struct instruction *insn;
 661         struct section *sec;
 662         int idx;
 663
 664         sec = find_section_by_name(file->elf, ".return_sites");
 665         if (sec) {
 666                 WARN("file already has .return_sites, skipping");
 667                 return 0;
 668         }
 669
 670         idx = 0;
 671         list_for_each_entry(insn, &file->return_thunk_list, call_node)
 672                 idx++;
 673
 674         if (!idx)
 675                 return 0;
 676
 677         sec = elf_create_section(file->elf, ".return_sites", 0,
 678                                  sizeof(int), idx);
 679         if (!sec) {
 680                 WARN("elf_create_section: .return_sites");
 681                 return -1;
 682         }
 683
 684         idx = 0;
 685         list_for_each_entry(insn, &file->return_thunk_list, call_node) {
 686
 687                 int *site = (int *)sec->data->d_buf + idx;
 688                 *site = 0;
 689
 690                 if (elf_add_reloc_to_insn(file->elf, sec,
 691                                           idx * sizeof(int),
 692                                           R_X86_64_PC32,
 693                                           insn->sec, insn->offset)) {
 694                         WARN("elf_add_reloc_to_insn: .return_sites");
 695                         return -1;
 696                 }
 697
 698                 idx++;
 699         }
 700
 701         return 0;
 702 }
 703
 704 static int create_mcount_loc_sections(struct objtool_file *file)
 705 {
 706         struct section *sec;
 707         unsigned long *loc;
 708         struct instruction *insn;
 709         int idx;
 710
 711         sec = find_section_by_name(file->elf, "__mcount_loc");
 712         if (sec) {
 713                 INIT_LIST_HEAD(&file->mcount_loc_list);
 714                 WARN("file already has __mcount_loc section, skipping");
 715                 return 0;
 716         }
 717
 718         if (list_empty(&file->mcount_loc_list))
 719                 return 0;
 720
 721         idx = 0;
 722         list_for_each_entry(insn, &file->mcount_loc_list, call_node)
 723                 idx++;
 724
 725         sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
 726         if (!sec)
 727                 return -1;
 728
 729         idx = 0;
 730         list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
 731
 732                 loc = (unsigned long *)sec->data->d_buf + idx;
 733                 memset(loc, 0, sizeof(unsigned long));
 734
 735                 if (elf_add_reloc_to_insn(file->elf, sec,
 736                                           idx * sizeof(unsigned long),
 737                                           R_X86_64_64,
 738                                           insn->sec, insn->offset))
 739                         return -1;
 740
 741                 idx++;
 742         }
 743
 744         return 0;
 745 }
 746
 747 /*
 748  * Warnings shouldn't be reported for ignored functions.
 749  */
 750 static void add_ignores(struct objtool_file *file)
 751 {
 752         struct instruction *insn;
 753         struct section *sec;
 754         struct symbol *func;
 755         struct reloc *reloc;
 756
 757         sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
 758         if (!sec)
 759                 return;
 760
 761         list_for_each_entry(reloc, &sec->reloc_list, list) {
 762                 switch (reloc->sym->type) {
 763                 case STT_FUNC:
 764                         func = reloc->sym;
 765                         break;
 766
 767                 case STT_SECTION:
 768                         func = find_func_by_offset(reloc->sym->sec, reloc->addend);
 769                         if (!func)
 770                                 continue;
 771                         break;
 772
 773                 default:
 774                         WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
 775                         continue;
 776                 }
 777
 778                 func_for_each_insn(file, func, insn)
 779                         insn->ignore = true;
 780         }
 781 }
 782
 783 /*
 784  * This is a whitelist of functions that is allowed to be called with AC set.
 785  * The list is meant to be minimal and only contains compiler instrumentation
 786  * ABI and a few functions used to implement *_{to,from}_user() functions.
 787  *
 788  * These functions must not directly change AC, but may PUSHF/POPF.
 789  */
 790 static const char *uaccess_safe_builtin[] = {
 791         /* KASAN */
 792         "kasan_report",
 793         "kasan_check_range",
 794         /* KASAN out-of-line */
 795         "__asan_loadN_noabort",
 796         "__asan_load1_noabort",
 797         "__asan_load2_noabort",
 798         "__asan_load4_noabort",
 799         "__asan_load8_noabort",
 800         "__asan_load16_noabort",
 801         "__asan_storeN_noabort",
 802         "__asan_store1_noabort",
 803         "__asan_store2_noabort",
 804         "__asan_store4_noabort",
 805         "__asan_store8_noabort",
 806         "__asan_store16_noabort",
 807         "__kasan_check_read",
 808         "__kasan_check_write",
 809         /* KASAN in-line */
 810         "__asan_report_load_n_noabort",
 811         "__asan_report_load1_noabort",
 812         "__asan_report_load2_noabort",
 813         "__asan_report_load4_noabort",
 814         "__asan_report_load8_noabort",
 815         "__asan_report_load16_noabort",
 816         "__asan_report_store_n_noabort",
 817         "__asan_report_store1_noabort",
 818         "__asan_report_store2_noabort",
 819         "__asan_report_store4_noabort",
 820         "__asan_report_store8_noabort",
 821         "__asan_report_store16_noabort",
 822         /* KCSAN */
 823         "__kcsan_check_access",
 824         "kcsan_found_watchpoint",
 825         "kcsan_setup_watchpoint",
 826         "kcsan_check_scoped_accesses",
 827         "kcsan_disable_current",
 828         "kcsan_enable_current_nowarn",
 829         /* KCSAN/TSAN */
 830         "__tsan_func_entry",
 831         "__tsan_func_exit",
 832         "__tsan_read_range",
 833         "__tsan_write_range",
 834         "__tsan_read1",
 835         "__tsan_read2",
 836         "__tsan_read4",
 837         "__tsan_read8",
 838         "__tsan_read16",
 839         "__tsan_write1",
 840         "__tsan_write2",
 841         "__tsan_write4",
 842         "__tsan_write8",
 843         "__tsan_write16",
 844         "__tsan_read_write1",
 845         "__tsan_read_write2",
 846         "__tsan_read_write4",
 847         "__tsan_read_write8",
 848         "__tsan_read_write16",
 849         "__tsan_atomic8_load",
 850         "__tsan_atomic16_load",
 851         "__tsan_atomic32_load",
 852         "__tsan_atomic64_load",
 853         "__tsan_atomic8_store",
 854         "__tsan_atomic16_store",
 855         "__tsan_atomic32_store",
 856         "__tsan_atomic64_store",
 857         "__tsan_atomic8_exchange",
 858         "__tsan_atomic16_exchange",
 859         "__tsan_atomic32_exchange",
 860         "__tsan_atomic64_exchange",
 861         "__tsan_atomic8_fetch_add",
 862         "__tsan_atomic16_fetch_add",
 863         "__tsan_atomic32_fetch_add",
 864         "__tsan_atomic64_fetch_add",
 865         "__tsan_atomic8_fetch_sub",
 866         "__tsan_atomic16_fetch_sub",
 867         "__tsan_atomic32_fetch_sub",
 868         "__tsan_atomic64_fetch_sub",
 869         "__tsan_atomic8_fetch_and",
 870         "__tsan_atomic16_fetch_and",
 871         "__tsan_atomic32_fetch_and",
 872         "__tsan_atomic64_fetch_and",
 873         "__tsan_atomic8_fetch_or",
 874         "__tsan_atomic16_fetch_or",
 875         "__tsan_atomic32_fetch_or",
 876         "__tsan_atomic64_fetch_or",
 877         "__tsan_atomic8_fetch_xor",
 878         "__tsan_atomic16_fetch_xor",
 879         "__tsan_atomic32_fetch_xor",
 880         "__tsan_atomic64_fetch_xor",
 881         "__tsan_atomic8_fetch_nand",
 882         "__tsan_atomic16_fetch_nand",
 883         "__tsan_atomic32_fetch_nand",
 884         "__tsan_atomic64_fetch_nand",
 885         "__tsan_atomic8_compare_exchange_strong",
 886         "__tsan_atomic16_compare_exchange_strong",
 887         "__tsan_atomic32_compare_exchange_strong",
 888         "__tsan_atomic64_compare_exchange_strong",
 889         "__tsan_atomic8_compare_exchange_weak",
 890         "__tsan_atomic16_compare_exchange_weak",
 891         "__tsan_atomic32_compare_exchange_weak",
 892         "__tsan_atomic64_compare_exchange_weak",
 893         "__tsan_atomic8_compare_exchange_val",
 894         "__tsan_atomic16_compare_exchange_val",
 895         "__tsan_atomic32_compare_exchange_val",
 896         "__tsan_atomic64_compare_exchange_val",
 897         "__tsan_atomic_thread_fence",
 898         "__tsan_atomic_signal_fence",
 899         /* KCOV */
 900         "write_comp_data",
 901         "check_kcov_mode",
 902         "__sanitizer_cov_trace_pc",
 903         "__sanitizer_cov_trace_const_cmp1",
 904         "__sanitizer_cov_trace_const_cmp2",
 905         "__sanitizer_cov_trace_const_cmp4",
 906         "__sanitizer_cov_trace_const_cmp8",
 907         "__sanitizer_cov_trace_cmp1",
 908         "__sanitizer_cov_trace_cmp2",
 909         "__sanitizer_cov_trace_cmp4",
 910         "__sanitizer_cov_trace_cmp8",
 911         "__sanitizer_cov_trace_switch",
 912         /* UBSAN */
 913         "ubsan_type_mismatch_common",
 914         "__ubsan_handle_type_mismatch",
 915         "__ubsan_handle_type_mismatch_v1",
 916         "__ubsan_handle_shift_out_of_bounds",
 917         /* misc */
 918         "csum_partial_copy_generic",
 919         "copy_mc_fragile",
 920         "copy_mc_fragile_handle_tail",
 921         "copy_mc_enhanced_fast_string",
 922         "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
 923         NULL
 924 };
 925
 926 static void add_uaccess_safe(struct objtool_file *file)
 927 {
 928         struct symbol *func;
 929         const char **name;
 930
 931         if (!uaccess)
 932                 return;
 933
 934         for (name = uaccess_safe_builtin; *name; name++) {
 935                 func = find_symbol_by_name(file->elf, *name);
 936                 if (!func)
 937                         continue;
 938
 939                 func->uaccess_safe = true;
 940         }
 941 }
 942
 943 /*
 944  * FIXME: For now, just ignore any alternatives which add retpolines.  This is
 945  * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
 946  * But it at least allows objtool to understand the control flow *around* the
 947  * retpoline.
 948  */
 949 static int add_ignore_alternatives(struct objtool_file *file)
 950 {
 951         struct section *sec;
 952         struct reloc *reloc;
 953         struct instruction *insn;
 954
 955         sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
 956         if (!sec)
 957                 return 0;
 958
 959         list_for_each_entry(reloc, &sec->reloc_list, list) {
 960                 if (reloc->sym->type != STT_SECTION) {
 961                         WARN("unexpected relocation symbol type in %s", sec->name);
 962                         return -1;
 963                 }
 964
 965                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 966                 if (!insn) {
 967                         WARN("bad .discard.ignore_alts entry");
 968                         return -1;
 969                 }
 970
 971                 insn->ignore_alts = true;
 972         }
 973
 974         return 0;
 975 }
 976
 977 __weak bool arch_is_retpoline(struct symbol *sym)
 978 {
 979         return false;
 980 }
 981
 982 __weak bool arch_is_rethunk(struct symbol *sym)
 983 {
 984         return false;
 985 }
 986
 987 #define NEGATIVE_RELOC  ((void *)-1L)
 988
 989 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
 990 {
 991         if (insn->reloc == NEGATIVE_RELOC)
 992                 return NULL;
 993
 994         if (!insn->reloc) {
 995                 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
 996                                                        insn->offset, insn->len);
 997                 if (!insn->reloc) {
 998                         insn->reloc = NEGATIVE_RELOC;
 999                         return NULL;
1000                 }
1001         }
1002
1003         return insn->reloc;
1004 }
1005
1006 static void remove_insn_ops(struct instruction *insn)
1007 {
1008         struct stack_op *op, *tmp;
1009
1010         list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
1011                 list_del(&op->list);
1012                 free(op);
1013         }
1014 }
1015
1016 static void annotate_call_site(struct objtool_file *file,
1017                                struct instruction *insn, bool sibling)
1018 {
1019         struct reloc *reloc = insn_reloc(file, insn);
1020         struct symbol *sym = insn->call_dest;
1021
1022         if (!sym)
1023                 sym = reloc->sym;
1024
1025         /*
1026          * Alternative replacement code is just template code which is
1027          * sometimes copied to the original instruction. For now, don't
1028          * annotate it. (In the future we might consider annotating the
1029          * original instruction if/when it ever makes sense to do so.)
1030          */
1031         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1032                 return;
1033
1034         if (sym->static_call_tramp) {
1035                 list_add_tail(&insn->call_node, &file->static_call_list);
1036                 return;
1037         }
1038
1039         if (sym->retpoline_thunk) {
1040                 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1041                 return;
1042         }
1043
1044         /*
1045          * Many compilers cannot disable KCOV with a function attribute
1046          * so they need a little help, NOP out any KCOV calls from noinstr
1047          * text.
1048          */
1049         if (insn->sec->noinstr && sym->kcov) {
1050                 if (reloc) {
1051                         reloc->type = R_NONE;
1052                         elf_write_reloc(file->elf, reloc);
1053                 }
1054
1055                 elf_write_insn(file->elf, insn->sec,
1056                                insn->offset, insn->len,
1057                                sibling ? arch_ret_insn(insn->len)
1058                                        : arch_nop_insn(insn->len));
1059
1060                 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1061
1062                 if (sibling) {
1063                         /*
1064                          * We've replaced the tail-call JMP insn by two new
1065                          * insn: RET; INT3, except we only have a single struct
1066                          * insn here. Mark it retpoline_safe to avoid the SLS
1067                          * warning, instead of adding another insn.
1068                          */
1069                         insn->retpoline_safe = true;
1070                 }
1071
1072                 return;
1073         }
1074
1075         if (mcount && sym->fentry) {
1076                 if (sibling)
1077                         WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
1078
1079                 if (reloc) {
1080                         reloc->type = R_NONE;
1081                         elf_write_reloc(file->elf, reloc);
1082                 }
1083
1084                 elf_write_insn(file->elf, insn->sec,
1085                                insn->offset, insn->len,
1086                                arch_nop_insn(insn->len));
1087
1088                 insn->type = INSN_NOP;
1089
1090                 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1091                 return;
1092         }
1093 }
1094
1095 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1096                           struct symbol *dest, bool sibling)
1097 {
1098         insn->call_dest = dest;
1099         if (!dest)
1100                 return;
1101
1102         /*
1103          * Whatever stack impact regular CALLs have, should be undone
1104          * by the RETURN of the called function.
1105          *
1106          * Annotated intra-function calls retain the stack_ops but
1107          * are converted to JUMP, see read_intra_function_calls().
1108          */
1109         remove_insn_ops(insn);
1110
1111         annotate_call_site(file, insn, sibling);
1112 }
1113
1114 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1115 {
1116         /*
1117          * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1118          * so convert them accordingly.
1119          */
1120         switch (insn->type) {
1121         case INSN_CALL:
1122                 insn->type = INSN_CALL_DYNAMIC;
1123                 break;
1124         case INSN_JUMP_UNCONDITIONAL:
1125                 insn->type = INSN_JUMP_DYNAMIC;
1126                 break;
1127         case INSN_JUMP_CONDITIONAL:
1128                 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1129                 break;
1130         default:
1131                 return;
1132         }
1133
1134         insn->retpoline_safe = true;
1135
1136         /*
1137          * Whatever stack impact regular CALLs have, should be undone
1138          * by the RETURN of the called function.
1139          *
1140          * Annotated intra-function calls retain the stack_ops but
1141          * are converted to JUMP, see read_intra_function_calls().
1142          */
1143         remove_insn_ops(insn);
1144
1145         annotate_call_site(file, insn, false);
1146 }
1147
1148 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1149 {
1150         /*
1151          * Return thunk tail calls are really just returns in disguise,
1152          * so convert them accordingly.
1153          */
1154         insn->type = INSN_RETURN;
1155         insn->retpoline_safe = true;
1156
1157         /* Skip the non-text sections, specially .discard ones */
1158         if (add && insn->sec->text)
1159                 list_add_tail(&insn->call_node, &file->return_thunk_list);
1160 }
1161
1162 /*
1163  * Find the destination instructions for all jumps.
1164  */
1165 static int add_jump_destinations(struct objtool_file *file)
1166 {
1167         struct instruction *insn;
1168         struct reloc *reloc;
1169         struct section *dest_sec;
1170         unsigned long dest_off;
1171
1172         for_each_insn(file, insn) {
1173                 if (!is_static_jump(insn))
1174                         continue;
1175
1176                 reloc = insn_reloc(file, insn);
1177                 if (!reloc) {
1178                         dest_sec = insn->sec;
1179                         dest_off = arch_jump_destination(insn);
1180                 } else if (reloc->sym->type == STT_SECTION) {
1181                         dest_sec = reloc->sym->sec;
1182                         dest_off = arch_dest_reloc_offset(reloc->addend);
1183                 } else if (reloc->sym->retpoline_thunk) {
1184                         add_retpoline_call(file, insn);
1185                         continue;
1186                 } else if (reloc->sym->return_thunk) {
1187                         add_return_call(file, insn, true);
1188                         continue;
1189                 } else if (insn->func) {
1190                         /* internal or external sibling call (with reloc) */
1191                         add_call_dest(file, insn, reloc->sym, true);
1192                         continue;
1193                 } else if (reloc->sym->sec->idx) {
1194                         dest_sec = reloc->sym->sec;
1195                         dest_off = reloc->sym->sym.st_value +
1196                                    arch_dest_reloc_offset(reloc->addend);
1197                 } else {
1198                         /* non-func asm code jumping to another file */
1199                         continue;
1200                 }
1201
1202                 insn->jump_dest = find_insn(file, dest_sec, dest_off);
1203                 if (!insn->jump_dest) {
1204                         struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1205
1206                         /*
1207                          * This is a special case where an alt instruction
1208                          * jumps past the end of the section.  These are
1209                          * handled later in handle_group_alt().
1210                          */
1211                         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1212                                 continue;
1213
1214                         /*
1215                          * This is a special case for zen_untrain_ret().
1216                          * It jumps to __x86_return_thunk(), but objtool
1217                          * can't find the thunk's starting RET
1218                          * instruction, because the RET is also in the
1219                          * middle of another instruction.  Objtool only
1220                          * knows about the outer instruction.
1221                          */
1222                         if (sym && sym->return_thunk) {
1223                                 add_return_call(file, insn, false);
1224                                 continue;
1225                         }
1226
1227                         WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
1228                                   insn->sec, insn->offset, dest_sec->name,
1229                                   dest_off);
1230                         return -1;
1231                 }
1232
1233                 /*
1234                  * Cross-function jump.
1235                  */
1236                 if (insn->func && insn->jump_dest->func &&
1237                     insn->func != insn->jump_dest->func) {
1238
1239                         /*
1240                          * For GCC 8+, create parent/child links for any cold
1241                          * subfunctions.  This is _mostly_ redundant with a
1242                          * similar initialization in read_symbols().
1243                          *
1244                          * If a function has aliases, we want the *first* such
1245                          * function in the symbol table to be the subfunction's
1246                          * parent.  In that case we overwrite the
1247                          * initialization done in read_symbols().
1248                          *
1249                          * However this code can't completely replace the
1250                          * read_symbols() code because this doesn't detect the
1251                          * case where the parent function's only reference to a
1252                          * subfunction is through a jump table.
1253                          */
1254                         if (!strstr(insn->func->name, ".cold") &&
1255                             strstr(insn->jump_dest->func->name, ".cold")) {
1256                                 insn->func->cfunc = insn->jump_dest->func;
1257                                 insn->jump_dest->func->pfunc = insn->func;
1258
1259                         } else if (insn->jump_dest->func->pfunc != insn->func->pfunc &&
1260                                    insn->jump_dest->offset == insn->jump_dest->func->offset) {
1261                                 /* internal sibling call (without reloc) */
1262                                 add_call_dest(file, insn, insn->jump_dest->func, true);
1263                         }
1264                 }
1265         }
1266
1267         return 0;
1268 }
1269
1270 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1271 {
1272         struct symbol *call_dest;
1273
1274         call_dest = find_func_by_offset(sec, offset);
1275         if (!call_dest)
1276                 call_dest = find_symbol_by_offset(sec, offset);
1277
1278         return call_dest;
1279 }
1280
1281 /*
1282  * Find the destination instructions for all calls.
1283  */
1284 static int add_call_destinations(struct objtool_file *file)
1285 {
1286         struct instruction *insn;
1287         unsigned long dest_off;
1288         struct symbol *dest;
1289         struct reloc *reloc;
1290
1291         for_each_insn(file, insn) {
1292                 if (insn->type != INSN_CALL)
1293                         continue;
1294
1295                 reloc = insn_reloc(file, insn);
1296                 if (!reloc) {
1297                         dest_off = arch_jump_destination(insn);
1298                         dest = find_call_destination(insn->sec, dest_off);
1299
1300                         add_call_dest(file, insn, dest, false);
1301
1302                         if (insn->ignore)
1303                                 continue;
1304
1305                         if (!insn->call_dest) {
1306                                 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
1307                                 return -1;
1308                         }
1309
1310                         if (insn->func && insn->call_dest->type != STT_FUNC) {
1311                                 WARN_FUNC("unsupported call to non-function",
1312                                           insn->sec, insn->offset);
1313                                 return -1;
1314                         }
1315
1316                 } else if (reloc->sym->type == STT_SECTION) {
1317                         dest_off = arch_dest_reloc_offset(reloc->addend);
1318                         dest = find_call_destination(reloc->sym->sec, dest_off);
1319                         if (!dest) {
1320                                 WARN_FUNC("can't find call dest symbol at %s+0x%lx",
1321                                           insn->sec, insn->offset,
1322                                           reloc->sym->sec->name,
1323                                           dest_off);
1324                                 return -1;
1325                         }
1326
1327                         add_call_dest(file, insn, dest, false);
1328
1329                 } else if (reloc->sym->retpoline_thunk) {
1330                         add_retpoline_call(file, insn);
1331
1332                 } else
1333                         add_call_dest(file, insn, reloc->sym, false);
1334         }
1335
1336         return 0;
1337 }
1338
1339 /*
1340  * The .alternatives section requires some extra special care over and above
1341  * other special sections because alternatives are patched in place.
1342  */
1343 static int handle_group_alt(struct objtool_file *file,
1344                             struct special_alt *special_alt,
1345                             struct instruction *orig_insn,
1346                             struct instruction **new_insn)
1347 {
1348         struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
1349         struct alt_group *orig_alt_group, *new_alt_group;
1350         unsigned long dest_off;
1351
1352
1353         orig_alt_group = malloc(sizeof(*orig_alt_group));
1354         if (!orig_alt_group) {
1355                 WARN("malloc failed");
1356                 return -1;
1357         }
1358         orig_alt_group->cfi = calloc(special_alt->orig_len,
1359                                      sizeof(struct cfi_state *));
1360         if (!orig_alt_group->cfi) {
1361                 WARN("calloc failed");
1362                 return -1;
1363         }
1364
1365         last_orig_insn = NULL;
1366         insn = orig_insn;
1367         sec_for_each_insn_from(file, insn) {
1368                 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1369                         break;
1370
1371                 insn->alt_group = orig_alt_group;
1372                 last_orig_insn = insn;
1373         }
1374         orig_alt_group->orig_group = NULL;
1375         orig_alt_group->first_insn = orig_insn;
1376         orig_alt_group->last_insn = last_orig_insn;
1377
1378
1379         new_alt_group = malloc(sizeof(*new_alt_group));
1380         if (!new_alt_group) {
1381                 WARN("malloc failed");
1382                 return -1;
1383         }
1384
1385         if (special_alt->new_len < special_alt->orig_len) {
1386                 /*
1387                  * Insert a fake nop at the end to make the replacement
1388                  * alt_group the same size as the original.  This is needed to
1389                  * allow propagate_alt_cfi() to do its magic.  When the last
1390                  * instruction affects the stack, the instruction after it (the
1391                  * nop) will propagate the new state to the shared CFI array.
1392                  */
1393                 nop = malloc(sizeof(*nop));
1394                 if (!nop) {
1395                         WARN("malloc failed");
1396                         return -1;
1397                 }
1398                 memset(nop, 0, sizeof(*nop));
1399                 INIT_LIST_HEAD(&nop->alts);
1400                 INIT_LIST_HEAD(&nop->stack_ops);
1401
1402                 nop->sec = special_alt->new_sec;
1403                 nop->offset = special_alt->new_off + special_alt->new_len;
1404                 nop->len = special_alt->orig_len - special_alt->new_len;
1405                 nop->type = INSN_NOP;
1406                 nop->func = orig_insn->func;
1407                 nop->alt_group = new_alt_group;
1408                 nop->ignore = orig_insn->ignore_alts;
1409         }
1410
1411         if (!special_alt->new_len) {
1412                 *new_insn = nop;
1413                 goto end;
1414         }
1415
1416         insn = *new_insn;
1417         sec_for_each_insn_from(file, insn) {
1418                 struct reloc *alt_reloc;
1419
1420                 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1421                         break;
1422
1423                 last_new_insn = insn;
1424
1425                 insn->ignore = orig_insn->ignore_alts;
1426                 insn->func = orig_insn->func;
1427                 insn->alt_group = new_alt_group;
1428
1429                 /*
1430                  * Since alternative replacement code is copy/pasted by the
1431                  * kernel after applying relocations, generally such code can't
1432                  * have relative-address relocation references to outside the
1433                  * .altinstr_replacement section, unless the arch's
1434                  * alternatives code can adjust the relative offsets
1435                  * accordingly.
1436                  */
1437                 alt_reloc = insn_reloc(file, insn);
1438                 if (alt_reloc &&
1439                     !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1440
1441                         WARN_FUNC("unsupported relocation in alternatives section",
1442                                   insn->sec, insn->offset);
1443                         return -1;
1444                 }
1445
1446                 if (!is_static_jump(insn))
1447                         continue;
1448
1449                 if (!insn->immediate)
1450                         continue;
1451
1452                 dest_off = arch_jump_destination(insn);
1453                 if (dest_off == special_alt->new_off + special_alt->new_len)
1454                         insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
1455
1456                 if (!insn->jump_dest) {
1457                         WARN_FUNC("can't find alternative jump destination",
1458                                   insn->sec, insn->offset);
1459                         return -1;
1460                 }
1461         }
1462
1463         if (!last_new_insn) {
1464                 WARN_FUNC("can't find last new alternative instruction",
1465                           special_alt->new_sec, special_alt->new_off);
1466                 return -1;
1467         }
1468
1469         if (nop)
1470                 list_add(&nop->list, &last_new_insn->list);
1471 end:
1472         new_alt_group->orig_group = orig_alt_group;
1473         new_alt_group->first_insn = *new_insn;
1474         new_alt_group->last_insn = nop ? : last_new_insn;
1475         new_alt_group->cfi = orig_alt_group->cfi;
1476         return 0;
1477 }
1478
1479 /*
1480  * A jump table entry can either convert a nop to a jump or a jump to a nop.
1481  * If the original instruction is a jump, make the alt entry an effective nop
1482  * by just skipping the original instruction.
1483  */
1484 static int handle_jump_alt(struct objtool_file *file,
1485                            struct special_alt *special_alt,
1486                            struct instruction *orig_insn,
1487                            struct instruction **new_insn)
1488 {
1489         if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1490             orig_insn->type != INSN_NOP) {
1491
1492                 WARN_FUNC("unsupported instruction at jump label",
1493                           orig_insn->sec, orig_insn->offset);
1494                 return -1;
1495         }
1496
1497         if (special_alt->key_addend & 2) {
1498                 struct reloc *reloc = insn_reloc(file, orig_insn);
1499
1500                 if (reloc) {
1501                         reloc->type = R_NONE;
1502                         elf_write_reloc(file->elf, reloc);
1503                 }
1504                 elf_write_insn(file->elf, orig_insn->sec,
1505                                orig_insn->offset, orig_insn->len,
1506                                arch_nop_insn(orig_insn->len));
1507                 orig_insn->type = INSN_NOP;
1508         }
1509
1510         if (orig_insn->type == INSN_NOP) {
1511                 if (orig_insn->len == 2)
1512                         file->jl_nop_short++;
1513                 else
1514                         file->jl_nop_long++;
1515
1516                 return 0;
1517         }
1518
1519         if (orig_insn->len == 2)
1520                 file->jl_short++;
1521         else
1522                 file->jl_long++;
1523
1524         *new_insn = list_next_entry(orig_insn, list);
1525         return 0;
1526 }
1527
1528 /*
1529  * Read all the special sections which have alternate instructions which can be
1530  * patched in or redirected to at runtime.  Each instruction having alternate
1531  * instruction(s) has them added to its insn->alts list, which will be
1532  * traversed in validate_branch().
1533  */
1534 static int add_special_section_alts(struct objtool_file *file)
1535 {
1536         struct list_head special_alts;
1537         struct instruction *orig_insn, *new_insn;
1538         struct special_alt *special_alt, *tmp;
1539         struct alternative *alt;
1540         int ret;
1541
1542         ret = special_get_alts(file->elf, &special_alts);
1543         if (ret)
1544                 return ret;
1545
1546         list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1547
1548                 orig_insn = find_insn(file, special_alt->orig_sec,
1549                                       special_alt->orig_off);
1550                 if (!orig_insn) {
1551                         WARN_FUNC("special: can't find orig instruction",
1552                                   special_alt->orig_sec, special_alt->orig_off);
1553                         ret = -1;
1554                         goto out;
1555                 }
1556
1557                 new_insn = NULL;
1558                 if (!special_alt->group || special_alt->new_len) {
1559                         new_insn = find_insn(file, special_alt->new_sec,
1560                                              special_alt->new_off);
1561                         if (!new_insn) {
1562                                 WARN_FUNC("special: can't find new instruction",
1563                                           special_alt->new_sec,
1564                                           special_alt->new_off);
1565                                 ret = -1;
1566                                 goto out;
1567                         }
1568                 }
1569
1570                 if (special_alt->group) {
1571                         if (!special_alt->orig_len) {
1572                                 WARN_FUNC("empty alternative entry",
1573                                           orig_insn->sec, orig_insn->offset);
1574                                 continue;
1575                         }
1576
1577                         ret = handle_group_alt(file, special_alt, orig_insn,
1578                                                &new_insn);
1579                         if (ret)
1580                                 goto out;
1581                 } else if (special_alt->jump_or_nop) {
1582                         ret = handle_jump_alt(file, special_alt, orig_insn,
1583                                               &new_insn);
1584                         if (ret)
1585                                 goto out;
1586                 }
1587
1588                 alt = malloc(sizeof(*alt));
1589                 if (!alt) {
1590                         WARN("malloc failed");
1591                         ret = -1;
1592                         goto out;
1593                 }
1594
1595                 alt->insn = new_insn;
1596                 alt->skip_orig = special_alt->skip_orig;
1597                 orig_insn->ignore_alts |= special_alt->skip_alt;
1598                 list_add_tail(&alt->list, &orig_insn->alts);
1599
1600                 list_del(&special_alt->list);
1601                 free(special_alt);
1602         }
1603
1604         if (stats) {
1605                 printf("jl\\\tNOP\tJMP\n");
1606                 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1607                 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1608         }
1609
1610 out:
1611         return ret;
1612 }
1613
1614 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
1615                             struct reloc *table)
1616 {
1617         struct reloc *reloc = table;
1618         struct instruction *dest_insn;
1619         struct alternative *alt;
1620         struct symbol *pfunc = insn->func->pfunc;
1621         unsigned int prev_offset = 0;
1622
1623         /*
1624          * Each @reloc is a switch table relocation which points to the target
1625          * instruction.
1626          */
1627         list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
1628
1629                 /* Check for the end of the table: */
1630                 if (reloc != table && reloc->jump_table_start)
1631                         break;
1632
1633                 /* Make sure the table entries are consecutive: */
1634                 if (prev_offset && reloc->offset != prev_offset + 8)
1635                         break;
1636
1637                 /* Detect function pointers from contiguous objects: */
1638                 if (reloc->sym->sec == pfunc->sec &&
1639                     reloc->addend == pfunc->offset)
1640                         break;
1641
1642                 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
1643                 if (!dest_insn)
1644                         break;
1645
1646                 /* Make sure the destination is in the same function: */
1647                 if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
1648                         break;
1649
1650                 alt = malloc(sizeof(*alt));
1651                 if (!alt) {
1652                         WARN("malloc failed");
1653                         return -1;
1654                 }
1655
1656                 alt->insn = dest_insn;
1657                 list_add_tail(&alt->list, &insn->alts);
1658                 prev_offset = reloc->offset;
1659         }
1660
1661         if (!prev_offset) {
1662                 WARN_FUNC("can't find switch jump table",
1663                           insn->sec, insn->offset);
1664                 return -1;
1665         }
1666
1667         return 0;
1668 }
1669
1670 /*
1671  * find_jump_table() - Given a dynamic jump, find the switch jump table
1672  * associated with it.
1673  */
1674 static struct reloc *find_jump_table(struct objtool_file *file,
1675                                       struct symbol *func,
1676                                       struct instruction *insn)
1677 {
1678         struct reloc *table_reloc;
1679         struct instruction *dest_insn, *orig_insn = insn;
1680
1681         /*
1682          * Backward search using the @first_jump_src links, these help avoid
1683          * much of the 'in between' code. Which avoids us getting confused by
1684          * it.
1685          */
1686         for (;
1687              insn && insn->func && insn->func->pfunc == func;
1688              insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
1689
1690                 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
1691                         break;
1692
1693                 /* allow small jumps within the range */
1694                 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
1695                     insn->jump_dest &&
1696                     (insn->jump_dest->offset <= insn->offset ||
1697                      insn->jump_dest->offset > orig_insn->offset))
1698                     break;
1699
1700                 table_reloc = arch_find_switch_table(file, insn);
1701                 if (!table_reloc)
1702                         continue;
1703                 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
1704                 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
1705                         continue;
1706
1707                 return table_reloc;
1708         }
1709
1710         return NULL;
1711 }
1712
1713 /*
1714  * First pass: Mark the head of each jump table so that in the next pass,
1715  * we know when a given jump table ends and the next one starts.
1716  */
1717 static void mark_func_jump_tables(struct objtool_file *file,
1718                                     struct symbol *func)
1719 {
1720         struct instruction *insn, *last = NULL;
1721         struct reloc *reloc;
1722
1723         func_for_each_insn(file, func, insn) {
1724                 if (!last)
1725                         last = insn;
1726
1727                 /*
1728                  * Store back-pointers for unconditional forward jumps such
1729                  * that find_jump_table() can back-track using those and
1730                  * avoid some potentially confusing code.
1731                  */
1732                 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
1733                     insn->offset > last->offset &&
1734                     insn->jump_dest->offset > insn->offset &&
1735                     !insn->jump_dest->first_jump_src) {
1736
1737                         insn->jump_dest->first_jump_src = insn;
1738                         last = insn->jump_dest;
1739                 }
1740
1741                 if (insn->type != INSN_JUMP_DYNAMIC)
1742                         continue;
1743
1744                 reloc = find_jump_table(file, func, insn);
1745                 if (reloc) {
1746                         reloc->jump_table_start = true;
1747                         insn->jump_table = reloc;
1748                 }
1749         }
1750 }
1751
1752 static int add_func_jump_tables(struct objtool_file *file,
1753                                   struct symbol *func)
1754 {
1755         struct instruction *insn;
1756         int ret;
1757
1758         func_for_each_insn(file, func, insn) {
1759                 if (!insn->jump_table)
1760                         continue;
1761
1762                 ret = add_jump_table(file, insn, insn->jump_table);
1763                 if (ret)
1764                         return ret;
1765         }
1766
1767         return 0;
1768 }
1769
1770 /*
1771  * For some switch statements, gcc generates a jump table in the .rodata
1772  * section which contains a list of addresses within the function to jump to.
1773  * This finds these jump tables and adds them to the insn->alts lists.
1774  */
1775 static int add_jump_table_alts(struct objtool_file *file)
1776 {
1777         struct section *sec;
1778         struct symbol *func;
1779         int ret;
1780
1781         if (!file->rodata)
1782                 return 0;
1783
1784         for_each_sec(file, sec) {
1785                 list_for_each_entry(func, &sec->symbol_list, list) {
1786                         if (func->type != STT_FUNC)
1787                                 continue;
1788
1789                         mark_func_jump_tables(file, func);
1790                         ret = add_func_jump_tables(file, func);
1791                         if (ret)
1792                                 return ret;
1793                 }
1794         }
1795
1796         return 0;
1797 }
1798
1799 static void set_func_state(struct cfi_state *state)
1800 {
1801         state->cfa = initial_func_cfi.cfa;
1802         memcpy(&state->regs, &initial_func_cfi.regs,
1803                CFI_NUM_REGS * sizeof(struct cfi_reg));
1804         state->stack_size = initial_func_cfi.cfa.offset;
1805 }
1806
1807 static int read_unwind_hints(struct objtool_file *file)
1808 {
1809         struct cfi_state cfi = init_cfi;
1810         struct section *sec, *relocsec;
1811         struct unwind_hint *hint;
1812         struct instruction *insn;
1813         struct reloc *reloc;
1814         int i;
1815
1816         sec = find_section_by_name(file->elf, ".discard.unwind_hints");
1817         if (!sec)
1818                 return 0;
1819
1820         relocsec = sec->reloc;
1821         if (!relocsec) {
1822                 WARN("missing .rela.discard.unwind_hints section");
1823                 return -1;
1824         }
1825
1826         if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
1827                 WARN("struct unwind_hint size mismatch");
1828                 return -1;
1829         }
1830
1831         file->hints = true;
1832
1833         for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
1834                 hint = (struct unwind_hint *)sec->data->d_buf + i;
1835
1836                 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
1837                 if (!reloc) {
1838                         WARN("can't find reloc for unwind_hints[%d]", i);
1839                         return -1;
1840                 }
1841
1842                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1843                 if (!insn) {
1844                         WARN("can't find insn for unwind_hints[%d]", i);
1845                         return -1;
1846                 }
1847
1848                 insn->hint = true;
1849
1850                 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
1851                         insn->hint = false;
1852                         insn->save = true;
1853                         continue;
1854                 }
1855
1856                 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
1857                         insn->restore = true;
1858                         continue;
1859                 }
1860
1861                 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
1862                         struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
1863
1864                         if (sym && sym->bind == STB_GLOBAL) {
1865                                 insn->entry = 1;
1866                         }
1867                 }
1868
1869                 if (hint->type == UNWIND_HINT_TYPE_ENTRY) {
1870                         hint->type = UNWIND_HINT_TYPE_CALL;
1871                         insn->entry = 1;
1872                 }
1873
1874                 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
1875                         insn->cfi = &func_cfi;
1876                         continue;
1877                 }
1878
1879                 if (insn->cfi)
1880                         cfi = *(insn->cfi);
1881
1882                 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
1883                         WARN_FUNC("unsupported unwind_hint sp base reg %d",
1884                                   insn->sec, insn->offset, hint->sp_reg);
1885                         return -1;
1886                 }
1887
1888                 cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
1889                 cfi.type = hint->type;
1890                 cfi.end = hint->end;
1891
1892                 insn->cfi = cfi_hash_find_or_add(&cfi);
1893         }
1894
1895         return 0;
1896 }
1897
1898 static int read_retpoline_hints(struct objtool_file *file)
1899 {
1900         struct section *sec;
1901         struct instruction *insn;
1902         struct reloc *reloc;
1903
1904         sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
1905         if (!sec)
1906                 return 0;
1907
1908         list_for_each_entry(reloc, &sec->reloc_list, list) {
1909                 if (reloc->sym->type != STT_SECTION) {
1910                         WARN("unexpected relocation symbol type in %s", sec->name);
1911                         return -1;
1912                 }
1913
1914                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1915                 if (!insn) {
1916                         WARN("bad .discard.retpoline_safe entry");
1917                         return -1;
1918                 }
1919
1920                 if (insn->type != INSN_JUMP_DYNAMIC &&
1921                     insn->type != INSN_CALL_DYNAMIC &&
1922                     insn->type != INSN_RETURN &&
1923                     insn->type != INSN_NOP) {
1924                         WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop",
1925                                   insn->sec, insn->offset);
1926                         return -1;
1927                 }
1928
1929                 insn->retpoline_safe = true;
1930         }
1931
1932         return 0;
1933 }
1934
1935 static int read_instr_hints(struct objtool_file *file)
1936 {
1937         struct section *sec;
1938         struct instruction *insn;
1939         struct reloc *reloc;
1940
1941         sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
1942         if (!sec)
1943                 return 0;
1944
1945         list_for_each_entry(reloc, &sec->reloc_list, list) {
1946                 if (reloc->sym->type != STT_SECTION) {
1947                         WARN("unexpected relocation symbol type in %s", sec->name);
1948                         return -1;
1949                 }
1950
1951                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1952                 if (!insn) {
1953                         WARN("bad .discard.instr_end entry");
1954                         return -1;
1955                 }
1956
1957                 insn->instr--;
1958         }
1959
1960         sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
1961         if (!sec)
1962                 return 0;
1963
1964         list_for_each_entry(reloc, &sec->reloc_list, list) {
1965                 if (reloc->sym->type != STT_SECTION) {
1966                         WARN("unexpected relocation symbol type in %s", sec->name);
1967                         return -1;
1968                 }
1969
1970                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1971                 if (!insn) {
1972                         WARN("bad .discard.instr_begin entry");
1973                         return -1;
1974                 }
1975
1976                 insn->instr++;
1977         }
1978
1979         return 0;
1980 }
1981
1982 static int read_intra_function_calls(struct objtool_file *file)
1983 {
1984         struct instruction *insn;
1985         struct section *sec;
1986         struct reloc *reloc;
1987
1988         sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
1989         if (!sec)
1990                 return 0;
1991
1992         list_for_each_entry(reloc, &sec->reloc_list, list) {
1993                 unsigned long dest_off;
1994
1995                 if (reloc->sym->type != STT_SECTION) {
1996                         WARN("unexpected relocation symbol type in %s",
1997                              sec->name);
1998                         return -1;
1999                 }
2000
2001                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
2002                 if (!insn) {
2003                         WARN("bad .discard.intra_function_call entry");
2004                         return -1;
2005                 }
2006
2007                 if (insn->type != INSN_CALL) {
2008                         WARN_FUNC("intra_function_call not a direct call",
2009                                   insn->sec, insn->offset);
2010                         return -1;
2011                 }
2012
2013                 /*
2014                  * Treat intra-function CALLs as JMPs, but with a stack_op.
2015                  * See add_call_destinations(), which strips stack_ops from
2016                  * normal CALLs.
2017                  */
2018                 insn->type = INSN_JUMP_UNCONDITIONAL;
2019
2020                 dest_off = insn->offset + insn->len + insn->immediate;
2021                 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2022                 if (!insn->jump_dest) {
2023                         WARN_FUNC("can't find call dest at %s+0x%lx",
2024                                   insn->sec, insn->offset,
2025                                   insn->sec->name, dest_off);
2026                         return -1;
2027                 }
2028         }
2029
2030         return 0;
2031 }
2032
2033 static int classify_symbols(struct objtool_file *file)
2034 {
2035         struct section *sec;
2036         struct symbol *func;
2037
2038         for_each_sec(file, sec) {
2039                 list_for_each_entry(func, &sec->symbol_list, list) {
2040                         if (func->bind != STB_GLOBAL)
2041                                 continue;
2042
2043                         if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2044                                      strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2045                                 func->static_call_tramp = true;
2046
2047                         if (arch_is_retpoline(func))
2048                                 func->retpoline_thunk = true;
2049
2050                         if (arch_is_rethunk(func))
2051                                 func->return_thunk = true;
2052
2053                         if (!strcmp(func->name, "__fentry__"))
2054                                 func->fentry = true;
2055
2056                         if (!strncmp(func->name, "__sanitizer_cov_", 16))
2057                                 func->kcov = true;
2058                 }
2059         }
2060
2061         return 0;
2062 }
2063
2064 static void mark_rodata(struct objtool_file *file)
2065 {
2066         struct section *sec;
2067         bool found = false;
2068
2069         /*
2070          * Search for the following rodata sections, each of which can
2071          * potentially contain jump tables:
2072          *
2073          * - .rodata: can contain GCC switch tables
2074          * - .rodata.<func>: same, if -fdata-sections is being used
2075          * - .rodata..c_jump_table: contains C annotated jump tables
2076          *
2077          * .rodata.str1.* sections are ignored; they don't contain jump tables.
2078          */
2079         for_each_sec(file, sec) {
2080                 if (!strncmp(sec->name, ".rodata", 7) &&
2081                     !strstr(sec->name, ".str1.")) {
2082                         sec->rodata = true;
2083                         found = true;
2084                 }
2085         }
2086
2087         file->rodata = found;
2088 }
2089
2090 static int decode_sections(struct objtool_file *file)
2091 {
2092         int ret;
2093
2094         mark_rodata(file);
2095
2096         ret = decode_instructions(file);
2097         if (ret)
2098                 return ret;
2099
2100         ret = add_dead_ends(file);
2101         if (ret)
2102                 return ret;
2103
2104         add_ignores(file);
2105         add_uaccess_safe(file);
2106
2107         ret = add_ignore_alternatives(file);
2108         if (ret)
2109                 return ret;
2110
2111         /*
2112          * Must be before add_{jump_call}_destination.
2113          */
2114         ret = classify_symbols(file);
2115         if (ret)
2116                 return ret;
2117
2118         /*
2119          * Must be before add_special_section_alts() as that depends on
2120          * jump_dest being set.
2121          */
2122         ret = add_jump_destinations(file);
2123         if (ret)
2124                 return ret;
2125
2126         ret = add_special_section_alts(file);
2127         if (ret)
2128                 return ret;
2129
2130         /*
2131          * Must be before add_call_destination(); it changes INSN_CALL to
2132          * INSN_JUMP.
2133          */
2134         ret = read_intra_function_calls(file);
2135         if (ret)
2136                 return ret;
2137
2138         ret = add_call_destinations(file);
2139         if (ret)
2140                 return ret;
2141
2142         ret = add_jump_table_alts(file);
2143         if (ret)
2144                 return ret;
2145
2146         ret = read_unwind_hints(file);
2147         if (ret)
2148                 return ret;
2149
2150         ret = read_retpoline_hints(file);
2151         if (ret)
2152                 return ret;
2153
2154         ret = read_instr_hints(file);
2155         if (ret)
2156                 return ret;
2157
2158         return 0;
2159 }
2160
2161 static bool is_fentry_call(struct instruction *insn)
2162 {
2163         if (insn->type == INSN_CALL &&
2164             insn->call_dest &&
2165             insn->call_dest->fentry)
2166                 return true;
2167
2168         return false;
2169 }
2170
2171 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2172 {
2173         struct cfi_state *cfi = &state->cfi;
2174         int i;
2175
2176         if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2177                 return true;
2178
2179         if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2180                 return true;
2181
2182         if (cfi->stack_size != initial_func_cfi.cfa.offset)
2183                 return true;
2184
2185         for (i = 0; i < CFI_NUM_REGS; i++) {
2186                 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2187                     cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2188                         return true;
2189         }
2190
2191         return false;
2192 }
2193
2194 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2195                                 int expected_offset)
2196 {
2197         return reg->base == CFI_CFA &&
2198                reg->offset == expected_offset;
2199 }
2200
2201 static bool has_valid_stack_frame(struct insn_state *state)
2202 {
2203         struct cfi_state *cfi = &state->cfi;
2204
2205         if (cfi->cfa.base == CFI_BP &&
2206             check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2207             check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2208                 return true;
2209
2210         if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2211                 return true;
2212
2213         return false;
2214 }
2215
2216 static int update_cfi_state_regs(struct instruction *insn,
2217                                   struct cfi_state *cfi,
2218                                   struct stack_op *op)
2219 {
2220         struct cfi_reg *cfa = &cfi->cfa;
2221
2222         if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2223                 return 0;
2224
2225         /* push */
2226         if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2227                 cfa->offset += 8;
2228
2229         /* pop */
2230         if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2231                 cfa->offset -= 8;
2232
2233         /* add immediate to sp */
2234         if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2235             op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2236                 cfa->offset -= op->src.offset;
2237
2238         return 0;
2239 }
2240
2241 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2242 {
2243         if (arch_callee_saved_reg(reg) &&
2244             cfi->regs[reg].base == CFI_UNDEFINED) {
2245                 cfi->regs[reg].base = base;
2246                 cfi->regs[reg].offset = offset;
2247         }
2248 }
2249
2250 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2251 {
2252         cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2253         cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2254 }
2255
2256 /*
2257  * A note about DRAP stack alignment:
2258  *
2259  * GCC has the concept of a DRAP register, which is used to help keep track of
2260  * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2261  * register.  The typical DRAP pattern is:
2262  *
2263  *   4c 8d 54 24 08             lea    0x8(%rsp),%r10
2264  *   48 83 e4 c0                and    $0xffffffffffffffc0,%rsp
2265  *   41 ff 72 f8                pushq  -0x8(%r10)
2266  *   55                         push   %rbp
2267  *   48 89 e5                   mov    %rsp,%rbp
2268  *                              (more pushes)
2269  *   41 52                      push   %r10
2270  *                              ...
2271  *   41 5a                      pop    %r10
2272  *                              (more pops)
2273  *   5d                         pop    %rbp
2274  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2275  *   c3                         retq
2276  *
2277  * There are some variations in the epilogues, like:
2278  *
2279  *   5b                         pop    %rbx
2280  *   41 5a                      pop    %r10
2281  *   41 5c                      pop    %r12
2282  *   41 5d                      pop    %r13
2283  *   41 5e                      pop    %r14
2284  *   c9                         leaveq
2285  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2286  *   c3                         retq
2287  *
2288  * and:
2289  *
2290  *   4c 8b 55 e8                mov    -0x18(%rbp),%r10
2291  *   48 8b 5d e0                mov    -0x20(%rbp),%rbx
2292  *   4c 8b 65 f0                mov    -0x10(%rbp),%r12
2293  *   4c 8b 6d f8                mov    -0x8(%rbp),%r13
2294  *   c9                         leaveq
2295  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2296  *   c3                         retq
2297  *
2298  * Sometimes r13 is used as the DRAP register, in which case it's saved and
2299  * restored beforehand:
2300  *
2301  *   41 55                      push   %r13
2302  *   4c 8d 6c 24 10             lea    0x10(%rsp),%r13
2303  *   48 83 e4 f0                and    $0xfffffffffffffff0,%rsp
2304  *                              ...
2305  *   49 8d 65 f0                lea    -0x10(%r13),%rsp
2306  *   41 5d                      pop    %r13
2307  *   c3                         retq
2308  */
2309 static int update_cfi_state(struct instruction *insn,
2310                             struct instruction *next_insn,
2311                             struct cfi_state *cfi, struct stack_op *op)
2312 {
2313         struct cfi_reg *cfa = &cfi->cfa;
2314         struct cfi_reg *regs = cfi->regs;
2315
2316         /* stack operations don't make sense with an undefined CFA */
2317         if (cfa->base == CFI_UNDEFINED) {
2318                 if (insn->func) {
2319                         WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2320                         return -1;
2321                 }
2322                 return 0;
2323         }
2324
2325         if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2326             cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2327                 return update_cfi_state_regs(insn, cfi, op);
2328
2329         switch (op->dest.type) {
2330
2331         case OP_DEST_REG:
2332                 switch (op->src.type) {
2333
2334                 case OP_SRC_REG:
2335                         if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2336                             cfa->base == CFI_SP &&
2337                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2338
2339                                 /* mov %rsp, %rbp */
2340                                 cfa->base = op->dest.reg;
2341                                 cfi->bp_scratch = false;
2342                         }
2343
2344                         else if (op->src.reg == CFI_SP &&
2345                                  op->dest.reg == CFI_BP && cfi->drap) {
2346
2347                                 /* drap: mov %rsp, %rbp */
2348                                 regs[CFI_BP].base = CFI_BP;
2349                                 regs[CFI_BP].offset = -cfi->stack_size;
2350                                 cfi->bp_scratch = false;
2351                         }
2352
2353                         else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2354
2355                                 /*
2356                                  * mov %rsp, %reg
2357                                  *
2358                                  * This is needed for the rare case where GCC
2359                                  * does:
2360                                  *
2361                                  *   mov    %rsp, %rax
2362                                  *   ...
2363                                  *   mov    %rax, %rsp
2364                                  */
2365                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2366                                 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2367                         }
2368
2369                         else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2370                                  (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2371
2372                                 /*
2373                                  * mov %rbp, %rsp
2374                                  *
2375                                  * Restore the original stack pointer (Clang).
2376                                  */
2377                                 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2378                         }
2379
2380                         else if (op->dest.reg == cfa->base) {
2381
2382                                 /* mov %reg, %rsp */
2383                                 if (cfa->base == CFI_SP &&
2384                                     cfi->vals[op->src.reg].base == CFI_CFA) {
2385
2386                                         /*
2387                                          * This is needed for the rare case
2388                                          * where GCC does something dumb like:
2389                                          *
2390                                          *   lea    0x8(%rsp), %rcx
2391                                          *   ...
2392                                          *   mov    %rcx, %rsp
2393                                          */
2394                                         cfa->offset = -cfi->vals[op->src.reg].offset;
2395                                         cfi->stack_size = cfa->offset;
2396
2397                                 } else if (cfa->base == CFI_SP &&
2398                                            cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2399                                            cfi->vals[op->src.reg].offset == cfa->offset) {
2400
2401                                         /*
2402                                          * Stack swizzle:
2403                                          *
2404                                          * 1: mov %rsp, (%[tos])
2405                                          * 2: mov %[tos], %rsp
2406                                          *    ...
2407                                          * 3: pop %rsp
2408                                          *
2409                                          * Where:
2410                                          *
2411                                          * 1 - places a pointer to the previous
2412                                          *     stack at the Top-of-Stack of the
2413                                          *     new stack.
2414                                          *
2415                                          * 2 - switches to the new stack.
2416                                          *
2417                                          * 3 - pops the Top-of-Stack to restore
2418                                          *     the original stack.
2419                                          *
2420                                          * Note: we set base to SP_INDIRECT
2421                                          * here and preserve offset. Therefore
2422                                          * when the unwinder reaches ToS it
2423                                          * will dereference SP and then add the
2424                                          * offset to find the next frame, IOW:
2425                                          * (%rsp) + offset.
2426                                          */
2427                                         cfa->base = CFI_SP_INDIRECT;
2428
2429                                 } else {
2430                                         cfa->base = CFI_UNDEFINED;
2431                                         cfa->offset = 0;
2432                                 }
2433                         }
2434
2435                         else if (op->dest.reg == CFI_SP &&
2436                                  cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2437                                  cfi->vals[op->src.reg].offset == cfa->offset) {
2438
2439                                 /*
2440                                  * The same stack swizzle case 2) as above. But
2441                                  * because we can't change cfa->base, case 3)
2442                                  * will become a regular POP. Pretend we're a
2443                                  * PUSH so things don't go unbalanced.
2444                                  */
2445                                 cfi->stack_size += 8;
2446                         }
2447
2448
2449                         break;
2450
2451                 case OP_SRC_ADD:
2452                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2453
2454                                 /* add imm, %rsp */
2455                                 cfi->stack_size -= op->src.offset;
2456                                 if (cfa->base == CFI_SP)
2457                                         cfa->offset -= op->src.offset;
2458                                 break;
2459                         }
2460
2461                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2462
2463                                 /* lea disp(%rbp), %rsp */
2464                                 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2465                                 break;
2466                         }
2467
2468                         if (!cfi->drap && op->src.reg == CFI_SP &&
2469                             op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2470                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset + op->src.offset)) {
2471
2472                                 /* lea disp(%rsp), %rbp */
2473                                 cfa->base = CFI_BP;
2474                                 cfa->offset -= op->src.offset;
2475                                 cfi->bp_scratch = false;
2476                                 break;
2477                         }
2478
2479                         if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2480
2481                                 /* drap: lea disp(%rsp), %drap */
2482                                 cfi->drap_reg = op->dest.reg;
2483
2484                                 /*
2485                                  * lea disp(%rsp), %reg
2486                                  *
2487                                  * This is needed for the rare case where GCC
2488                                  * does something dumb like:
2489                                  *
2490                                  *   lea    0x8(%rsp), %rcx
2491                                  *   ...
2492                                  *   mov    %rcx, %rsp
2493                                  */
2494                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2495                                 cfi->vals[op->dest.reg].offset = \
2496                                         -cfi->stack_size + op->src.offset;
2497
2498                                 break;
2499                         }
2500
2501                         if (cfi->drap && op->dest.reg == CFI_SP &&
2502                             op->src.reg == cfi->drap_reg) {
2503
2504                                  /* drap: lea disp(%drap), %rsp */
2505                                 cfa->base = CFI_SP;
2506                                 cfa->offset = cfi->stack_size = -op->src.offset;
2507                                 cfi->drap_reg = CFI_UNDEFINED;
2508                                 cfi->drap = false;
2509                                 break;
2510                         }
2511
2512                         if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2513                                 WARN_FUNC("unsupported stack register modification",
2514                                           insn->sec, insn->offset);
2515                                 return -1;
2516                         }
2517
2518                         break;
2519
2520                 case OP_SRC_AND:
2521                         if (op->dest.reg != CFI_SP ||
2522                             (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2523                             (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2524                                 WARN_FUNC("unsupported stack pointer realignment",
2525                                           insn->sec, insn->offset);
2526                                 return -1;
2527                         }
2528
2529                         if (cfi->drap_reg != CFI_UNDEFINED) {
2530                                 /* drap: and imm, %rsp */
2531                                 cfa->base = cfi->drap_reg;
2532                                 cfa->offset = cfi->stack_size = 0;
2533                                 cfi->drap = true;
2534                         }
2535
2536                         /*
2537                          * Older versions of GCC (4.8ish) realign the stack
2538                          * without DRAP, with a frame pointer.
2539                          */
2540
2541                         break;
2542
2543                 case OP_SRC_POP:
2544                 case OP_SRC_POPF:
2545                         if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2546
2547                                 /* pop %rsp; # restore from a stack swizzle */
2548                                 cfa->base = CFI_SP;
2549                                 break;
2550                         }
2551
2552                         if (!cfi->drap && op->dest.reg == cfa->base) {
2553
2554                                 /* pop %rbp */
2555                                 cfa->base = CFI_SP;
2556                         }
2557
2558                         if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2559                             op->dest.reg == cfi->drap_reg &&
2560                             cfi->drap_offset == -cfi->stack_size) {
2561
2562                                 /* drap: pop %drap */
2563                                 cfa->base = cfi->drap_reg;
2564                                 cfa->offset = 0;
2565                                 cfi->drap_offset = -1;
2566
2567                         } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
2568
2569                                 /* pop %reg */
2570                                 restore_reg(cfi, op->dest.reg);
2571                         }
2572
2573                         cfi->stack_size -= 8;
2574                         if (cfa->base == CFI_SP)
2575                                 cfa->offset -= 8;
2576
2577                         break;
2578
2579                 case OP_SRC_REG_INDIRECT:
2580                         if (!cfi->drap && op->dest.reg == cfa->base &&
2581                             op->dest.reg == CFI_BP) {
2582
2583                                 /* mov disp(%rsp), %rbp */
2584                                 cfa->base = CFI_SP;
2585                                 cfa->offset = cfi->stack_size;
2586                         }
2587
2588                         if (cfi->drap && op->src.reg == CFI_BP &&
2589                             op->src.offset == cfi->drap_offset) {
2590
2591                                 /* drap: mov disp(%rbp), %drap */
2592                                 cfa->base = cfi->drap_reg;
2593                                 cfa->offset = 0;
2594                                 cfi->drap_offset = -1;
2595                         }
2596
2597                         if (cfi->drap && op->src.reg == CFI_BP &&
2598                             op->src.offset == regs[op->dest.reg].offset) {
2599
2600                                 /* drap: mov disp(%rbp), %reg */
2601                                 restore_reg(cfi, op->dest.reg);
2602
2603                         } else if (op->src.reg == cfa->base &&
2604                             op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
2605
2606                                 /* mov disp(%rbp), %reg */
2607                                 /* mov disp(%rsp), %reg */
2608                                 restore_reg(cfi, op->dest.reg);
2609
2610                         } else if (op->src.reg == CFI_SP &&
2611                                    op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
2612
2613                                 /* mov disp(%rsp), %reg */
2614                                 restore_reg(cfi, op->dest.reg);
2615                         }
2616
2617                         break;
2618
2619                 default:
2620                         WARN_FUNC("unknown stack-related instruction",
2621                                   insn->sec, insn->offset);
2622                         return -1;
2623                 }
2624
2625                 break;
2626
2627         case OP_DEST_PUSH:
2628         case OP_DEST_PUSHF:
2629                 cfi->stack_size += 8;
2630                 if (cfa->base == CFI_SP)
2631                         cfa->offset += 8;
2632
2633                 if (op->src.type != OP_SRC_REG)
2634                         break;
2635
2636                 if (cfi->drap) {
2637                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2638
2639                                 /* drap: push %drap */
2640                                 cfa->base = CFI_BP_INDIRECT;
2641                                 cfa->offset = -cfi->stack_size;
2642
2643                                 /* save drap so we know when to restore it */
2644                                 cfi->drap_offset = -cfi->stack_size;
2645
2646                         } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
2647
2648                                 /* drap: push %rbp */
2649                                 cfi->stack_size = 0;
2650
2651                         } else {
2652
2653                                 /* drap: push %reg */
2654                                 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
2655                         }
2656
2657                 } else {
2658
2659                         /* push %reg */
2660                         save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
2661                 }
2662
2663                 /* detect when asm code uses rbp as a scratch register */
2664                 if (!no_fp && insn->func && op->src.reg == CFI_BP &&
2665                     cfa->base != CFI_BP)
2666                         cfi->bp_scratch = true;
2667                 break;
2668
2669         case OP_DEST_REG_INDIRECT:
2670
2671                 if (cfi->drap) {
2672                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2673
2674                                 /* drap: mov %drap, disp(%rbp) */
2675                                 cfa->base = CFI_BP_INDIRECT;
2676                                 cfa->offset = op->dest.offset;
2677
2678                                 /* save drap offset so we know when to restore it */
2679                                 cfi->drap_offset = op->dest.offset;
2680                         } else {
2681
2682                                 /* drap: mov reg, disp(%rbp) */
2683                                 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
2684                         }
2685
2686                 } else if (op->dest.reg == cfa->base) {
2687
2688                         /* mov reg, disp(%rbp) */
2689                         /* mov reg, disp(%rsp) */
2690                         save_reg(cfi, op->src.reg, CFI_CFA,
2691                                  op->dest.offset - cfi->cfa.offset);
2692
2693                 } else if (op->dest.reg == CFI_SP) {
2694
2695                         /* mov reg, disp(%rsp) */
2696                         save_reg(cfi, op->src.reg, CFI_CFA,
2697                                  op->dest.offset - cfi->stack_size);
2698
2699                 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
2700
2701                         /* mov %rsp, (%reg); # setup a stack swizzle. */
2702                         cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
2703                         cfi->vals[op->dest.reg].offset = cfa->offset;
2704                 }
2705
2706                 break;
2707
2708         case OP_DEST_MEM:
2709                 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
2710                         WARN_FUNC("unknown stack-related memory operation",
2711                                   insn->sec, insn->offset);
2712                         return -1;
2713                 }
2714
2715                 /* pop mem */
2716                 cfi->stack_size -= 8;
2717                 if (cfa->base == CFI_SP)
2718                         cfa->offset -= 8;
2719
2720                 break;
2721
2722         default:
2723                 WARN_FUNC("unknown stack-related instruction",
2724                           insn->sec, insn->offset);
2725                 return -1;
2726         }
2727
2728         return 0;
2729 }
2730
2731 /*
2732  * The stack layouts of alternatives instructions can sometimes diverge when
2733  * they have stack modifications.  That's fine as long as the potential stack
2734  * layouts don't conflict at any given potential instruction boundary.
2735  *
2736  * Flatten the CFIs of the different alternative code streams (both original
2737  * and replacement) into a single shared CFI array which can be used to detect
2738  * conflicts and nicely feed a linear array of ORC entries to the unwinder.
2739  */
2740 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
2741 {
2742         struct cfi_state **alt_cfi;
2743         int group_off;
2744
2745         if (!insn->alt_group)
2746                 return 0;
2747
2748         if (!insn->cfi) {
2749                 WARN("CFI missing");
2750                 return -1;
2751         }
2752
2753         alt_cfi = insn->alt_group->cfi;
2754         group_off = insn->offset - insn->alt_group->first_insn->offset;
2755
2756         if (!alt_cfi[group_off]) {
2757                 alt_cfi[group_off] = insn->cfi;
2758         } else {
2759                 if (cficmp(alt_cfi[group_off], insn->cfi)) {
2760                         WARN_FUNC("stack layout conflict in alternatives",
2761                                   insn->sec, insn->offset);
2762                         return -1;
2763                 }
2764         }
2765
2766         return 0;
2767 }
2768
2769 static int handle_insn_ops(struct instruction *insn,
2770                            struct instruction *next_insn,
2771                            struct insn_state *state)
2772 {
2773         struct stack_op *op;
2774
2775         list_for_each_entry(op, &insn->stack_ops, list) {
2776
2777                 if (update_cfi_state(insn, next_insn, &state->cfi, op))
2778                         return 1;
2779
2780                 if (!insn->alt_group)
2781                         continue;
2782
2783                 if (op->dest.type == OP_DEST_PUSHF) {
2784                         if (!state->uaccess_stack) {
2785                                 state->uaccess_stack = 1;
2786                         } else if (state->uaccess_stack >> 31) {
2787                                 WARN_FUNC("PUSHF stack exhausted",
2788                                           insn->sec, insn->offset);
2789                                 return 1;
2790                         }
2791                         state->uaccess_stack <<= 1;
2792                         state->uaccess_stack  |= state->uaccess;
2793                 }
2794
2795                 if (op->src.type == OP_SRC_POPF) {
2796                         if (state->uaccess_stack) {
2797                                 state->uaccess = state->uaccess_stack & 1;
2798                                 state->uaccess_stack >>= 1;
2799                                 if (state->uaccess_stack == 1)
2800                                         state->uaccess_stack = 0;
2801                         }
2802                 }
2803         }
2804
2805         return 0;
2806 }
2807
2808 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
2809 {
2810         struct cfi_state *cfi1 = insn->cfi;
2811         int i;
2812
2813         if (!cfi1) {
2814                 WARN("CFI missing");
2815                 return false;
2816         }
2817
2818         if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
2819
2820                 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
2821                           insn->sec, insn->offset,
2822                           cfi1->cfa.base, cfi1->cfa.offset,
2823                           cfi2->cfa.base, cfi2->cfa.offset);
2824
2825         } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
2826                 for (i = 0; i < CFI_NUM_REGS; i++) {
2827                         if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
2828                                     sizeof(struct cfi_reg)))
2829                                 continue;
2830
2831                         WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
2832                                   insn->sec, insn->offset,
2833                                   i, cfi1->regs[i].base, cfi1->regs[i].offset,
2834                                   i, cfi2->regs[i].base, cfi2->regs[i].offset);
2835                         break;
2836                 }
2837
2838         } else if (cfi1->type != cfi2->type) {
2839
2840                 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
2841                           insn->sec, insn->offset, cfi1->type, cfi2->type);
2842
2843         } else if (cfi1->drap != cfi2->drap ||
2844                    (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
2845                    (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
2846
2847                 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
2848                           insn->sec, insn->offset,
2849                           cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
2850                           cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
2851
2852         } else
2853                 return true;
2854
2855         return false;
2856 }
2857
2858 static inline bool func_uaccess_safe(struct symbol *func)
2859 {
2860         if (func)
2861                 return func->uaccess_safe;
2862
2863         return false;
2864 }
2865
2866 static inline const char *call_dest_name(struct instruction *insn)
2867 {
2868         if (insn->call_dest)
2869                 return insn->call_dest->name;
2870
2871         return "{dynamic}";
2872 }
2873
2874 static inline bool noinstr_call_dest(struct symbol *func)
2875 {
2876         /*
2877          * We can't deal with indirect function calls at present;
2878          * assume they're instrumented.
2879          */
2880         if (!func)
2881                 return false;
2882
2883         /*
2884          * If the symbol is from a noinstr section; we good.
2885          */
2886         if (func->sec->noinstr)
2887                 return true;
2888
2889         /*
2890          * The __ubsan_handle_*() calls are like WARN(), they only happen when
2891          * something 'BAD' happened. At the risk of taking the machine down,
2892          * let them proceed to get the message out.
2893          */
2894         if (!strncmp(func->name, "__ubsan_handle_", 15))
2895                 return true;
2896
2897         return false;
2898 }
2899
2900 static int validate_call(struct instruction *insn, struct insn_state *state)
2901 {
2902         if (state->noinstr && state->instr <= 0 &&
2903             !noinstr_call_dest(insn->call_dest)) {
2904                 WARN_FUNC("call to %s() leaves .noinstr.text section",
2905                                 insn->sec, insn->offset, call_dest_name(insn));
2906                 return 1;
2907         }
2908
2909         if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
2910                 WARN_FUNC("call to %s() with UACCESS enabled",
2911                                 insn->sec, insn->offset, call_dest_name(insn));
2912                 return 1;
2913         }
2914
2915         if (state->df) {
2916                 WARN_FUNC("call to %s() with DF set",
2917                                 insn->sec, insn->offset, call_dest_name(insn));
2918                 return 1;
2919         }
2920
2921         return 0;
2922 }
2923
2924 static int validate_sibling_call(struct instruction *insn, struct insn_state *state)
2925 {
2926         if (has_modified_stack_frame(insn, state)) {
2927                 WARN_FUNC("sibling call from callable instruction with modified stack frame",
2928                                 insn->sec, insn->offset);
2929                 return 1;
2930         }
2931
2932         return validate_call(insn, state);
2933 }
2934
2935 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
2936 {
2937         if (state->noinstr && state->instr > 0) {
2938                 WARN_FUNC("return with instrumentation enabled",
2939                           insn->sec, insn->offset);
2940                 return 1;
2941         }
2942
2943         if (state->uaccess && !func_uaccess_safe(func)) {
2944                 WARN_FUNC("return with UACCESS enabled",
2945                           insn->sec, insn->offset);
2946                 return 1;
2947         }
2948
2949         if (!state->uaccess && func_uaccess_safe(func)) {
2950                 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
2951                           insn->sec, insn->offset);
2952                 return 1;
2953         }
2954
2955         if (state->df) {
2956                 WARN_FUNC("return with DF set",
2957                           insn->sec, insn->offset);
2958                 return 1;
2959         }
2960
2961         if (func && has_modified_stack_frame(insn, state)) {
2962                 WARN_FUNC("return with modified stack frame",
2963                           insn->sec, insn->offset);
2964                 return 1;
2965         }
2966
2967         if (state->cfi.bp_scratch) {
2968                 WARN_FUNC("BP used as a scratch register",
2969                           insn->sec, insn->offset);
2970                 return 1;
2971         }
2972
2973         return 0;
2974 }
2975
2976 static struct instruction *next_insn_to_validate(struct objtool_file *file,
2977                                                  struct instruction *insn)
2978 {
2979         struct alt_group *alt_group = insn->alt_group;
2980
2981         /*
2982          * Simulate the fact that alternatives are patched in-place.  When the
2983          * end of a replacement alt_group is reached, redirect objtool flow to
2984          * the end of the original alt_group.
2985          */
2986         if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
2987                 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
2988
2989         return next_insn_same_sec(file, insn);
2990 }
2991
2992 /*
2993  * Follow the branch starting at the given instruction, and recursively follow
2994  * any other branches (jumps).  Meanwhile, track the frame pointer state at
2995  * each instruction and validate all the rules described in
2996  * tools/objtool/Documentation/stack-validation.txt.
2997  */
2998 static int validate_branch(struct objtool_file *file, struct symbol *func,
2999                            struct instruction *insn, struct insn_state state)
3000 {
3001         struct alternative *alt;
3002         struct instruction *next_insn, *prev_insn = NULL;
3003         struct section *sec;
3004         u8 visited;
3005         int ret;
3006
3007         sec = insn->sec;
3008
3009         while (1) {
3010                 next_insn = next_insn_to_validate(file, insn);
3011
3012                 if (file->c_file && func && insn->func && func != insn->func->pfunc) {
3013                         WARN("%s() falls through to next function %s()",
3014                              func->name, insn->func->name);
3015                         return 1;
3016                 }
3017
3018                 if (func && insn->ignore) {
3019                         WARN_FUNC("BUG: why am I validating an ignored function?",
3020                                   sec, insn->offset);
3021                         return 1;
3022                 }
3023
3024                 visited = VISITED_BRANCH << state.uaccess;
3025                 if (insn->visited & VISITED_BRANCH_MASK) {
3026                         if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3027                                 return 1;
3028
3029                         if (insn->visited & visited)
3030                                 return 0;
3031                 } else {
3032                         nr_insns_visited++;
3033                 }
3034
3035                 if (state.noinstr)
3036                         state.instr += insn->instr;
3037
3038                 if (insn->hint) {
3039                         if (insn->restore) {
3040                                 struct instruction *save_insn, *i;
3041
3042                                 i = insn;
3043                                 save_insn = NULL;
3044
3045                                 sym_for_each_insn_continue_reverse(file, func, i) {
3046                                         if (i->save) {
3047                                                 save_insn = i;
3048                                                 break;
3049                                         }
3050                                 }
3051
3052                                 if (!save_insn) {
3053                                         WARN_FUNC("no corresponding CFI save for CFI restore",
3054                                                   sec, insn->offset);
3055                                         return 1;
3056                                 }
3057
3058                                 if (!save_insn->visited) {
3059                                         WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo",
3060                                                   sec, insn->offset);
3061                                         return 1;
3062                                 }
3063
3064                                 insn->cfi = save_insn->cfi;
3065                                 nr_cfi_reused++;
3066                         }
3067
3068                         state.cfi = *insn->cfi;
3069                 } else {
3070                         /* XXX track if we actually changed state.cfi */
3071
3072                         if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3073                                 insn->cfi = prev_insn->cfi;
3074                                 nr_cfi_reused++;
3075                         } else {
3076                                 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3077                         }
3078                 }
3079
3080                 insn->visited |= visited;
3081
3082                 if (propagate_alt_cfi(file, insn))
3083                         return 1;
3084
3085                 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3086                         bool skip_orig = false;
3087
3088                         list_for_each_entry(alt, &insn->alts, list) {
3089                                 if (alt->skip_orig)
3090                                         skip_orig = true;
3091
3092                                 ret = validate_branch(file, func, alt->insn, state);
3093                                 if (ret) {
3094                                         if (backtrace)
3095                                                 BT_FUNC("(alt)", insn);
3096                                         return ret;
3097                                 }
3098                         }
3099
3100                         if (skip_orig)
3101                                 return 0;
3102                 }
3103
3104                 if (handle_insn_ops(insn, next_insn, &state))
3105                         return 1;
3106
3107                 switch (insn->type) {
3108
3109                 case INSN_RETURN:
3110                         if (sls && !insn->retpoline_safe &&
3111                             next_insn && next_insn->type != INSN_TRAP) {
3112                                 WARN_FUNC("missing int3 after ret",
3113                                           insn->sec, insn->offset);
3114                         }
3115                         return validate_return(func, insn, &state);
3116
3117                 case INSN_CALL:
3118                 case INSN_CALL_DYNAMIC:
3119                         ret = validate_call(insn, &state);
3120                         if (ret)
3121                                 return ret;
3122
3123                         if (!no_fp && func && !is_fentry_call(insn) &&
3124                             !has_valid_stack_frame(&state)) {
3125                                 WARN_FUNC("call without frame pointer save/setup",
3126                                           sec, insn->offset);
3127                                 return 1;
3128                         }
3129
3130                         if (dead_end_function(file, insn->call_dest))
3131                                 return 0;
3132
3133                         break;
3134
3135                 case INSN_JUMP_CONDITIONAL:
3136                 case INSN_JUMP_UNCONDITIONAL:
3137                         if (is_sibling_call(insn)) {
3138                                 ret = validate_sibling_call(insn, &state);
3139                                 if (ret)
3140                                         return ret;
3141
3142                         } else if (insn->jump_dest) {
3143                                 ret = validate_branch(file, func,
3144                                                       insn->jump_dest, state);
3145                                 if (ret) {
3146                                         if (backtrace)
3147                                                 BT_FUNC("(branch)", insn);
3148                                         return ret;
3149                                 }
3150                         }
3151
3152                         if (insn->type == INSN_JUMP_UNCONDITIONAL)
3153                                 return 0;
3154
3155                         break;
3156
3157                 case INSN_JUMP_DYNAMIC:
3158                         if (sls && !insn->retpoline_safe &&
3159                             next_insn && next_insn->type != INSN_TRAP) {
3160                                 WARN_FUNC("missing int3 after indirect jump",
3161                                           insn->sec, insn->offset);
3162                         }
3163
3164                         /* fallthrough */
3165                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3166                         if (is_sibling_call(insn)) {
3167                                 ret = validate_sibling_call(insn, &state);
3168                                 if (ret)
3169                                         return ret;
3170                         }
3171
3172                         if (insn->type == INSN_JUMP_DYNAMIC)
3173                                 return 0;
3174
3175                         break;
3176
3177                 case INSN_CONTEXT_SWITCH:
3178                         if (func && (!next_insn || !next_insn->hint)) {
3179                                 WARN_FUNC("unsupported instruction in callable function",
3180                                           sec, insn->offset);
3181                                 return 1;
3182                         }
3183                         return 0;
3184
3185                 case INSN_STAC:
3186                         if (state.uaccess) {
3187                                 WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
3188                                 return 1;
3189                         }
3190
3191                         state.uaccess = true;
3192                         break;
3193
3194                 case INSN_CLAC:
3195                         if (!state.uaccess && func) {
3196                                 WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
3197                                 return 1;
3198                         }
3199
3200                         if (func_uaccess_safe(func) && !state.uaccess_stack) {
3201                                 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
3202                                 return 1;
3203                         }
3204
3205                         state.uaccess = false;
3206                         break;
3207
3208                 case INSN_STD:
3209                         if (state.df) {
3210                                 WARN_FUNC("recursive STD", sec, insn->offset);
3211                                 return 1;
3212                         }
3213
3214                         state.df = true;
3215                         break;
3216
3217                 case INSN_CLD:
3218                         if (!state.df && func) {
3219                                 WARN_FUNC("redundant CLD", sec, insn->offset);
3220                                 return 1;
3221                         }
3222
3223                         state.df = false;
3224                         break;
3225
3226                 default:
3227                         break;
3228                 }
3229
3230                 if (insn->dead_end)
3231                         return 0;
3232
3233                 if (!next_insn) {
3234                         if (state.cfi.cfa.base == CFI_UNDEFINED)
3235                                 return 0;
3236                         WARN("%s: unexpected end of section", sec->name);
3237                         return 1;
3238                 }
3239
3240                 prev_insn = insn;
3241                 insn = next_insn;
3242         }
3243
3244         return 0;
3245 }
3246
3247 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3248 {
3249         struct instruction *insn;
3250         struct insn_state state;
3251         int ret, warnings = 0;
3252
3253         if (!file->hints)
3254                 return 0;
3255
3256         init_insn_state(&state, sec);
3257
3258         if (sec) {
3259                 insn = find_insn(file, sec, 0);
3260                 if (!insn)
3261                         return 0;
3262         } else {
3263                 insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3264         }
3265
3266         while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
3267                 if (insn->hint && !insn->visited) {
3268                         ret = validate_branch(file, insn->func, insn, state);
3269                         if (ret && backtrace)
3270                                 BT_FUNC("<=== (hint)", insn);
3271                         warnings += ret;
3272                 }
3273
3274                 insn = list_next_entry(insn, list);
3275         }
3276
3277         return warnings;
3278 }
3279
3280 /*
3281  * Validate rethunk entry constraint: must untrain RET before the first RET.
3282  *
3283  * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes
3284  * before an actual RET instruction.
3285  */
3286 static int validate_entry(struct objtool_file *file, struct instruction *insn)
3287 {
3288         struct instruction *next, *dest;
3289         int ret, warnings = 0;
3290
3291         for (;;) {
3292                 next = next_insn_to_validate(file, insn);
3293
3294                 if (insn->visited & VISITED_ENTRY)
3295                         return 0;
3296
3297                 insn->visited |= VISITED_ENTRY;
3298
3299                 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3300                         struct alternative *alt;
3301                         bool skip_orig = false;
3302
3303                         list_for_each_entry(alt, &insn->alts, list) {
3304                                 if (alt->skip_orig)
3305                                         skip_orig = true;
3306
3307                                 ret = validate_entry(file, alt->insn);
3308                                 if (ret) {
3309                                         if (backtrace)
3310                                                 BT_FUNC("(alt)", insn);
3311                                         return ret;
3312                                 }
3313                         }
3314
3315                         if (skip_orig)
3316                                 return 0;
3317                 }
3318
3319                 switch (insn->type) {
3320
3321                 case INSN_CALL_DYNAMIC:
3322                 case INSN_JUMP_DYNAMIC:
3323                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3324                         WARN_FUNC("early indirect call", insn->sec, insn->offset);
3325                         return 1;
3326
3327                 case INSN_JUMP_UNCONDITIONAL:
3328                 case INSN_JUMP_CONDITIONAL:
3329                         if (!is_sibling_call(insn)) {
3330                                 if (!insn->jump_dest) {
3331                                         WARN_FUNC("unresolved jump target after linking?!?",
3332                                                   insn->sec, insn->offset);
3333                                         return -1;
3334                                 }
3335                                 ret = validate_entry(file, insn->jump_dest);
3336                                 if (ret) {
3337                                         if (backtrace) {
3338                                                 BT_FUNC("(branch%s)", insn,
3339                                                         insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3340                                         }
3341                                         return ret;
3342                                 }
3343
3344                                 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3345                                         return 0;
3346
3347                                 break;
3348                         }
3349
3350                         /* fallthrough */
3351                 case INSN_CALL:
3352                         dest = find_insn(file, insn->call_dest->sec,
3353                                          insn->call_dest->offset);
3354                         if (!dest) {
3355                                 WARN("Unresolved function after linking!?: %s",
3356                                      insn->call_dest->name);
3357                                 return -1;
3358                         }
3359
3360                         ret = validate_entry(file, dest);
3361                         if (ret) {
3362                                 if (backtrace)
3363                                         BT_FUNC("(call)", insn);
3364                                 return ret;
3365                         }
3366                         /*
3367                          * If a call returns without error, it must have seen UNTRAIN_RET.
3368                          * Therefore any non-error return is a success.
3369                          */
3370                         return 0;
3371
3372                 case INSN_RETURN:
3373                         WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset);
3374                         return 1;
3375
3376                 case INSN_NOP:
3377                         if (insn->retpoline_safe)
3378                                 return 0;
3379                         break;
3380
3381                 default:
3382                         break;
3383                 }
3384
3385                 if (!next) {
3386                         WARN_FUNC("teh end!", insn->sec, insn->offset);
3387                         return -1;
3388                 }
3389                 insn = next;
3390         }
3391
3392         return warnings;
3393 }
3394
3395 /*
3396  * Validate that all branches starting at 'insn->entry' encounter UNRET_END
3397  * before RET.
3398  */
3399 static int validate_unret(struct objtool_file *file)
3400 {
3401         struct instruction *insn;
3402         int ret, warnings = 0;
3403
3404         for_each_insn(file, insn) {
3405                 if (!insn->entry)
3406                         continue;
3407
3408                 ret = validate_entry(file, insn);
3409                 if (ret < 0) {
3410                         WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset);
3411                         return ret;
3412                 }
3413                 warnings += ret;
3414         }
3415
3416         return warnings;
3417 }
3418
3419 static int validate_retpoline(struct objtool_file *file)
3420 {
3421         struct instruction *insn;
3422         int warnings = 0;
3423
3424         for_each_insn(file, insn) {
3425                 if (insn->type != INSN_JUMP_DYNAMIC &&
3426                     insn->type != INSN_CALL_DYNAMIC &&
3427                     insn->type != INSN_RETURN)
3428                         continue;
3429
3430                 if (insn->retpoline_safe)
3431                         continue;
3432
3433                 /*
3434                  * .init.text code is ran before userspace and thus doesn't
3435                  * strictly need retpolines, except for modules which are
3436                  * loaded late, they very much do need retpoline in their
3437                  * .init.text
3438                  */
3439                 if (!strcmp(insn->sec->name, ".init.text") && !module)
3440                         continue;
3441
3442                 if (insn->type == INSN_RETURN) {
3443                         if (rethunk) {
3444                                 WARN_FUNC("'naked' return found in RETHUNK build",
3445                                           insn->sec, insn->offset);
3446                         } else
3447                                 continue;
3448                 } else {
3449                         WARN_FUNC("indirect %s found in RETPOLINE build",
3450                                   insn->sec, insn->offset,
3451                                   insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3452                 }
3453
3454                 warnings++;
3455         }
3456
3457         return warnings;
3458 }
3459
3460 static bool is_kasan_insn(struct instruction *insn)
3461 {
3462         return (insn->type == INSN_CALL &&
3463                 !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3464 }
3465
3466 static bool is_ubsan_insn(struct instruction *insn)
3467 {
3468         return (insn->type == INSN_CALL &&
3469                 !strcmp(insn->call_dest->name,
3470                         "__ubsan_handle_builtin_unreachable"));
3471 }
3472
3473 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
3474 {
3475         int i;
3476         struct instruction *prev_insn;
3477
3478         if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
3479                 return true;
3480
3481         /*
3482          * Ignore any unused exceptions.  This can happen when a whitelisted
3483          * function has an exception table entry.
3484          *
3485          * Also ignore alternative replacement instructions.  This can happen
3486          * when a whitelisted function uses one of the ALTERNATIVE macros.
3487          */
3488         if (!strcmp(insn->sec->name, ".fixup") ||
3489             !strcmp(insn->sec->name, ".altinstr_replacement") ||
3490             !strcmp(insn->sec->name, ".altinstr_aux"))
3491                 return true;
3492
3493         if (!insn->func)
3494                 return false;
3495
3496         /*
3497          * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
3498          * __builtin_unreachable().  The BUG() macro has an unreachable() after
3499          * the UD2, which causes GCC's undefined trap logic to emit another UD2
3500          * (or occasionally a JMP to UD2).
3501          *
3502          * It may also insert a UD2 after calling a __noreturn function.
3503          */
3504         prev_insn = list_prev_entry(insn, list);
3505         if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
3506             (insn->type == INSN_BUG ||
3507              (insn->type == INSN_JUMP_UNCONDITIONAL &&
3508               insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
3509                 return true;
3510
3511         /*
3512          * Check if this (or a subsequent) instruction is related to
3513          * CONFIG_UBSAN or CONFIG_KASAN.
3514          *
3515          * End the search at 5 instructions to avoid going into the weeds.
3516          */
3517         for (i = 0; i < 5; i++) {
3518
3519                 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
3520                         return true;
3521
3522                 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3523                         if (insn->jump_dest &&
3524                             insn->jump_dest->func == insn->func) {
3525                                 insn = insn->jump_dest;
3526                                 continue;
3527                         }
3528
3529                         break;
3530                 }
3531
3532                 if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
3533                         break;
3534
3535                 insn = list_next_entry(insn, list);
3536         }
3537
3538         return false;
3539 }
3540
3541 static int validate_symbol(struct objtool_file *file, struct section *sec,
3542                            struct symbol *sym, struct insn_state *state)
3543 {
3544         struct instruction *insn;
3545         int ret;
3546
3547         if (!sym->len) {
3548                 WARN("%s() is missing an ELF size annotation", sym->name);
3549                 return 1;
3550         }
3551
3552         if (sym->pfunc != sym || sym->alias != sym)
3553                 return 0;
3554
3555         insn = find_insn(file, sec, sym->offset);
3556         if (!insn || insn->ignore || insn->visited)
3557                 return 0;
3558
3559         state->uaccess = sym->uaccess_safe;
3560
3561         ret = validate_branch(file, insn->func, insn, *state);
3562         if (ret && backtrace)
3563                 BT_FUNC("<=== (sym)", insn);
3564         return ret;
3565 }
3566
3567 static int validate_section(struct objtool_file *file, struct section *sec)
3568 {
3569         struct insn_state state;
3570         struct symbol *func;
3571         int warnings = 0;
3572
3573         list_for_each_entry(func, &sec->symbol_list, list) {
3574                 if (func->type != STT_FUNC)
3575                         continue;
3576
3577                 init_insn_state(&state, sec);
3578                 set_func_state(&state.cfi);
3579
3580                 warnings += validate_symbol(file, sec, func, &state);
3581         }
3582
3583         return warnings;
3584 }
3585
3586 static int validate_vmlinux_functions(struct objtool_file *file)
3587 {
3588         struct section *sec;
3589         int warnings = 0;
3590
3591         sec = find_section_by_name(file->elf, ".noinstr.text");
3592         if (sec) {
3593                 warnings += validate_section(file, sec);
3594                 warnings += validate_unwind_hints(file, sec);
3595         }
3596
3597         sec = find_section_by_name(file->elf, ".entry.text");
3598         if (sec) {
3599                 warnings += validate_section(file, sec);
3600                 warnings += validate_unwind_hints(file, sec);
3601         }
3602
3603         return warnings;
3604 }
3605
3606 static int validate_functions(struct objtool_file *file)
3607 {
3608         struct section *sec;
3609         int warnings = 0;
3610
3611         for_each_sec(file, sec) {
3612                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
3613                         continue;
3614
3615                 warnings += validate_section(file, sec);
3616         }
3617
3618         return warnings;
3619 }
3620
3621 static int validate_reachable_instructions(struct objtool_file *file)
3622 {
3623         struct instruction *insn;
3624
3625         if (file->ignore_unreachables)
3626                 return 0;
3627
3628         for_each_insn(file, insn) {
3629                 if (insn->visited || ignore_unreachable_insn(file, insn))
3630                         continue;
3631
3632                 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
3633                 return 1;
3634         }
3635
3636         return 0;
3637 }
3638
3639 int check(struct objtool_file *file)
3640 {
3641         int ret, warnings = 0;
3642
3643         arch_initial_func_cfi_state(&initial_func_cfi);
3644         init_cfi_state(&init_cfi);
3645         init_cfi_state(&func_cfi);
3646         set_func_state(&func_cfi);
3647
3648         if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
3649                 goto out;
3650
3651         cfi_hash_add(&init_cfi);
3652         cfi_hash_add(&func_cfi);
3653
3654         ret = decode_sections(file);
3655         if (ret < 0)
3656                 goto out;
3657
3658         warnings += ret;
3659
3660         if (list_empty(&file->insn_list))
3661                 goto out;
3662
3663         if (vmlinux && !validate_dup) {
3664                 ret = validate_vmlinux_functions(file);
3665                 if (ret < 0)
3666                         goto out;
3667
3668                 warnings += ret;
3669                 goto out;
3670         }
3671
3672         if (retpoline) {
3673                 ret = validate_retpoline(file);
3674                 if (ret < 0)
3675                         return ret;
3676                 warnings += ret;
3677         }
3678
3679         ret = validate_functions(file);
3680         if (ret < 0)
3681                 goto out;
3682         warnings += ret;
3683
3684         ret = validate_unwind_hints(file, NULL);
3685         if (ret < 0)
3686                 goto out;
3687         warnings += ret;
3688
3689         if (unret) {
3690                 /*
3691                  * Must be after validate_branch() and friends, it plays
3692                  * further games with insn->visited.
3693                  */
3694                 ret = validate_unret(file);
3695                 if (ret < 0)
3696                         return ret;
3697                 warnings += ret;
3698         }
3699
3700         if (!warnings) {
3701                 ret = validate_reachable_instructions(file);
3702                 if (ret < 0)
3703                         goto out;
3704                 warnings += ret;
3705         }
3706
3707         ret = create_static_call_sections(file);
3708         if (ret < 0)
3709                 goto out;
3710         warnings += ret;
3711
3712         if (retpoline) {
3713                 ret = create_retpoline_sites_sections(file);
3714                 if (ret < 0)
3715                         goto out;
3716                 warnings += ret;
3717         }
3718
3719         if (rethunk) {
3720                 ret = create_return_sites_sections(file);
3721                 if (ret < 0)
3722                         goto out;
3723                 warnings += ret;
3724         }
3725
3726         if (mcount) {
3727                 ret = create_mcount_loc_sections(file);
3728                 if (ret < 0)
3729                         goto out;
3730                 warnings += ret;
3731         }
3732
3733         if (stats) {
3734                 printf("nr_insns_visited: %ld\n", nr_insns_visited);
3735                 printf("nr_cfi: %ld\n", nr_cfi);
3736                 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
3737                 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
3738         }
3739
3740 out:
3741         /*
3742          *  For now, don't fail the kernel build on fatal warnings.  These
3743          *  errors are still fairly common due to the growing matrix of
3744          *  supported toolchains and their recent pace of change.
3745          */
3746         return 0;
3747 }