tools/objtool/check.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
   4  */
   5
   6 #include <string.h>
   7 #include <stdlib.h>
   8 #include <inttypes.h>
   9 #include <sys/mman.h>
  10
  11 #include <arch/elf.h>
  12 #include <objtool/builtin.h>
  13 #include <objtool/cfi.h>
  14 #include <objtool/arch.h>
  15 #include <objtool/check.h>
  16 #include <objtool/special.h>
  17 #include <objtool/warn.h>
  18 #include <objtool/endianness.h>
  19
  20 #include <linux/objtool.h>
  21 #include <linux/hashtable.h>
  22 #include <linux/kernel.h>
  23 #include <linux/static_call_types.h>
  24
  25 struct alternative {
  26         struct list_head list;
  27         struct instruction *insn;
  28         bool skip_orig;
  29 };
  30
  31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
  32
  33 static struct cfi_init_state initial_func_cfi;
  34 static struct cfi_state init_cfi;
  35 static struct cfi_state func_cfi;
  36
  37 struct instruction *find_insn(struct objtool_file *file,
  38                               struct section *sec, unsigned long offset)
  39 {
  40         struct instruction *insn;
  41
  42         hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
  43                 if (insn->sec == sec && insn->offset == offset)
  44                         return insn;
  45         }
  46
  47         return NULL;
  48 }
  49
  50 static struct instruction *next_insn_same_sec(struct objtool_file *file,
  51                                               struct instruction *insn)
  52 {
  53         struct instruction *next = list_next_entry(insn, list);
  54
  55         if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
  56                 return NULL;
  57
  58         return next;
  59 }
  60
  61 static struct instruction *next_insn_same_func(struct objtool_file *file,
  62                                                struct instruction *insn)
  63 {
  64         struct instruction *next = list_next_entry(insn, list);
  65         struct symbol *func = insn->func;
  66
  67         if (!func)
  68                 return NULL;
  69
  70         if (&next->list != &file->insn_list && next->func == func)
  71                 return next;
  72
  73         /* Check if we're already in the subfunction: */
  74         if (func == func->cfunc)
  75                 return NULL;
  76
  77         /* Move to the subfunction: */
  78         return find_insn(file, func->cfunc->sec, func->cfunc->offset);
  79 }
  80
  81 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
  82                                                struct instruction *insn)
  83 {
  84         struct instruction *prev = list_prev_entry(insn, list);
  85
  86         if (&prev->list != &file->insn_list && prev->func == insn->func)
  87                 return prev;
  88
  89         return NULL;
  90 }
  91
  92 #define func_for_each_insn(file, func, insn)                            \
  93         for (insn = find_insn(file, func->sec, func->offset);           \
  94              insn;                                                      \
  95              insn = next_insn_same_func(file, insn))
  96
  97 #define sym_for_each_insn(file, sym, insn)                              \
  98         for (insn = find_insn(file, sym->sec, sym->offset);             \
  99              insn && &insn->list != &file->insn_list &&                 \
 100                 insn->sec == sym->sec &&                                \
 101                 insn->offset < sym->offset + sym->len;                  \
 102              insn = list_next_entry(insn, list))
 103
 104 #define sym_for_each_insn_continue_reverse(file, sym, insn)             \
 105         for (insn = list_prev_entry(insn, list);                        \
 106              &insn->list != &file->insn_list &&                         \
 107                 insn->sec == sym->sec && insn->offset >= sym->offset;   \
 108              insn = list_prev_entry(insn, list))
 109
 110 #define sec_for_each_insn_from(file, insn)                              \
 111         for (; insn; insn = next_insn_same_sec(file, insn))
 112
 113 #define sec_for_each_insn_continue(file, insn)                          \
 114         for (insn = next_insn_same_sec(file, insn); insn;               \
 115              insn = next_insn_same_sec(file, insn))
 116
 117 static bool is_jump_table_jump(struct instruction *insn)
 118 {
 119         struct alt_group *alt_group = insn->alt_group;
 120
 121         if (insn->jump_table)
 122                 return true;
 123
 124         /* Retpoline alternative for a jump table? */
 125         return alt_group && alt_group->orig_group &&
 126                alt_group->orig_group->first_insn->jump_table;
 127 }
 128
 129 static bool is_sibling_call(struct instruction *insn)
 130 {
 131         /*
 132          * Assume only ELF functions can make sibling calls.  This ensures
 133          * sibling call detection consistency between vmlinux.o and individual
 134          * objects.
 135          */
 136         if (!insn->func)
 137                 return false;
 138
 139         /* An indirect jump is either a sibling call or a jump to a table. */
 140         if (insn->type == INSN_JUMP_DYNAMIC)
 141                 return !is_jump_table_jump(insn);
 142
 143         /* add_jump_destinations() sets insn->call_dest for sibling calls. */
 144         return (is_static_jump(insn) && insn->call_dest);
 145 }
 146
 147 /*
 148  * This checks to see if the given function is a "noreturn" function.
 149  *
 150  * For global functions which are outside the scope of this object file, we
 151  * have to keep a manual list of them.
 152  *
 153  * For local functions, we have to detect them manually by simply looking for
 154  * the lack of a return instruction.
 155  */
 156 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 157                                 int recursion)
 158 {
 159         int i;
 160         struct instruction *insn;
 161         bool empty = true;
 162
 163         /*
 164          * Unfortunately these have to be hard coded because the noreturn
 165          * attribute isn't provided in ELF data.
 166          */
 167         static const char * const global_noreturns[] = {
 168                 "__stack_chk_fail",
 169                 "panic",
 170                 "do_exit",
 171                 "do_task_dead",
 172                 "__module_put_and_exit",
 173                 "complete_and_exit",
 174                 "__reiserfs_panic",
 175                 "lbug_with_loc",
 176                 "fortify_panic",
 177                 "usercopy_abort",
 178                 "machine_real_restart",
 179                 "rewind_stack_do_exit",
 180                 "kunit_try_catch_throw",
 181                 "xen_start_kernel",
 182                 "cpu_bringup_and_idle",
 183         };
 184
 185         if (!func)
 186                 return false;
 187
 188         if (func->bind == STB_WEAK)
 189                 return false;
 190
 191         if (func->bind == STB_GLOBAL)
 192                 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
 193                         if (!strcmp(func->name, global_noreturns[i]))
 194                                 return true;
 195
 196         if (!func->len)
 197                 return false;
 198
 199         insn = find_insn(file, func->sec, func->offset);
 200         if (!insn->func)
 201                 return false;
 202
 203         func_for_each_insn(file, func, insn) {
 204                 empty = false;
 205
 206                 if (insn->type == INSN_RETURN)
 207                         return false;
 208         }
 209
 210         if (empty)
 211                 return false;
 212
 213         /*
 214          * A function can have a sibling call instead of a return.  In that
 215          * case, the function's dead-end status depends on whether the target
 216          * of the sibling call returns.
 217          */
 218         func_for_each_insn(file, func, insn) {
 219                 if (is_sibling_call(insn)) {
 220                         struct instruction *dest = insn->jump_dest;
 221
 222                         if (!dest)
 223                                 /* sibling call to another file */
 224                                 return false;
 225
 226                         /* local sibling call */
 227                         if (recursion == 5) {
 228                                 /*
 229                                  * Infinite recursion: two functions have
 230                                  * sibling calls to each other.  This is a very
 231                                  * rare case.  It means they aren't dead ends.
 232                                  */
 233                                 return false;
 234                         }
 235
 236                         return __dead_end_function(file, dest->func, recursion+1);
 237                 }
 238         }
 239
 240         return true;
 241 }
 242
 243 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
 244 {
 245         return __dead_end_function(file, func, 0);
 246 }
 247
 248 static void init_cfi_state(struct cfi_state *cfi)
 249 {
 250         int i;
 251
 252         for (i = 0; i < CFI_NUM_REGS; i++) {
 253                 cfi->regs[i].base = CFI_UNDEFINED;
 254                 cfi->vals[i].base = CFI_UNDEFINED;
 255         }
 256         cfi->cfa.base = CFI_UNDEFINED;
 257         cfi->drap_reg = CFI_UNDEFINED;
 258         cfi->drap_offset = -1;
 259 }
 260
 261 static void init_insn_state(struct insn_state *state, struct section *sec)
 262 {
 263         memset(state, 0, sizeof(*state));
 264         init_cfi_state(&state->cfi);
 265
 266         /*
 267          * We need the full vmlinux for noinstr validation, otherwise we can
 268          * not correctly determine insn->call_dest->sec (external symbols do
 269          * not have a section).
 270          */
 271         if (vmlinux && noinstr && sec)
 272                 state->noinstr = sec->noinstr;
 273 }
 274
 275 static struct cfi_state *cfi_alloc(void)
 276 {
 277         struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
 278         if (!cfi) {
 279                 WARN("calloc failed");
 280                 exit(1);
 281         }
 282         nr_cfi++;
 283         return cfi;
 284 }
 285
 286 static int cfi_bits;
 287 static struct hlist_head *cfi_hash;
 288
 289 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
 290 {
 291         return memcmp((void *)cfi1 + sizeof(cfi1->hash),
 292                       (void *)cfi2 + sizeof(cfi2->hash),
 293                       sizeof(struct cfi_state) - sizeof(struct hlist_node));
 294 }
 295
 296 static inline u32 cfi_key(struct cfi_state *cfi)
 297 {
 298         return jhash((void *)cfi + sizeof(cfi->hash),
 299                      sizeof(*cfi) - sizeof(cfi->hash), 0);
 300 }
 301
 302 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
 303 {
 304         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 305         struct cfi_state *obj;
 306
 307         hlist_for_each_entry(obj, head, hash) {
 308                 if (!cficmp(cfi, obj)) {
 309                         nr_cfi_cache++;
 310                         return obj;
 311                 }
 312         }
 313
 314         obj = cfi_alloc();
 315         *obj = *cfi;
 316         hlist_add_head(&obj->hash, head);
 317
 318         return obj;
 319 }
 320
 321 static void cfi_hash_add(struct cfi_state *cfi)
 322 {
 323         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 324
 325         hlist_add_head(&cfi->hash, head);
 326 }
 327
 328 static void *cfi_hash_alloc(unsigned long size)
 329 {
 330         cfi_bits = max(10, ilog2(size));
 331         cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
 332                         PROT_READ|PROT_WRITE,
 333                         MAP_PRIVATE|MAP_ANON, -1, 0);
 334         if (cfi_hash == (void *)-1L) {
 335                 WARN("mmap fail cfi_hash");
 336                 cfi_hash = NULL;
 337         }  else if (stats) {
 338                 printf("cfi_bits: %d\n", cfi_bits);
 339         }
 340
 341         return cfi_hash;
 342 }
 343
 344 static unsigned long nr_insns;
 345 static unsigned long nr_insns_visited;
 346
 347 /*
 348  * Call the arch-specific instruction decoder for all the instructions and add
 349  * them to the global instruction list.
 350  */
 351 static int decode_instructions(struct objtool_file *file)
 352 {
 353         struct section *sec;
 354         struct symbol *func;
 355         unsigned long offset;
 356         struct instruction *insn;
 357         int ret;
 358
 359         for_each_sec(file, sec) {
 360
 361                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
 362                         continue;
 363
 364                 if (strcmp(sec->name, ".altinstr_replacement") &&
 365                     strcmp(sec->name, ".altinstr_aux") &&
 366                     strncmp(sec->name, ".discard.", 9))
 367                         sec->text = true;
 368
 369                 if (!strcmp(sec->name, ".noinstr.text") ||
 370                     !strcmp(sec->name, ".entry.text") ||
 371                     !strncmp(sec->name, ".text.__x86.", 12))
 372                         sec->noinstr = true;
 373
 374                 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
 375                         insn = malloc(sizeof(*insn));
 376                         if (!insn) {
 377                                 WARN("malloc failed");
 378                                 return -1;
 379                         }
 380                         memset(insn, 0, sizeof(*insn));
 381                         INIT_LIST_HEAD(&insn->alts);
 382                         INIT_LIST_HEAD(&insn->stack_ops);
 383
 384                         insn->sec = sec;
 385                         insn->offset = offset;
 386
 387                         ret = arch_decode_instruction(file->elf, sec, offset,
 388                                                       sec->sh.sh_size - offset,
 389                                                       &insn->len, &insn->type,
 390                                                       &insn->immediate,
 391                                                       &insn->stack_ops);
 392                         if (ret)
 393                                 goto err;
 394
 395                         hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
 396                         list_add_tail(&insn->list, &file->insn_list);
 397                         nr_insns++;
 398                 }
 399
 400                 list_for_each_entry(func, &sec->symbol_list, list) {
 401                         if (func->type != STT_FUNC || func->alias != func)
 402                                 continue;
 403
 404                         if (!find_insn(file, sec, func->offset)) {
 405                                 WARN("%s(): can't find starting instruction",
 406                                      func->name);
 407                                 return -1;
 408                         }
 409
 410                         sym_for_each_insn(file, func, insn)
 411                                 insn->func = func;
 412                 }
 413         }
 414
 415         if (stats)
 416                 printf("nr_insns: %lu\n", nr_insns);
 417
 418         return 0;
 419
 420 err:
 421         free(insn);
 422         return ret;
 423 }
 424
 425 static struct instruction *find_last_insn(struct objtool_file *file,
 426                                           struct section *sec)
 427 {
 428         struct instruction *insn = NULL;
 429         unsigned int offset;
 430         unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
 431
 432         for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
 433                 insn = find_insn(file, sec, offset);
 434
 435         return insn;
 436 }
 437
 438 /*
 439  * Mark "ud2" instructions and manually annotated dead ends.
 440  */
 441 static int add_dead_ends(struct objtool_file *file)
 442 {
 443         struct section *sec;
 444         struct reloc *reloc;
 445         struct instruction *insn;
 446
 447         /*
 448          * By default, "ud2" is a dead end unless otherwise annotated, because
 449          * GCC 7 inserts it for certain divide-by-zero cases.
 450          */
 451         for_each_insn(file, insn)
 452                 if (insn->type == INSN_BUG)
 453                         insn->dead_end = true;
 454
 455         /*
 456          * Check for manually annotated dead ends.
 457          */
 458         sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
 459         if (!sec)
 460                 goto reachable;
 461
 462         list_for_each_entry(reloc, &sec->reloc_list, list) {
 463                 if (reloc->sym->type != STT_SECTION) {
 464                         WARN("unexpected relocation symbol type in %s", sec->name);
 465                         return -1;
 466                 }
 467                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 468                 if (insn)
 469                         insn = list_prev_entry(insn, list);
 470                 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 471                         insn = find_last_insn(file, reloc->sym->sec);
 472                         if (!insn) {
 473                                 WARN("can't find unreachable insn at %s+0x%" PRIx64,
 474                                      reloc->sym->sec->name, reloc->addend);
 475                                 return -1;
 476                         }
 477                 } else {
 478                         WARN("can't find unreachable insn at %s+0x%" PRIx64,
 479                              reloc->sym->sec->name, reloc->addend);
 480                         return -1;
 481                 }
 482
 483                 insn->dead_end = true;
 484         }
 485
 486 reachable:
 487         /*
 488          * These manually annotated reachable checks are needed for GCC 4.4,
 489          * where the Linux unreachable() macro isn't supported.  In that case
 490          * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
 491          * not a dead end.
 492          */
 493         sec = find_section_by_name(file->elf, ".rela.discard.reachable");
 494         if (!sec)
 495                 return 0;
 496
 497         list_for_each_entry(reloc, &sec->reloc_list, list) {
 498                 if (reloc->sym->type != STT_SECTION) {
 499                         WARN("unexpected relocation symbol type in %s", sec->name);
 500                         return -1;
 501                 }
 502                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 503                 if (insn)
 504                         insn = list_prev_entry(insn, list);
 505                 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
 506                         insn = find_last_insn(file, reloc->sym->sec);
 507                         if (!insn) {
 508                                 WARN("can't find reachable insn at %s+0x%" PRIx64,
 509                                      reloc->sym->sec->name, reloc->addend);
 510                                 return -1;
 511                         }
 512                 } else {
 513                         WARN("can't find reachable insn at %s+0x%" PRIx64,
 514                              reloc->sym->sec->name, reloc->addend);
 515                         return -1;
 516                 }
 517
 518                 insn->dead_end = false;
 519         }
 520
 521         return 0;
 522 }
 523
 524 static int create_static_call_sections(struct objtool_file *file)
 525 {
 526         struct section *sec;
 527         struct static_call_site *site;
 528         struct instruction *insn;
 529         struct symbol *key_sym;
 530         char *key_name, *tmp;
 531         int idx;
 532
 533         sec = find_section_by_name(file->elf, ".static_call_sites");
 534         if (sec) {
 535                 INIT_LIST_HEAD(&file->static_call_list);
 536                 WARN("file already has .static_call_sites section, skipping");
 537                 return 0;
 538         }
 539
 540         if (list_empty(&file->static_call_list))
 541                 return 0;
 542
 543         idx = 0;
 544         list_for_each_entry(insn, &file->static_call_list, call_node)
 545                 idx++;
 546
 547         sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
 548                                  sizeof(struct static_call_site), idx);
 549         if (!sec)
 550                 return -1;
 551
 552         idx = 0;
 553         list_for_each_entry(insn, &file->static_call_list, call_node) {
 554
 555                 site = (struct static_call_site *)sec->data->d_buf + idx;
 556                 memset(site, 0, sizeof(struct static_call_site));
 557
 558                 /* populate reloc for 'addr' */
 559                 if (elf_add_reloc_to_insn(file->elf, sec,
 560                                           idx * sizeof(struct static_call_site),
 561                                           R_X86_64_PC32,
 562                                           insn->sec, insn->offset))
 563                         return -1;
 564
 565                 /* find key symbol */
 566                 key_name = strdup(insn->call_dest->name);
 567                 if (!key_name) {
 568                         perror("strdup");
 569                         return -1;
 570                 }
 571                 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
 572                             STATIC_CALL_TRAMP_PREFIX_LEN)) {
 573                         WARN("static_call: trampoline name malformed: %s", key_name);
 574                         return -1;
 575                 }
 576                 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
 577                 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
 578
 579                 key_sym = find_symbol_by_name(file->elf, tmp);
 580                 if (!key_sym) {
 581                         if (!module) {
 582                                 WARN("static_call: can't find static_call_key symbol: %s", tmp);
 583                                 return -1;
 584                         }
 585
 586                         /*
 587                          * For modules(), the key might not be exported, which
 588                          * means the module can make static calls but isn't
 589                          * allowed to change them.
 590                          *
 591                          * In that case we temporarily set the key to be the
 592                          * trampoline address.  This is fixed up in
 593                          * static_call_add_module().
 594                          */
 595                         key_sym = insn->call_dest;
 596                 }
 597                 free(key_name);
 598
 599                 /* populate reloc for 'key' */
 600                 if (elf_add_reloc(file->elf, sec,
 601                                   idx * sizeof(struct static_call_site) + 4,
 602                                   R_X86_64_PC32, key_sym,
 603                                   is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
 604                         return -1;
 605
 606                 idx++;
 607         }
 608
 609         return 0;
 610 }
 611
 612 static int create_retpoline_sites_sections(struct objtool_file *file)
 613 {
 614         struct instruction *insn;
 615         struct section *sec;
 616         int idx;
 617
 618         sec = find_section_by_name(file->elf, ".retpoline_sites");
 619         if (sec) {
 620                 WARN("file already has .retpoline_sites, skipping");
 621                 return 0;
 622         }
 623
 624         idx = 0;
 625         list_for_each_entry(insn, &file->retpoline_call_list, call_node)
 626                 idx++;
 627
 628         if (!idx)
 629                 return 0;
 630
 631         sec = elf_create_section(file->elf, ".retpoline_sites", 0,
 632                                  sizeof(int), idx);
 633         if (!sec) {
 634                 WARN("elf_create_section: .retpoline_sites");
 635                 return -1;
 636         }
 637
 638         idx = 0;
 639         list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
 640
 641                 int *site = (int *)sec->data->d_buf + idx;
 642                 *site = 0;
 643
 644                 if (elf_add_reloc_to_insn(file->elf, sec,
 645                                           idx * sizeof(int),
 646                                           R_X86_64_PC32,
 647                                           insn->sec, insn->offset)) {
 648                         WARN("elf_add_reloc_to_insn: .retpoline_sites");
 649                         return -1;
 650                 }
 651
 652                 idx++;
 653         }
 654
 655         return 0;
 656 }
 657
 658 static int create_return_sites_sections(struct objtool_file *file)
 659 {
 660         struct instruction *insn;
 661         struct section *sec;
 662         int idx;
 663
 664         sec = find_section_by_name(file->elf, ".return_sites");
 665         if (sec) {
 666                 WARN("file already has .return_sites, skipping");
 667                 return 0;
 668         }
 669
 670         idx = 0;
 671         list_for_each_entry(insn, &file->return_thunk_list, call_node)
 672                 idx++;
 673
 674         if (!idx)
 675                 return 0;
 676
 677         sec = elf_create_section(file->elf, ".return_sites", 0,
 678                                  sizeof(int), idx);
 679         if (!sec) {
 680                 WARN("elf_create_section: .return_sites");
 681                 return -1;
 682         }
 683
 684         idx = 0;
 685         list_for_each_entry(insn, &file->return_thunk_list, call_node) {
 686
 687                 int *site = (int *)sec->data->d_buf + idx;
 688                 *site = 0;
 689
 690                 if (elf_add_reloc_to_insn(file->elf, sec,
 691                                           idx * sizeof(int),
 692                                           R_X86_64_PC32,
 693                                           insn->sec, insn->offset)) {
 694                         WARN("elf_add_reloc_to_insn: .return_sites");
 695                         return -1;
 696                 }
 697
 698                 idx++;
 699         }
 700
 701         return 0;
 702 }
 703
 704 static int create_mcount_loc_sections(struct objtool_file *file)
 705 {
 706         struct section *sec;
 707         unsigned long *loc;
 708         struct instruction *insn;
 709         int idx;
 710
 711         sec = find_section_by_name(file->elf, "__mcount_loc");
 712         if (sec) {
 713                 INIT_LIST_HEAD(&file->mcount_loc_list);
 714                 WARN("file already has __mcount_loc section, skipping");
 715                 return 0;
 716         }
 717
 718         if (list_empty(&file->mcount_loc_list))
 719                 return 0;
 720
 721         idx = 0;
 722         list_for_each_entry(insn, &file->mcount_loc_list, call_node)
 723                 idx++;
 724
 725         sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
 726         if (!sec)
 727                 return -1;
 728
 729         idx = 0;
 730         list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
 731
 732                 loc = (unsigned long *)sec->data->d_buf + idx;
 733                 memset(loc, 0, sizeof(unsigned long));
 734
 735                 if (elf_add_reloc_to_insn(file->elf, sec,
 736                                           idx * sizeof(unsigned long),
 737                                           R_X86_64_64,
 738                                           insn->sec, insn->offset))
 739                         return -1;
 740
 741                 idx++;
 742         }
 743
 744         return 0;
 745 }
 746
 747 /*
 748  * Warnings shouldn't be reported for ignored functions.
 749  */
 750 static void add_ignores(struct objtool_file *file)
 751 {
 752         struct instruction *insn;
 753         struct section *sec;
 754         struct symbol *func;
 755         struct reloc *reloc;
 756
 757         sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
 758         if (!sec)
 759                 return;
 760
 761         list_for_each_entry(reloc, &sec->reloc_list, list) {
 762                 switch (reloc->sym->type) {
 763                 case STT_FUNC:
 764                         func = reloc->sym;
 765                         break;
 766
 767                 case STT_SECTION:
 768                         func = find_func_by_offset(reloc->sym->sec, reloc->addend);
 769                         if (!func)
 770                                 continue;
 771                         break;
 772
 773                 default:
 774                         WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
 775                         continue;
 776                 }
 777
 778                 func_for_each_insn(file, func, insn)
 779                         insn->ignore = true;
 780         }
 781 }
 782
 783 /*
 784  * This is a whitelist of functions that is allowed to be called with AC set.
 785  * The list is meant to be minimal and only contains compiler instrumentation
 786  * ABI and a few functions used to implement *_{to,from}_user() functions.
 787  *
 788  * These functions must not directly change AC, but may PUSHF/POPF.
 789  */
 790 static const char *uaccess_safe_builtin[] = {
 791         /* KASAN */
 792         "kasan_report",
 793         "kasan_check_range",
 794         /* KASAN out-of-line */
 795         "__asan_loadN_noabort",
 796         "__asan_load1_noabort",
 797         "__asan_load2_noabort",
 798         "__asan_load4_noabort",
 799         "__asan_load8_noabort",
 800         "__asan_load16_noabort",
 801         "__asan_storeN_noabort",
 802         "__asan_store1_noabort",
 803         "__asan_store2_noabort",
 804         "__asan_store4_noabort",
 805         "__asan_store8_noabort",
 806         "__asan_store16_noabort",
 807         "__kasan_check_read",
 808         "__kasan_check_write",
 809         /* KASAN in-line */
 810         "__asan_report_load_n_noabort",
 811         "__asan_report_load1_noabort",
 812         "__asan_report_load2_noabort",
 813         "__asan_report_load4_noabort",
 814         "__asan_report_load8_noabort",
 815         "__asan_report_load16_noabort",
 816         "__asan_report_store_n_noabort",
 817         "__asan_report_store1_noabort",
 818         "__asan_report_store2_noabort",
 819         "__asan_report_store4_noabort",
 820         "__asan_report_store8_noabort",
 821         "__asan_report_store16_noabort",
 822         /* KCSAN */
 823         "__kcsan_check_access",
 824         "kcsan_found_watchpoint",
 825         "kcsan_setup_watchpoint",
 826         "kcsan_check_scoped_accesses",
 827         "kcsan_disable_current",
 828         "kcsan_enable_current_nowarn",
 829         /* KCSAN/TSAN */
 830         "__tsan_func_entry",
 831         "__tsan_func_exit",
 832         "__tsan_read_range",
 833         "__tsan_write_range",
 834         "__tsan_read1",
 835         "__tsan_read2",
 836         "__tsan_read4",
 837         "__tsan_read8",
 838         "__tsan_read16",
 839         "__tsan_write1",
 840         "__tsan_write2",
 841         "__tsan_write4",
 842         "__tsan_write8",
 843         "__tsan_write16",
 844         "__tsan_read_write1",
 845         "__tsan_read_write2",
 846         "__tsan_read_write4",
 847         "__tsan_read_write8",
 848         "__tsan_read_write16",
 849         "__tsan_atomic8_load",
 850         "__tsan_atomic16_load",
 851         "__tsan_atomic32_load",
 852         "__tsan_atomic64_load",
 853         "__tsan_atomic8_store",
 854         "__tsan_atomic16_store",
 855         "__tsan_atomic32_store",
 856         "__tsan_atomic64_store",
 857         "__tsan_atomic8_exchange",
 858         "__tsan_atomic16_exchange",
 859         "__tsan_atomic32_exchange",
 860         "__tsan_atomic64_exchange",
 861         "__tsan_atomic8_fetch_add",
 862         "__tsan_atomic16_fetch_add",
 863         "__tsan_atomic32_fetch_add",
 864         "__tsan_atomic64_fetch_add",
 865         "__tsan_atomic8_fetch_sub",
 866         "__tsan_atomic16_fetch_sub",
 867         "__tsan_atomic32_fetch_sub",
 868         "__tsan_atomic64_fetch_sub",
 869         "__tsan_atomic8_fetch_and",
 870         "__tsan_atomic16_fetch_and",
 871         "__tsan_atomic32_fetch_and",
 872         "__tsan_atomic64_fetch_and",
 873         "__tsan_atomic8_fetch_or",
 874         "__tsan_atomic16_fetch_or",
 875         "__tsan_atomic32_fetch_or",
 876         "__tsan_atomic64_fetch_or",
 877         "__tsan_atomic8_fetch_xor",
 878         "__tsan_atomic16_fetch_xor",
 879         "__tsan_atomic32_fetch_xor",
 880         "__tsan_atomic64_fetch_xor",
 881         "__tsan_atomic8_fetch_nand",
 882         "__tsan_atomic16_fetch_nand",
 883         "__tsan_atomic32_fetch_nand",
 884         "__tsan_atomic64_fetch_nand",
 885         "__tsan_atomic8_compare_exchange_strong",
 886         "__tsan_atomic16_compare_exchange_strong",
 887         "__tsan_atomic32_compare_exchange_strong",
 888         "__tsan_atomic64_compare_exchange_strong",
 889         "__tsan_atomic8_compare_exchange_weak",
 890         "__tsan_atomic16_compare_exchange_weak",
 891         "__tsan_atomic32_compare_exchange_weak",
 892         "__tsan_atomic64_compare_exchange_weak",
 893         "__tsan_atomic8_compare_exchange_val",
 894         "__tsan_atomic16_compare_exchange_val",
 895         "__tsan_atomic32_compare_exchange_val",
 896         "__tsan_atomic64_compare_exchange_val",
 897         "__tsan_atomic_thread_fence",
 898         "__tsan_atomic_signal_fence",
 899         /* KCOV */
 900         "write_comp_data",
 901         "check_kcov_mode",
 902         "__sanitizer_cov_trace_pc",
 903         "__sanitizer_cov_trace_const_cmp1",
 904         "__sanitizer_cov_trace_const_cmp2",
 905         "__sanitizer_cov_trace_const_cmp4",
 906         "__sanitizer_cov_trace_const_cmp8",
 907         "__sanitizer_cov_trace_cmp1",
 908         "__sanitizer_cov_trace_cmp2",
 909         "__sanitizer_cov_trace_cmp4",
 910         "__sanitizer_cov_trace_cmp8",
 911         "__sanitizer_cov_trace_switch",
 912         /* UBSAN */
 913         "ubsan_type_mismatch_common",
 914         "__ubsan_handle_type_mismatch",
 915         "__ubsan_handle_type_mismatch_v1",
 916         "__ubsan_handle_shift_out_of_bounds",
 917         /* misc */
 918         "csum_partial_copy_generic",
 919         "copy_mc_fragile",
 920         "copy_mc_fragile_handle_tail",
 921         "copy_mc_enhanced_fast_string",
 922         "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
 923         NULL
 924 };
 925
 926 static void add_uaccess_safe(struct objtool_file *file)
 927 {
 928         struct symbol *func;
 929         const char **name;
 930
 931         if (!uaccess)
 932                 return;
 933
 934         for (name = uaccess_safe_builtin; *name; name++) {
 935                 func = find_symbol_by_name(file->elf, *name);
 936                 if (!func)
 937                         continue;
 938
 939                 func->uaccess_safe = true;
 940         }
 941 }
 942
 943 /*
 944  * FIXME: For now, just ignore any alternatives which add retpolines.  This is
 945  * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
 946  * But it at least allows objtool to understand the control flow *around* the
 947  * retpoline.
 948  */
 949 static int add_ignore_alternatives(struct objtool_file *file)
 950 {
 951         struct section *sec;
 952         struct reloc *reloc;
 953         struct instruction *insn;
 954
 955         sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
 956         if (!sec)
 957                 return 0;
 958
 959         list_for_each_entry(reloc, &sec->reloc_list, list) {
 960                 if (reloc->sym->type != STT_SECTION) {
 961                         WARN("unexpected relocation symbol type in %s", sec->name);
 962                         return -1;
 963                 }
 964
 965                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
 966                 if (!insn) {
 967                         WARN("bad .discard.ignore_alts entry");
 968                         return -1;
 969                 }
 970
 971                 insn->ignore_alts = true;
 972         }
 973
 974         return 0;
 975 }
 976
 977 __weak bool arch_is_retpoline(struct symbol *sym)
 978 {
 979         return false;
 980 }
 981
 982 __weak bool arch_is_rethunk(struct symbol *sym)
 983 {
 984         return false;
 985 }
 986
 987 #define NEGATIVE_RELOC  ((void *)-1L)
 988
 989 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
 990 {
 991         if (insn->reloc == NEGATIVE_RELOC)
 992                 return NULL;
 993
 994         if (!insn->reloc) {
 995                 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
 996                                                        insn->offset, insn->len);
 997                 if (!insn->reloc) {
 998                         insn->reloc = NEGATIVE_RELOC;
 999                         return NULL;
1000                 }
1001         }
1002
1003         return insn->reloc;
1004 }
1005
1006 static void remove_insn_ops(struct instruction *insn)
1007 {
1008         struct stack_op *op, *tmp;
1009
1010         list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
1011                 list_del(&op->list);
1012                 free(op);
1013         }
1014 }
1015
1016 static void annotate_call_site(struct objtool_file *file,
1017                                struct instruction *insn, bool sibling)
1018 {
1019         struct reloc *reloc = insn_reloc(file, insn);
1020         struct symbol *sym = insn->call_dest;
1021
1022         if (!sym)
1023                 sym = reloc->sym;
1024
1025         /*
1026          * Alternative replacement code is just template code which is
1027          * sometimes copied to the original instruction. For now, don't
1028          * annotate it. (In the future we might consider annotating the
1029          * original instruction if/when it ever makes sense to do so.)
1030          */
1031         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1032                 return;
1033
1034         if (sym->static_call_tramp) {
1035                 list_add_tail(&insn->call_node, &file->static_call_list);
1036                 return;
1037         }
1038
1039         if (sym->retpoline_thunk) {
1040                 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1041                 return;
1042         }
1043
1044         /*
1045          * Many compilers cannot disable KCOV with a function attribute
1046          * so they need a little help, NOP out any KCOV calls from noinstr
1047          * text.
1048          */
1049         if (insn->sec->noinstr && sym->kcov) {
1050                 if (reloc) {
1051                         reloc->type = R_NONE;
1052                         elf_write_reloc(file->elf, reloc);
1053                 }
1054
1055                 elf_write_insn(file->elf, insn->sec,
1056                                insn->offset, insn->len,
1057                                sibling ? arch_ret_insn(insn->len)
1058                                        : arch_nop_insn(insn->len));
1059
1060                 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1061
1062                 if (sibling) {
1063                         /*
1064                          * We've replaced the tail-call JMP insn by two new
1065                          * insn: RET; INT3, except we only have a single struct
1066                          * insn here. Mark it retpoline_safe to avoid the SLS
1067                          * warning, instead of adding another insn.
1068                          */
1069                         insn->retpoline_safe = true;
1070                 }
1071
1072                 return;
1073         }
1074
1075         if (mcount && sym->fentry) {
1076                 if (sibling)
1077                         WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
1078
1079                 if (reloc) {
1080                         reloc->type = R_NONE;
1081                         elf_write_reloc(file->elf, reloc);
1082                 }
1083
1084                 elf_write_insn(file->elf, insn->sec,
1085                                insn->offset, insn->len,
1086                                arch_nop_insn(insn->len));
1087
1088                 insn->type = INSN_NOP;
1089
1090                 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1091                 return;
1092         }
1093 }
1094
1095 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1096                           struct symbol *dest, bool sibling)
1097 {
1098         insn->call_dest = dest;
1099         if (!dest)
1100                 return;
1101
1102         /*
1103          * Whatever stack impact regular CALLs have, should be undone
1104          * by the RETURN of the called function.
1105          *
1106          * Annotated intra-function calls retain the stack_ops but
1107          * are converted to JUMP, see read_intra_function_calls().
1108          */
1109         remove_insn_ops(insn);
1110
1111         annotate_call_site(file, insn, sibling);
1112 }
1113
1114 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1115 {
1116         /*
1117          * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1118          * so convert them accordingly.
1119          */
1120         switch (insn->type) {
1121         case INSN_CALL:
1122                 insn->type = INSN_CALL_DYNAMIC;
1123                 break;
1124         case INSN_JUMP_UNCONDITIONAL:
1125                 insn->type = INSN_JUMP_DYNAMIC;
1126                 break;
1127         case INSN_JUMP_CONDITIONAL:
1128                 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1129                 break;
1130         default:
1131                 return;
1132         }
1133
1134         insn->retpoline_safe = true;
1135
1136         /*
1137          * Whatever stack impact regular CALLs have, should be undone
1138          * by the RETURN of the called function.
1139          *
1140          * Annotated intra-function calls retain the stack_ops but
1141          * are converted to JUMP, see read_intra_function_calls().
1142          */
1143         remove_insn_ops(insn);
1144
1145         annotate_call_site(file, insn, false);
1146 }
1147
1148 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1149 {
1150         /*
1151          * Return thunk tail calls are really just returns in disguise,
1152          * so convert them accordingly.
1153          */
1154         insn->type = INSN_RETURN;
1155         insn->retpoline_safe = true;
1156
1157         /* Skip the non-text sections, specially .discard ones */
1158         if (add && insn->sec->text)
1159                 list_add_tail(&insn->call_node, &file->return_thunk_list);
1160 }
1161
1162 /*
1163  * Find the destination instructions for all jumps.
1164  */
1165 static int add_jump_destinations(struct objtool_file *file)
1166 {
1167         struct instruction *insn;
1168         struct reloc *reloc;
1169         struct section *dest_sec;
1170         unsigned long dest_off;
1171
1172         for_each_insn(file, insn) {
1173                 if (!is_static_jump(insn))
1174                         continue;
1175
1176                 reloc = insn_reloc(file, insn);
1177                 if (!reloc) {
1178                         dest_sec = insn->sec;
1179                         dest_off = arch_jump_destination(insn);
1180                 } else if (reloc->sym->type == STT_SECTION) {
1181                         dest_sec = reloc->sym->sec;
1182                         dest_off = arch_dest_reloc_offset(reloc->addend);
1183                 } else if (reloc->sym->retpoline_thunk) {
1184                         add_retpoline_call(file, insn);
1185                         continue;
1186                 } else if (reloc->sym->return_thunk) {
1187                         add_return_call(file, insn, true);
1188                         continue;
1189                 } else if (insn->func) {
1190                         /* internal or external sibling call (with reloc) */
1191                         add_call_dest(file, insn, reloc->sym, true);
1192                         continue;
1193                 } else if (reloc->sym->sec->idx) {
1194                         dest_sec = reloc->sym->sec;
1195                         dest_off = reloc->sym->sym.st_value +
1196                                    arch_dest_reloc_offset(reloc->addend);
1197                 } else {
1198                         /* non-func asm code jumping to another file */
1199                         continue;
1200                 }
1201
1202                 insn->jump_dest = find_insn(file, dest_sec, dest_off);
1203                 if (!insn->jump_dest) {
1204                         struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1205
1206                         /*
1207                          * This is a special case where an alt instruction
1208                          * jumps past the end of the section.  These are
1209                          * handled later in handle_group_alt().
1210                          */
1211                         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1212                                 continue;
1213
1214                         /*
1215                          * This is a special case for zen_untrain_ret().
1216                          * It jumps to __x86_return_thunk(), but objtool
1217                          * can't find the thunk's starting RET
1218                          * instruction, because the RET is also in the
1219                          * middle of another instruction.  Objtool only
1220                          * knows about the outer instruction.
1221                          */
1222                         if (sym && sym->return_thunk) {
1223                                 add_return_call(file, insn, false);
1224                                 continue;
1225                         }
1226
1227                         WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
1228                                   insn->sec, insn->offset, dest_sec->name,
1229                                   dest_off);
1230                         return -1;
1231                 }
1232
1233                 /*
1234                  * Cross-function jump.
1235                  */
1236                 if (insn->func && insn->jump_dest->func &&
1237                     insn->func != insn->jump_dest->func) {
1238
1239                         /*
1240                          * For GCC 8+, create parent/child links for any cold
1241                          * subfunctions.  This is _mostly_ redundant with a
1242                          * similar initialization in read_symbols().
1243                          *
1244                          * If a function has aliases, we want the *first* such
1245                          * function in the symbol table to be the subfunction's
1246                          * parent.  In that case we overwrite the
1247                          * initialization done in read_symbols().
1248                          *
1249                          * However this code can't completely replace the
1250                          * read_symbols() code because this doesn't detect the
1251                          * case where the parent function's only reference to a
1252                          * subfunction is through a jump table.
1253                          */
1254                         if (!strstr(insn->func->name, ".cold") &&
1255                             strstr(insn->jump_dest->func->name, ".cold")) {
1256                                 insn->func->cfunc = insn->jump_dest->func;
1257                                 insn->jump_dest->func->pfunc = insn->func;
1258
1259                         } else if (insn->jump_dest->func->pfunc != insn->func->pfunc &&
1260                                    insn->jump_dest->offset == insn->jump_dest->func->offset) {
1261                                 /* internal sibling call (without reloc) */
1262                                 add_call_dest(file, insn, insn->jump_dest->func, true);
1263                         }
1264                 }
1265         }
1266
1267         return 0;
1268 }
1269
1270 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1271 {
1272         struct symbol *call_dest;
1273
1274         call_dest = find_func_by_offset(sec, offset);
1275         if (!call_dest)
1276                 call_dest = find_symbol_by_offset(sec, offset);
1277
1278         return call_dest;
1279 }
1280
1281 /*
1282  * Find the destination instructions for all calls.
1283  */
1284 static int add_call_destinations(struct objtool_file *file)
1285 {
1286         struct instruction *insn;
1287         unsigned long dest_off;
1288         struct symbol *dest;
1289         struct reloc *reloc;
1290
1291         for_each_insn(file, insn) {
1292                 if (insn->type != INSN_CALL)
1293                         continue;
1294
1295                 reloc = insn_reloc(file, insn);
1296                 if (!reloc) {
1297                         dest_off = arch_jump_destination(insn);
1298                         dest = find_call_destination(insn->sec, dest_off);
1299
1300                         add_call_dest(file, insn, dest, false);
1301
1302                         if (insn->ignore)
1303                                 continue;
1304
1305                         if (!insn->call_dest) {
1306                                 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
1307                                 return -1;
1308                         }
1309
1310                         if (insn->func && insn->call_dest->type != STT_FUNC) {
1311                                 WARN_FUNC("unsupported call to non-function",
1312                                           insn->sec, insn->offset);
1313                                 return -1;
1314                         }
1315
1316                 } else if (reloc->sym->type == STT_SECTION) {
1317                         dest_off = arch_dest_reloc_offset(reloc->addend);
1318                         dest = find_call_destination(reloc->sym->sec, dest_off);
1319                         if (!dest) {
1320                                 WARN_FUNC("can't find call dest symbol at %s+0x%lx",
1321                                           insn->sec, insn->offset,
1322                                           reloc->sym->sec->name,
1323                                           dest_off);
1324                                 return -1;
1325                         }
1326
1327                         add_call_dest(file, insn, dest, false);
1328
1329                 } else if (reloc->sym->retpoline_thunk) {
1330                         add_retpoline_call(file, insn);
1331
1332                 } else
1333                         add_call_dest(file, insn, reloc->sym, false);
1334         }
1335
1336         return 0;
1337 }
1338
1339 /*
1340  * The .alternatives section requires some extra special care over and above
1341  * other special sections because alternatives are patched in place.
1342  */
1343 static int handle_group_alt(struct objtool_file *file,
1344                             struct special_alt *special_alt,
1345                             struct instruction *orig_insn,
1346                             struct instruction **new_insn)
1347 {
1348         struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
1349         struct alt_group *orig_alt_group, *new_alt_group;
1350         unsigned long dest_off;
1351
1352
1353         orig_alt_group = malloc(sizeof(*orig_alt_group));
1354         if (!orig_alt_group) {
1355                 WARN("malloc failed");
1356                 return -1;
1357         }
1358         orig_alt_group->cfi = calloc(special_alt->orig_len,
1359                                      sizeof(struct cfi_state *));
1360         if (!orig_alt_group->cfi) {
1361                 WARN("calloc failed");
1362                 return -1;
1363         }
1364
1365         last_orig_insn = NULL;
1366         insn = orig_insn;
1367         sec_for_each_insn_from(file, insn) {
1368                 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1369                         break;
1370
1371                 insn->alt_group = orig_alt_group;
1372                 last_orig_insn = insn;
1373         }
1374         orig_alt_group->orig_group = NULL;
1375         orig_alt_group->first_insn = orig_insn;
1376         orig_alt_group->last_insn = last_orig_insn;
1377
1378
1379         new_alt_group = malloc(sizeof(*new_alt_group));
1380         if (!new_alt_group) {
1381                 WARN("malloc failed");
1382                 return -1;
1383         }
1384
1385         if (special_alt->new_len < special_alt->orig_len) {
1386                 /*
1387                  * Insert a fake nop at the end to make the replacement
1388                  * alt_group the same size as the original.  This is needed to
1389                  * allow propagate_alt_cfi() to do its magic.  When the last
1390                  * instruction affects the stack, the instruction after it (the
1391                  * nop) will propagate the new state to the shared CFI array.
1392                  */
1393                 nop = malloc(sizeof(*nop));
1394                 if (!nop) {
1395                         WARN("malloc failed");
1396                         return -1;
1397                 }
1398                 memset(nop, 0, sizeof(*nop));
1399                 INIT_LIST_HEAD(&nop->alts);
1400                 INIT_LIST_HEAD(&nop->stack_ops);
1401
1402                 nop->sec = special_alt->new_sec;
1403                 nop->offset = special_alt->new_off + special_alt->new_len;
1404                 nop->len = special_alt->orig_len - special_alt->new_len;
1405                 nop->type = INSN_NOP;
1406                 nop->func = orig_insn->func;
1407                 nop->alt_group = new_alt_group;
1408                 nop->ignore = orig_insn->ignore_alts;
1409         }
1410
1411         if (!special_alt->new_len) {
1412                 *new_insn = nop;
1413                 goto end;
1414         }
1415
1416         insn = *new_insn;
1417         sec_for_each_insn_from(file, insn) {
1418                 struct reloc *alt_reloc;
1419
1420                 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1421                         break;
1422
1423                 last_new_insn = insn;
1424
1425                 insn->ignore = orig_insn->ignore_alts;
1426                 insn->func = orig_insn->func;
1427                 insn->alt_group = new_alt_group;
1428
1429                 /*
1430                  * Since alternative replacement code is copy/pasted by the
1431                  * kernel after applying relocations, generally such code can't
1432                  * have relative-address relocation references to outside the
1433                  * .altinstr_replacement section, unless the arch's
1434                  * alternatives code can adjust the relative offsets
1435                  * accordingly.
1436                  */
1437                 alt_reloc = insn_reloc(file, insn);
1438                 if (alt_reloc &&
1439                     !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1440
1441                         WARN_FUNC("unsupported relocation in alternatives section",
1442                                   insn->sec, insn->offset);
1443                         return -1;
1444                 }
1445
1446                 if (!is_static_jump(insn))
1447                         continue;
1448
1449                 if (!insn->immediate)
1450                         continue;
1451
1452                 dest_off = arch_jump_destination(insn);
1453                 if (dest_off == special_alt->new_off + special_alt->new_len)
1454                         insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
1455
1456                 if (!insn->jump_dest) {
1457                         WARN_FUNC("can't find alternative jump destination",
1458                                   insn->sec, insn->offset);
1459                         return -1;
1460                 }
1461         }
1462
1463         if (!last_new_insn) {
1464                 WARN_FUNC("can't find last new alternative instruction",
1465                           special_alt->new_sec, special_alt->new_off);
1466                 return -1;
1467         }
1468
1469         if (nop)
1470                 list_add(&nop->list, &last_new_insn->list);
1471 end:
1472         new_alt_group->orig_group = orig_alt_group;
1473         new_alt_group->first_insn = *new_insn;
1474         new_alt_group->last_insn = nop ? : last_new_insn;
1475         new_alt_group->cfi = orig_alt_group->cfi;
1476         return 0;
1477 }
1478
1479 /*
1480  * A jump table entry can either convert a nop to a jump or a jump to a nop.
1481  * If the original instruction is a jump, make the alt entry an effective nop
1482  * by just skipping the original instruction.
1483  */
1484 static int handle_jump_alt(struct objtool_file *file,
1485                            struct special_alt *special_alt,
1486                            struct instruction *orig_insn,
1487                            struct instruction **new_insn)
1488 {
1489         if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1490             orig_insn->type != INSN_NOP) {
1491
1492                 WARN_FUNC("unsupported instruction at jump label",
1493                           orig_insn->sec, orig_insn->offset);
1494                 return -1;
1495         }
1496
1497         if (special_alt->key_addend & 2) {
1498                 struct reloc *reloc = insn_reloc(file, orig_insn);
1499
1500                 if (reloc) {
1501                         reloc->type = R_NONE;
1502                         elf_write_reloc(file->elf, reloc);
1503                 }
1504                 elf_write_insn(file->elf, orig_insn->sec,
1505                                orig_insn->offset, orig_insn->len,
1506                                arch_nop_insn(orig_insn->len));
1507                 orig_insn->type = INSN_NOP;
1508         }
1509
1510         if (orig_insn->type == INSN_NOP) {
1511                 if (orig_insn->len == 2)
1512                         file->jl_nop_short++;
1513                 else
1514                         file->jl_nop_long++;
1515
1516                 return 0;
1517         }
1518
1519         if (orig_insn->len == 2)
1520                 file->jl_short++;
1521         else
1522                 file->jl_long++;
1523
1524         *new_insn = list_next_entry(orig_insn, list);
1525         return 0;
1526 }
1527
1528 /*
1529  * Read all the special sections which have alternate instructions which can be
1530  * patched in or redirected to at runtime.  Each instruction having alternate
1531  * instruction(s) has them added to its insn->alts list, which will be
1532  * traversed in validate_branch().
1533  */
1534 static int add_special_section_alts(struct objtool_file *file)
1535 {
1536         struct list_head special_alts;
1537         struct instruction *orig_insn, *new_insn;
1538         struct special_alt *special_alt, *tmp;
1539         struct alternative *alt;
1540         int ret;
1541
1542         ret = special_get_alts(file->elf, &special_alts);
1543         if (ret)
1544                 return ret;
1545
1546         list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1547
1548                 orig_insn = find_insn(file, special_alt->orig_sec,
1549                                       special_alt->orig_off);
1550                 if (!orig_insn) {
1551                         WARN_FUNC("special: can't find orig instruction",
1552                                   special_alt->orig_sec, special_alt->orig_off);
1553                         ret = -1;
1554                         goto out;
1555                 }
1556
1557                 new_insn = NULL;
1558                 if (!special_alt->group || special_alt->new_len) {
1559                         new_insn = find_insn(file, special_alt->new_sec,
1560                                              special_alt->new_off);
1561                         if (!new_insn) {
1562                                 WARN_FUNC("special: can't find new instruction",
1563                                           special_alt->new_sec,
1564                                           special_alt->new_off);
1565                                 ret = -1;
1566                                 goto out;
1567                         }
1568                 }
1569
1570                 if (special_alt->group) {
1571                         if (!special_alt->orig_len) {
1572                                 WARN_FUNC("empty alternative entry",
1573                                           orig_insn->sec, orig_insn->offset);
1574                                 continue;
1575                         }
1576
1577                         ret = handle_group_alt(file, special_alt, orig_insn,
1578                                                &new_insn);
1579                         if (ret)
1580                                 goto out;
1581                 } else if (special_alt->jump_or_nop) {
1582                         ret = handle_jump_alt(file, special_alt, orig_insn,
1583                                               &new_insn);
1584                         if (ret)
1585                                 goto out;
1586                 }
1587
1588                 alt = malloc(sizeof(*alt));
1589                 if (!alt) {
1590                         WARN("malloc failed");
1591                         ret = -1;
1592                         goto out;
1593                 }
1594
1595                 alt->insn = new_insn;
1596                 alt->skip_orig = special_alt->skip_orig;
1597                 orig_insn->ignore_alts |= special_alt->skip_alt;
1598                 list_add_tail(&alt->list, &orig_insn->alts);
1599
1600                 list_del(&special_alt->list);
1601                 free(special_alt);
1602         }
1603
1604         if (stats) {
1605                 printf("jl\\\tNOP\tJMP\n");
1606                 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1607                 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1608         }
1609
1610 out:
1611         return ret;
1612 }
1613
1614 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
1615                             struct reloc *table)
1616 {
1617         struct reloc *reloc = table;
1618         struct instruction *dest_insn;
1619         struct alternative *alt;
1620         struct symbol *pfunc = insn->func->pfunc;
1621         unsigned int prev_offset = 0;
1622
1623         /*
1624          * Each @reloc is a switch table relocation which points to the target
1625          * instruction.
1626          */
1627         list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
1628
1629                 /* Check for the end of the table: */
1630                 if (reloc != table && reloc->jump_table_start)
1631                         break;
1632
1633                 /* Make sure the table entries are consecutive: */
1634                 if (prev_offset && reloc->offset != prev_offset + 8)
1635                         break;
1636
1637                 /* Detect function pointers from contiguous objects: */
1638                 if (reloc->sym->sec == pfunc->sec &&
1639                     reloc->addend == pfunc->offset)
1640                         break;
1641
1642                 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
1643                 if (!dest_insn)
1644                         break;
1645
1646                 /* Make sure the destination is in the same function: */
1647                 if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
1648                         break;
1649
1650                 alt = malloc(sizeof(*alt));
1651                 if (!alt) {
1652                         WARN("malloc failed");
1653                         return -1;
1654                 }
1655
1656                 alt->insn = dest_insn;
1657                 list_add_tail(&alt->list, &insn->alts);
1658                 prev_offset = reloc->offset;
1659         }
1660
1661         if (!prev_offset) {
1662                 WARN_FUNC("can't find switch jump table",
1663                           insn->sec, insn->offset);
1664                 return -1;
1665         }
1666
1667         return 0;
1668 }
1669
1670 /*
1671  * find_jump_table() - Given a dynamic jump, find the switch jump table
1672  * associated with it.
1673  */
1674 static struct reloc *find_jump_table(struct objtool_file *file,
1675                                       struct symbol *func,
1676                                       struct instruction *insn)
1677 {
1678         struct reloc *table_reloc;
1679         struct instruction *dest_insn, *orig_insn = insn;
1680
1681         /*
1682          * Backward search using the @first_jump_src links, these help avoid
1683          * much of the 'in between' code. Which avoids us getting confused by
1684          * it.
1685          */
1686         for (;
1687              insn && insn->func && insn->func->pfunc == func;
1688              insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
1689
1690                 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
1691                         break;
1692
1693                 /* allow small jumps within the range */
1694                 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
1695                     insn->jump_dest &&
1696                     (insn->jump_dest->offset <= insn->offset ||
1697                      insn->jump_dest->offset > orig_insn->offset))
1698                     break;
1699
1700                 table_reloc = arch_find_switch_table(file, insn);
1701                 if (!table_reloc)
1702                         continue;
1703                 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
1704                 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
1705                         continue;
1706
1707                 return table_reloc;
1708         }
1709
1710         return NULL;
1711 }
1712
1713 /*
1714  * First pass: Mark the head of each jump table so that in the next pass,
1715  * we know when a given jump table ends and the next one starts.
1716  */
1717 static void mark_func_jump_tables(struct objtool_file *file,
1718                                     struct symbol *func)
1719 {
1720         struct instruction *insn, *last = NULL;
1721         struct reloc *reloc;
1722
1723         func_for_each_insn(file, func, insn) {
1724                 if (!last)
1725                         last = insn;
1726
1727                 /*
1728                  * Store back-pointers for unconditional forward jumps such
1729                  * that find_jump_table() can back-track using those and
1730                  * avoid some potentially confusing code.
1731                  */
1732                 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
1733                     insn->offset > last->offset &&
1734                     insn->jump_dest->offset > insn->offset &&
1735                     !insn->jump_dest->first_jump_src) {
1736
1737                         insn->jump_dest->first_jump_src = insn;
1738                         last = insn->jump_dest;
1739                 }
1740
1741                 if (insn->type != INSN_JUMP_DYNAMIC)
1742                         continue;
1743
1744                 reloc = find_jump_table(file, func, insn);
1745                 if (reloc) {
1746                         reloc->jump_table_start = true;
1747                         insn->jump_table = reloc;
1748                 }
1749         }
1750 }
1751
1752 static int add_func_jump_tables(struct objtool_file *file,
1753                                   struct symbol *func)
1754 {
1755         struct instruction *insn;
1756         int ret;
1757
1758         func_for_each_insn(file, func, insn) {
1759                 if (!insn->jump_table)
1760                         continue;
1761
1762                 ret = add_jump_table(file, insn, insn->jump_table);
1763                 if (ret)
1764                         return ret;
1765         }
1766
1767         return 0;
1768 }
1769
1770 /*
1771  * For some switch statements, gcc generates a jump table in the .rodata
1772  * section which contains a list of addresses within the function to jump to.
1773  * This finds these jump tables and adds them to the insn->alts lists.
1774  */
1775 static int add_jump_table_alts(struct objtool_file *file)
1776 {
1777         struct section *sec;
1778         struct symbol *func;
1779         int ret;
1780
1781         if (!file->rodata)
1782                 return 0;
1783
1784         for_each_sec(file, sec) {
1785                 list_for_each_entry(func, &sec->symbol_list, list) {
1786                         if (func->type != STT_FUNC)
1787                                 continue;
1788
1789                         mark_func_jump_tables(file, func);
1790                         ret = add_func_jump_tables(file, func);
1791                         if (ret)
1792                                 return ret;
1793                 }
1794         }
1795
1796         return 0;
1797 }
1798
1799 static void set_func_state(struct cfi_state *state)
1800 {
1801         state->cfa = initial_func_cfi.cfa;
1802         memcpy(&state->regs, &initial_func_cfi.regs,
1803                CFI_NUM_REGS * sizeof(struct cfi_reg));
1804         state->stack_size = initial_func_cfi.cfa.offset;
1805 }
1806
1807 static int read_unwind_hints(struct objtool_file *file)
1808 {
1809         struct cfi_state cfi = init_cfi;
1810         struct section *sec, *relocsec;
1811         struct unwind_hint *hint;
1812         struct instruction *insn;
1813         struct reloc *reloc;
1814         int i;
1815
1816         sec = find_section_by_name(file->elf, ".discard.unwind_hints");
1817         if (!sec)
1818                 return 0;
1819
1820         relocsec = sec->reloc;
1821         if (!relocsec) {
1822                 WARN("missing .rela.discard.unwind_hints section");
1823                 return -1;
1824         }
1825
1826         if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
1827                 WARN("struct unwind_hint size mismatch");
1828                 return -1;
1829         }
1830
1831         file->hints = true;
1832
1833         for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
1834                 hint = (struct unwind_hint *)sec->data->d_buf + i;
1835
1836                 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
1837                 if (!reloc) {
1838                         WARN("can't find reloc for unwind_hints[%d]", i);
1839                         return -1;
1840                 }
1841
1842                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1843                 if (!insn) {
1844                         WARN("can't find insn for unwind_hints[%d]", i);
1845                         return -1;
1846                 }
1847
1848                 insn->hint = true;
1849
1850                 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
1851                         insn->cfi = &func_cfi;
1852                         continue;
1853                 }
1854
1855                 if (insn->cfi)
1856                         cfi = *(insn->cfi);
1857
1858                 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
1859                         WARN_FUNC("unsupported unwind_hint sp base reg %d",
1860                                   insn->sec, insn->offset, hint->sp_reg);
1861                         return -1;
1862                 }
1863
1864                 cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
1865                 cfi.type = hint->type;
1866                 cfi.end = hint->end;
1867
1868                 insn->cfi = cfi_hash_find_or_add(&cfi);
1869         }
1870
1871         return 0;
1872 }
1873
1874 static int read_retpoline_hints(struct objtool_file *file)
1875 {
1876         struct section *sec;
1877         struct instruction *insn;
1878         struct reloc *reloc;
1879
1880         sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
1881         if (!sec)
1882                 return 0;
1883
1884         list_for_each_entry(reloc, &sec->reloc_list, list) {
1885                 if (reloc->sym->type != STT_SECTION) {
1886                         WARN("unexpected relocation symbol type in %s", sec->name);
1887                         return -1;
1888                 }
1889
1890                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1891                 if (!insn) {
1892                         WARN("bad .discard.retpoline_safe entry");
1893                         return -1;
1894                 }
1895
1896                 if (insn->type != INSN_JUMP_DYNAMIC &&
1897                     insn->type != INSN_CALL_DYNAMIC &&
1898                     insn->type != INSN_RETURN) {
1899                         WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret",
1900                                   insn->sec, insn->offset);
1901                         return -1;
1902                 }
1903
1904                 insn->retpoline_safe = true;
1905         }
1906
1907         return 0;
1908 }
1909
1910 static int read_instr_hints(struct objtool_file *file)
1911 {
1912         struct section *sec;
1913         struct instruction *insn;
1914         struct reloc *reloc;
1915
1916         sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
1917         if (!sec)
1918                 return 0;
1919
1920         list_for_each_entry(reloc, &sec->reloc_list, list) {
1921                 if (reloc->sym->type != STT_SECTION) {
1922                         WARN("unexpected relocation symbol type in %s", sec->name);
1923                         return -1;
1924                 }
1925
1926                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1927                 if (!insn) {
1928                         WARN("bad .discard.instr_end entry");
1929                         return -1;
1930                 }
1931
1932                 insn->instr--;
1933         }
1934
1935         sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
1936         if (!sec)
1937                 return 0;
1938
1939         list_for_each_entry(reloc, &sec->reloc_list, list) {
1940                 if (reloc->sym->type != STT_SECTION) {
1941                         WARN("unexpected relocation symbol type in %s", sec->name);
1942                         return -1;
1943                 }
1944
1945                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1946                 if (!insn) {
1947                         WARN("bad .discard.instr_begin entry");
1948                         return -1;
1949                 }
1950
1951                 insn->instr++;
1952         }
1953
1954         return 0;
1955 }
1956
1957 static int read_intra_function_calls(struct objtool_file *file)
1958 {
1959         struct instruction *insn;
1960         struct section *sec;
1961         struct reloc *reloc;
1962
1963         sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
1964         if (!sec)
1965                 return 0;
1966
1967         list_for_each_entry(reloc, &sec->reloc_list, list) {
1968                 unsigned long dest_off;
1969
1970                 if (reloc->sym->type != STT_SECTION) {
1971                         WARN("unexpected relocation symbol type in %s",
1972                              sec->name);
1973                         return -1;
1974                 }
1975
1976                 insn = find_insn(file, reloc->sym->sec, reloc->addend);
1977                 if (!insn) {
1978                         WARN("bad .discard.intra_function_call entry");
1979                         return -1;
1980                 }
1981
1982                 if (insn->type != INSN_CALL) {
1983                         WARN_FUNC("intra_function_call not a direct call",
1984                                   insn->sec, insn->offset);
1985                         return -1;
1986                 }
1987
1988                 /*
1989                  * Treat intra-function CALLs as JMPs, but with a stack_op.
1990                  * See add_call_destinations(), which strips stack_ops from
1991                  * normal CALLs.
1992                  */
1993                 insn->type = INSN_JUMP_UNCONDITIONAL;
1994
1995                 dest_off = insn->offset + insn->len + insn->immediate;
1996                 insn->jump_dest = find_insn(file, insn->sec, dest_off);
1997                 if (!insn->jump_dest) {
1998                         WARN_FUNC("can't find call dest at %s+0x%lx",
1999                                   insn->sec, insn->offset,
2000                                   insn->sec->name, dest_off);
2001                         return -1;
2002                 }
2003         }
2004
2005         return 0;
2006 }
2007
2008 static int classify_symbols(struct objtool_file *file)
2009 {
2010         struct section *sec;
2011         struct symbol *func;
2012
2013         for_each_sec(file, sec) {
2014                 list_for_each_entry(func, &sec->symbol_list, list) {
2015                         if (func->bind != STB_GLOBAL)
2016                                 continue;
2017
2018                         if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2019                                      strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2020                                 func->static_call_tramp = true;
2021
2022                         if (arch_is_retpoline(func))
2023                                 func->retpoline_thunk = true;
2024
2025                         if (arch_is_rethunk(func))
2026                                 func->return_thunk = true;
2027
2028                         if (!strcmp(func->name, "__fentry__"))
2029                                 func->fentry = true;
2030
2031                         if (!strncmp(func->name, "__sanitizer_cov_", 16))
2032                                 func->kcov = true;
2033                 }
2034         }
2035
2036         return 0;
2037 }
2038
2039 static void mark_rodata(struct objtool_file *file)
2040 {
2041         struct section *sec;
2042         bool found = false;
2043
2044         /*
2045          * Search for the following rodata sections, each of which can
2046          * potentially contain jump tables:
2047          *
2048          * - .rodata: can contain GCC switch tables
2049          * - .rodata.<func>: same, if -fdata-sections is being used
2050          * - .rodata..c_jump_table: contains C annotated jump tables
2051          *
2052          * .rodata.str1.* sections are ignored; they don't contain jump tables.
2053          */
2054         for_each_sec(file, sec) {
2055                 if (!strncmp(sec->name, ".rodata", 7) &&
2056                     !strstr(sec->name, ".str1.")) {
2057                         sec->rodata = true;
2058                         found = true;
2059                 }
2060         }
2061
2062         file->rodata = found;
2063 }
2064
2065 static int decode_sections(struct objtool_file *file)
2066 {
2067         int ret;
2068
2069         mark_rodata(file);
2070
2071         ret = decode_instructions(file);
2072         if (ret)
2073                 return ret;
2074
2075         ret = add_dead_ends(file);
2076         if (ret)
2077                 return ret;
2078
2079         add_ignores(file);
2080         add_uaccess_safe(file);
2081
2082         ret = add_ignore_alternatives(file);
2083         if (ret)
2084                 return ret;
2085
2086         /*
2087          * Must be before add_{jump_call}_destination.
2088          */
2089         ret = classify_symbols(file);
2090         if (ret)
2091                 return ret;
2092
2093         /*
2094          * Must be before add_special_section_alts() as that depends on
2095          * jump_dest being set.
2096          */
2097         ret = add_jump_destinations(file);
2098         if (ret)
2099                 return ret;
2100
2101         ret = add_special_section_alts(file);
2102         if (ret)
2103                 return ret;
2104
2105         /*
2106          * Must be before add_call_destination(); it changes INSN_CALL to
2107          * INSN_JUMP.
2108          */
2109         ret = read_intra_function_calls(file);
2110         if (ret)
2111                 return ret;
2112
2113         ret = add_call_destinations(file);
2114         if (ret)
2115                 return ret;
2116
2117         ret = add_jump_table_alts(file);
2118         if (ret)
2119                 return ret;
2120
2121         ret = read_unwind_hints(file);
2122         if (ret)
2123                 return ret;
2124
2125         ret = read_retpoline_hints(file);
2126         if (ret)
2127                 return ret;
2128
2129         ret = read_instr_hints(file);
2130         if (ret)
2131                 return ret;
2132
2133         return 0;
2134 }
2135
2136 static bool is_fentry_call(struct instruction *insn)
2137 {
2138         if (insn->type == INSN_CALL &&
2139             insn->call_dest &&
2140             insn->call_dest->fentry)
2141                 return true;
2142
2143         return false;
2144 }
2145
2146 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2147 {
2148         struct cfi_state *cfi = &state->cfi;
2149         int i;
2150
2151         if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2152                 return true;
2153
2154         if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2155                 return true;
2156
2157         if (cfi->stack_size != initial_func_cfi.cfa.offset)
2158                 return true;
2159
2160         for (i = 0; i < CFI_NUM_REGS; i++) {
2161                 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2162                     cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2163                         return true;
2164         }
2165
2166         return false;
2167 }
2168
2169 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2170                                 int expected_offset)
2171 {
2172         return reg->base == CFI_CFA &&
2173                reg->offset == expected_offset;
2174 }
2175
2176 static bool has_valid_stack_frame(struct insn_state *state)
2177 {
2178         struct cfi_state *cfi = &state->cfi;
2179
2180         if (cfi->cfa.base == CFI_BP &&
2181             check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2182             check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2183                 return true;
2184
2185         if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2186                 return true;
2187
2188         return false;
2189 }
2190
2191 static int update_cfi_state_regs(struct instruction *insn,
2192                                   struct cfi_state *cfi,
2193                                   struct stack_op *op)
2194 {
2195         struct cfi_reg *cfa = &cfi->cfa;
2196
2197         if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2198                 return 0;
2199
2200         /* push */
2201         if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2202                 cfa->offset += 8;
2203
2204         /* pop */
2205         if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2206                 cfa->offset -= 8;
2207
2208         /* add immediate to sp */
2209         if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2210             op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2211                 cfa->offset -= op->src.offset;
2212
2213         return 0;
2214 }
2215
2216 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2217 {
2218         if (arch_callee_saved_reg(reg) &&
2219             cfi->regs[reg].base == CFI_UNDEFINED) {
2220                 cfi->regs[reg].base = base;
2221                 cfi->regs[reg].offset = offset;
2222         }
2223 }
2224
2225 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2226 {
2227         cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2228         cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2229 }
2230
2231 /*
2232  * A note about DRAP stack alignment:
2233  *
2234  * GCC has the concept of a DRAP register, which is used to help keep track of
2235  * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2236  * register.  The typical DRAP pattern is:
2237  *
2238  *   4c 8d 54 24 08             lea    0x8(%rsp),%r10
2239  *   48 83 e4 c0                and    $0xffffffffffffffc0,%rsp
2240  *   41 ff 72 f8                pushq  -0x8(%r10)
2241  *   55                         push   %rbp
2242  *   48 89 e5                   mov    %rsp,%rbp
2243  *                              (more pushes)
2244  *   41 52                      push   %r10
2245  *                              ...
2246  *   41 5a                      pop    %r10
2247  *                              (more pops)
2248  *   5d                         pop    %rbp
2249  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2250  *   c3                         retq
2251  *
2252  * There are some variations in the epilogues, like:
2253  *
2254  *   5b                         pop    %rbx
2255  *   41 5a                      pop    %r10
2256  *   41 5c                      pop    %r12
2257  *   41 5d                      pop    %r13
2258  *   41 5e                      pop    %r14
2259  *   c9                         leaveq
2260  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2261  *   c3                         retq
2262  *
2263  * and:
2264  *
2265  *   4c 8b 55 e8                mov    -0x18(%rbp),%r10
2266  *   48 8b 5d e0                mov    -0x20(%rbp),%rbx
2267  *   4c 8b 65 f0                mov    -0x10(%rbp),%r12
2268  *   4c 8b 6d f8                mov    -0x8(%rbp),%r13
2269  *   c9                         leaveq
2270  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2271  *   c3                         retq
2272  *
2273  * Sometimes r13 is used as the DRAP register, in which case it's saved and
2274  * restored beforehand:
2275  *
2276  *   41 55                      push   %r13
2277  *   4c 8d 6c 24 10             lea    0x10(%rsp),%r13
2278  *   48 83 e4 f0                and    $0xfffffffffffffff0,%rsp
2279  *                              ...
2280  *   49 8d 65 f0                lea    -0x10(%r13),%rsp
2281  *   41 5d                      pop    %r13
2282  *   c3                         retq
2283  */
2284 static int update_cfi_state(struct instruction *insn,
2285                             struct instruction *next_insn,
2286                             struct cfi_state *cfi, struct stack_op *op)
2287 {
2288         struct cfi_reg *cfa = &cfi->cfa;
2289         struct cfi_reg *regs = cfi->regs;
2290
2291         /* stack operations don't make sense with an undefined CFA */
2292         if (cfa->base == CFI_UNDEFINED) {
2293                 if (insn->func) {
2294                         WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2295                         return -1;
2296                 }
2297                 return 0;
2298         }
2299
2300         if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2301             cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2302                 return update_cfi_state_regs(insn, cfi, op);
2303
2304         switch (op->dest.type) {
2305
2306         case OP_DEST_REG:
2307                 switch (op->src.type) {
2308
2309                 case OP_SRC_REG:
2310                         if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2311                             cfa->base == CFI_SP &&
2312                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2313
2314                                 /* mov %rsp, %rbp */
2315                                 cfa->base = op->dest.reg;
2316                                 cfi->bp_scratch = false;
2317                         }
2318
2319                         else if (op->src.reg == CFI_SP &&
2320                                  op->dest.reg == CFI_BP && cfi->drap) {
2321
2322                                 /* drap: mov %rsp, %rbp */
2323                                 regs[CFI_BP].base = CFI_BP;
2324                                 regs[CFI_BP].offset = -cfi->stack_size;
2325                                 cfi->bp_scratch = false;
2326                         }
2327
2328                         else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2329
2330                                 /*
2331                                  * mov %rsp, %reg
2332                                  *
2333                                  * This is needed for the rare case where GCC
2334                                  * does:
2335                                  *
2336                                  *   mov    %rsp, %rax
2337                                  *   ...
2338                                  *   mov    %rax, %rsp
2339                                  */
2340                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2341                                 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2342                         }
2343
2344                         else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2345                                  (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2346
2347                                 /*
2348                                  * mov %rbp, %rsp
2349                                  *
2350                                  * Restore the original stack pointer (Clang).
2351                                  */
2352                                 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2353                         }
2354
2355                         else if (op->dest.reg == cfa->base) {
2356
2357                                 /* mov %reg, %rsp */
2358                                 if (cfa->base == CFI_SP &&
2359                                     cfi->vals[op->src.reg].base == CFI_CFA) {
2360
2361                                         /*
2362                                          * This is needed for the rare case
2363                                          * where GCC does something dumb like:
2364                                          *
2365                                          *   lea    0x8(%rsp), %rcx
2366                                          *   ...
2367                                          *   mov    %rcx, %rsp
2368                                          */
2369                                         cfa->offset = -cfi->vals[op->src.reg].offset;
2370                                         cfi->stack_size = cfa->offset;
2371
2372                                 } else if (cfa->base == CFI_SP &&
2373                                            cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2374                                            cfi->vals[op->src.reg].offset == cfa->offset) {
2375
2376                                         /*
2377                                          * Stack swizzle:
2378                                          *
2379                                          * 1: mov %rsp, (%[tos])
2380                                          * 2: mov %[tos], %rsp
2381                                          *    ...
2382                                          * 3: pop %rsp
2383                                          *
2384                                          * Where:
2385                                          *
2386                                          * 1 - places a pointer to the previous
2387                                          *     stack at the Top-of-Stack of the
2388                                          *     new stack.
2389                                          *
2390                                          * 2 - switches to the new stack.
2391                                          *
2392                                          * 3 - pops the Top-of-Stack to restore
2393                                          *     the original stack.
2394                                          *
2395                                          * Note: we set base to SP_INDIRECT
2396                                          * here and preserve offset. Therefore
2397                                          * when the unwinder reaches ToS it
2398                                          * will dereference SP and then add the
2399                                          * offset to find the next frame, IOW:
2400                                          * (%rsp) + offset.
2401                                          */
2402                                         cfa->base = CFI_SP_INDIRECT;
2403
2404                                 } else {
2405                                         cfa->base = CFI_UNDEFINED;
2406                                         cfa->offset = 0;
2407                                 }
2408                         }
2409
2410                         else if (op->dest.reg == CFI_SP &&
2411                                  cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2412                                  cfi->vals[op->src.reg].offset == cfa->offset) {
2413
2414                                 /*
2415                                  * The same stack swizzle case 2) as above. But
2416                                  * because we can't change cfa->base, case 3)
2417                                  * will become a regular POP. Pretend we're a
2418                                  * PUSH so things don't go unbalanced.
2419                                  */
2420                                 cfi->stack_size += 8;
2421                         }
2422
2423
2424                         break;
2425
2426                 case OP_SRC_ADD:
2427                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2428
2429                                 /* add imm, %rsp */
2430                                 cfi->stack_size -= op->src.offset;
2431                                 if (cfa->base == CFI_SP)
2432                                         cfa->offset -= op->src.offset;
2433                                 break;
2434                         }
2435
2436                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2437
2438                                 /* lea disp(%rbp), %rsp */
2439                                 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2440                                 break;
2441                         }
2442
2443                         if (!cfi->drap && op->src.reg == CFI_SP &&
2444                             op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2445                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset + op->src.offset)) {
2446
2447                                 /* lea disp(%rsp), %rbp */
2448                                 cfa->base = CFI_BP;
2449                                 cfa->offset -= op->src.offset;
2450                                 cfi->bp_scratch = false;
2451                                 break;
2452                         }
2453
2454                         if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2455
2456                                 /* drap: lea disp(%rsp), %drap */
2457                                 cfi->drap_reg = op->dest.reg;
2458
2459                                 /*
2460                                  * lea disp(%rsp), %reg
2461                                  *
2462                                  * This is needed for the rare case where GCC
2463                                  * does something dumb like:
2464                                  *
2465                                  *   lea    0x8(%rsp), %rcx
2466                                  *   ...
2467                                  *   mov    %rcx, %rsp
2468                                  */
2469                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2470                                 cfi->vals[op->dest.reg].offset = \
2471                                         -cfi->stack_size + op->src.offset;
2472
2473                                 break;
2474                         }
2475
2476                         if (cfi->drap && op->dest.reg == CFI_SP &&
2477                             op->src.reg == cfi->drap_reg) {
2478
2479                                  /* drap: lea disp(%drap), %rsp */
2480                                 cfa->base = CFI_SP;
2481                                 cfa->offset = cfi->stack_size = -op->src.offset;
2482                                 cfi->drap_reg = CFI_UNDEFINED;
2483                                 cfi->drap = false;
2484                                 break;
2485                         }
2486
2487                         if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2488                                 WARN_FUNC("unsupported stack register modification",
2489                                           insn->sec, insn->offset);
2490                                 return -1;
2491                         }
2492
2493                         break;
2494
2495                 case OP_SRC_AND:
2496                         if (op->dest.reg != CFI_SP ||
2497                             (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2498                             (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2499                                 WARN_FUNC("unsupported stack pointer realignment",
2500                                           insn->sec, insn->offset);
2501                                 return -1;
2502                         }
2503
2504                         if (cfi->drap_reg != CFI_UNDEFINED) {
2505                                 /* drap: and imm, %rsp */
2506                                 cfa->base = cfi->drap_reg;
2507                                 cfa->offset = cfi->stack_size = 0;
2508                                 cfi->drap = true;
2509                         }
2510
2511                         /*
2512                          * Older versions of GCC (4.8ish) realign the stack
2513                          * without DRAP, with a frame pointer.
2514                          */
2515
2516                         break;
2517
2518                 case OP_SRC_POP:
2519                 case OP_SRC_POPF:
2520                         if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2521
2522                                 /* pop %rsp; # restore from a stack swizzle */
2523                                 cfa->base = CFI_SP;
2524                                 break;
2525                         }
2526
2527                         if (!cfi->drap && op->dest.reg == cfa->base) {
2528
2529                                 /* pop %rbp */
2530                                 cfa->base = CFI_SP;
2531                         }
2532
2533                         if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2534                             op->dest.reg == cfi->drap_reg &&
2535                             cfi->drap_offset == -cfi->stack_size) {
2536
2537                                 /* drap: pop %drap */
2538                                 cfa->base = cfi->drap_reg;
2539                                 cfa->offset = 0;
2540                                 cfi->drap_offset = -1;
2541
2542                         } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
2543
2544                                 /* pop %reg */
2545                                 restore_reg(cfi, op->dest.reg);
2546                         }
2547
2548                         cfi->stack_size -= 8;
2549                         if (cfa->base == CFI_SP)
2550                                 cfa->offset -= 8;
2551
2552                         break;
2553
2554                 case OP_SRC_REG_INDIRECT:
2555                         if (!cfi->drap && op->dest.reg == cfa->base &&
2556                             op->dest.reg == CFI_BP) {
2557
2558                                 /* mov disp(%rsp), %rbp */
2559                                 cfa->base = CFI_SP;
2560                                 cfa->offset = cfi->stack_size;
2561                         }
2562
2563                         if (cfi->drap && op->src.reg == CFI_BP &&
2564                             op->src.offset == cfi->drap_offset) {
2565
2566                                 /* drap: mov disp(%rbp), %drap */
2567                                 cfa->base = cfi->drap_reg;
2568                                 cfa->offset = 0;
2569                                 cfi->drap_offset = -1;
2570                         }
2571
2572                         if (cfi->drap && op->src.reg == CFI_BP &&
2573                             op->src.offset == regs[op->dest.reg].offset) {
2574
2575                                 /* drap: mov disp(%rbp), %reg */
2576                                 restore_reg(cfi, op->dest.reg);
2577
2578                         } else if (op->src.reg == cfa->base &&
2579                             op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
2580
2581                                 /* mov disp(%rbp), %reg */
2582                                 /* mov disp(%rsp), %reg */
2583                                 restore_reg(cfi, op->dest.reg);
2584
2585                         } else if (op->src.reg == CFI_SP &&
2586                                    op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
2587
2588                                 /* mov disp(%rsp), %reg */
2589                                 restore_reg(cfi, op->dest.reg);
2590                         }
2591
2592                         break;
2593
2594                 default:
2595                         WARN_FUNC("unknown stack-related instruction",
2596                                   insn->sec, insn->offset);
2597                         return -1;
2598                 }
2599
2600                 break;
2601
2602         case OP_DEST_PUSH:
2603         case OP_DEST_PUSHF:
2604                 cfi->stack_size += 8;
2605                 if (cfa->base == CFI_SP)
2606                         cfa->offset += 8;
2607
2608                 if (op->src.type != OP_SRC_REG)
2609                         break;
2610
2611                 if (cfi->drap) {
2612                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2613
2614                                 /* drap: push %drap */
2615                                 cfa->base = CFI_BP_INDIRECT;
2616                                 cfa->offset = -cfi->stack_size;
2617
2618                                 /* save drap so we know when to restore it */
2619                                 cfi->drap_offset = -cfi->stack_size;
2620
2621                         } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
2622
2623                                 /* drap: push %rbp */
2624                                 cfi->stack_size = 0;
2625
2626                         } else {
2627
2628                                 /* drap: push %reg */
2629                                 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
2630                         }
2631
2632                 } else {
2633
2634                         /* push %reg */
2635                         save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
2636                 }
2637
2638                 /* detect when asm code uses rbp as a scratch register */
2639                 if (!no_fp && insn->func && op->src.reg == CFI_BP &&
2640                     cfa->base != CFI_BP)
2641                         cfi->bp_scratch = true;
2642                 break;
2643
2644         case OP_DEST_REG_INDIRECT:
2645
2646                 if (cfi->drap) {
2647                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2648
2649                                 /* drap: mov %drap, disp(%rbp) */
2650                                 cfa->base = CFI_BP_INDIRECT;
2651                                 cfa->offset = op->dest.offset;
2652
2653                                 /* save drap offset so we know when to restore it */
2654                                 cfi->drap_offset = op->dest.offset;
2655                         } else {
2656
2657                                 /* drap: mov reg, disp(%rbp) */
2658                                 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
2659                         }
2660
2661                 } else if (op->dest.reg == cfa->base) {
2662
2663                         /* mov reg, disp(%rbp) */
2664                         /* mov reg, disp(%rsp) */
2665                         save_reg(cfi, op->src.reg, CFI_CFA,
2666                                  op->dest.offset - cfi->cfa.offset);
2667
2668                 } else if (op->dest.reg == CFI_SP) {
2669
2670                         /* mov reg, disp(%rsp) */
2671                         save_reg(cfi, op->src.reg, CFI_CFA,
2672                                  op->dest.offset - cfi->stack_size);
2673
2674                 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
2675
2676                         /* mov %rsp, (%reg); # setup a stack swizzle. */
2677                         cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
2678                         cfi->vals[op->dest.reg].offset = cfa->offset;
2679                 }
2680
2681                 break;
2682
2683         case OP_DEST_MEM:
2684                 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
2685                         WARN_FUNC("unknown stack-related memory operation",
2686                                   insn->sec, insn->offset);
2687                         return -1;
2688                 }
2689
2690                 /* pop mem */
2691                 cfi->stack_size -= 8;
2692                 if (cfa->base == CFI_SP)
2693                         cfa->offset -= 8;
2694
2695                 break;
2696
2697         default:
2698                 WARN_FUNC("unknown stack-related instruction",
2699                           insn->sec, insn->offset);
2700                 return -1;
2701         }
2702
2703         return 0;
2704 }
2705
2706 /*
2707  * The stack layouts of alternatives instructions can sometimes diverge when
2708  * they have stack modifications.  That's fine as long as the potential stack
2709  * layouts don't conflict at any given potential instruction boundary.
2710  *
2711  * Flatten the CFIs of the different alternative code streams (both original
2712  * and replacement) into a single shared CFI array which can be used to detect
2713  * conflicts and nicely feed a linear array of ORC entries to the unwinder.
2714  */
2715 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
2716 {
2717         struct cfi_state **alt_cfi;
2718         int group_off;
2719
2720         if (!insn->alt_group)
2721                 return 0;
2722
2723         if (!insn->cfi) {
2724                 WARN("CFI missing");
2725                 return -1;
2726         }
2727
2728         alt_cfi = insn->alt_group->cfi;
2729         group_off = insn->offset - insn->alt_group->first_insn->offset;
2730
2731         if (!alt_cfi[group_off]) {
2732                 alt_cfi[group_off] = insn->cfi;
2733         } else {
2734                 if (cficmp(alt_cfi[group_off], insn->cfi)) {
2735                         WARN_FUNC("stack layout conflict in alternatives",
2736                                   insn->sec, insn->offset);
2737                         return -1;
2738                 }
2739         }
2740
2741         return 0;
2742 }
2743
2744 static int handle_insn_ops(struct instruction *insn,
2745                            struct instruction *next_insn,
2746                            struct insn_state *state)
2747 {
2748         struct stack_op *op;
2749
2750         list_for_each_entry(op, &insn->stack_ops, list) {
2751
2752                 if (update_cfi_state(insn, next_insn, &state->cfi, op))
2753                         return 1;
2754
2755                 if (!insn->alt_group)
2756                         continue;
2757
2758                 if (op->dest.type == OP_DEST_PUSHF) {
2759                         if (!state->uaccess_stack) {
2760                                 state->uaccess_stack = 1;
2761                         } else if (state->uaccess_stack >> 31) {
2762                                 WARN_FUNC("PUSHF stack exhausted",
2763                                           insn->sec, insn->offset);
2764                                 return 1;
2765                         }
2766                         state->uaccess_stack <<= 1;
2767                         state->uaccess_stack  |= state->uaccess;
2768                 }
2769
2770                 if (op->src.type == OP_SRC_POPF) {
2771                         if (state->uaccess_stack) {
2772                                 state->uaccess = state->uaccess_stack & 1;
2773                                 state->uaccess_stack >>= 1;
2774                                 if (state->uaccess_stack == 1)
2775                                         state->uaccess_stack = 0;
2776                         }
2777                 }
2778         }
2779
2780         return 0;
2781 }
2782
2783 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
2784 {
2785         struct cfi_state *cfi1 = insn->cfi;
2786         int i;
2787
2788         if (!cfi1) {
2789                 WARN("CFI missing");
2790                 return false;
2791         }
2792
2793         if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
2794
2795                 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
2796                           insn->sec, insn->offset,
2797                           cfi1->cfa.base, cfi1->cfa.offset,
2798                           cfi2->cfa.base, cfi2->cfa.offset);
2799
2800         } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
2801                 for (i = 0; i < CFI_NUM_REGS; i++) {
2802                         if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
2803                                     sizeof(struct cfi_reg)))
2804                                 continue;
2805
2806                         WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
2807                                   insn->sec, insn->offset,
2808                                   i, cfi1->regs[i].base, cfi1->regs[i].offset,
2809                                   i, cfi2->regs[i].base, cfi2->regs[i].offset);
2810                         break;
2811                 }
2812
2813         } else if (cfi1->type != cfi2->type) {
2814
2815                 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
2816                           insn->sec, insn->offset, cfi1->type, cfi2->type);
2817
2818         } else if (cfi1->drap != cfi2->drap ||
2819                    (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
2820                    (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
2821
2822                 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
2823                           insn->sec, insn->offset,
2824                           cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
2825                           cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
2826
2827         } else
2828                 return true;
2829
2830         return false;
2831 }
2832
2833 static inline bool func_uaccess_safe(struct symbol *func)
2834 {
2835         if (func)
2836                 return func->uaccess_safe;
2837
2838         return false;
2839 }
2840
2841 static inline const char *call_dest_name(struct instruction *insn)
2842 {
2843         if (insn->call_dest)
2844                 return insn->call_dest->name;
2845
2846         return "{dynamic}";
2847 }
2848
2849 static inline bool noinstr_call_dest(struct symbol *func)
2850 {
2851         /*
2852          * We can't deal with indirect function calls at present;
2853          * assume they're instrumented.
2854          */
2855         if (!func)
2856                 return false;
2857
2858         /*
2859          * If the symbol is from a noinstr section; we good.
2860          */
2861         if (func->sec->noinstr)
2862                 return true;
2863
2864         /*
2865          * The __ubsan_handle_*() calls are like WARN(), they only happen when
2866          * something 'BAD' happened. At the risk of taking the machine down,
2867          * let them proceed to get the message out.
2868          */
2869         if (!strncmp(func->name, "__ubsan_handle_", 15))
2870                 return true;
2871
2872         return false;
2873 }
2874
2875 static int validate_call(struct instruction *insn, struct insn_state *state)
2876 {
2877         if (state->noinstr && state->instr <= 0 &&
2878             !noinstr_call_dest(insn->call_dest)) {
2879                 WARN_FUNC("call to %s() leaves .noinstr.text section",
2880                                 insn->sec, insn->offset, call_dest_name(insn));
2881                 return 1;
2882         }
2883
2884         if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
2885                 WARN_FUNC("call to %s() with UACCESS enabled",
2886                                 insn->sec, insn->offset, call_dest_name(insn));
2887                 return 1;
2888         }
2889
2890         if (state->df) {
2891                 WARN_FUNC("call to %s() with DF set",
2892                                 insn->sec, insn->offset, call_dest_name(insn));
2893                 return 1;
2894         }
2895
2896         return 0;
2897 }
2898
2899 static int validate_sibling_call(struct instruction *insn, struct insn_state *state)
2900 {
2901         if (has_modified_stack_frame(insn, state)) {
2902                 WARN_FUNC("sibling call from callable instruction with modified stack frame",
2903                                 insn->sec, insn->offset);
2904                 return 1;
2905         }
2906
2907         return validate_call(insn, state);
2908 }
2909
2910 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
2911 {
2912         if (state->noinstr && state->instr > 0) {
2913                 WARN_FUNC("return with instrumentation enabled",
2914                           insn->sec, insn->offset);
2915                 return 1;
2916         }
2917
2918         if (state->uaccess && !func_uaccess_safe(func)) {
2919                 WARN_FUNC("return with UACCESS enabled",
2920                           insn->sec, insn->offset);
2921                 return 1;
2922         }
2923
2924         if (!state->uaccess && func_uaccess_safe(func)) {
2925                 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
2926                           insn->sec, insn->offset);
2927                 return 1;
2928         }
2929
2930         if (state->df) {
2931                 WARN_FUNC("return with DF set",
2932                           insn->sec, insn->offset);
2933                 return 1;
2934         }
2935
2936         if (func && has_modified_stack_frame(insn, state)) {
2937                 WARN_FUNC("return with modified stack frame",
2938                           insn->sec, insn->offset);
2939                 return 1;
2940         }
2941
2942         if (state->cfi.bp_scratch) {
2943                 WARN_FUNC("BP used as a scratch register",
2944                           insn->sec, insn->offset);
2945                 return 1;
2946         }
2947
2948         return 0;
2949 }
2950
2951 static struct instruction *next_insn_to_validate(struct objtool_file *file,
2952                                                  struct instruction *insn)
2953 {
2954         struct alt_group *alt_group = insn->alt_group;
2955
2956         /*
2957          * Simulate the fact that alternatives are patched in-place.  When the
2958          * end of a replacement alt_group is reached, redirect objtool flow to
2959          * the end of the original alt_group.
2960          */
2961         if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
2962                 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
2963
2964         return next_insn_same_sec(file, insn);
2965 }
2966
2967 /*
2968  * Follow the branch starting at the given instruction, and recursively follow
2969  * any other branches (jumps).  Meanwhile, track the frame pointer state at
2970  * each instruction and validate all the rules described in
2971  * tools/objtool/Documentation/stack-validation.txt.
2972  */
2973 static int validate_branch(struct objtool_file *file, struct symbol *func,
2974                            struct instruction *insn, struct insn_state state)
2975 {
2976         struct alternative *alt;
2977         struct instruction *next_insn, *prev_insn = NULL;
2978         struct section *sec;
2979         u8 visited;
2980         int ret;
2981
2982         sec = insn->sec;
2983
2984         while (1) {
2985                 next_insn = next_insn_to_validate(file, insn);
2986
2987                 if (file->c_file && func && insn->func && func != insn->func->pfunc) {
2988                         WARN("%s() falls through to next function %s()",
2989                              func->name, insn->func->name);
2990                         return 1;
2991                 }
2992
2993                 if (func && insn->ignore) {
2994                         WARN_FUNC("BUG: why am I validating an ignored function?",
2995                                   sec, insn->offset);
2996                         return 1;
2997                 }
2998
2999                 visited = 1 << state.uaccess;
3000                 if (insn->visited) {
3001                         if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3002                                 return 1;
3003
3004                         if (insn->visited & visited)
3005                                 return 0;
3006                 } else {
3007                         nr_insns_visited++;
3008                 }
3009
3010                 if (state.noinstr)
3011                         state.instr += insn->instr;
3012
3013                 if (insn->hint) {
3014                         state.cfi = *insn->cfi;
3015                 } else {
3016                         /* XXX track if we actually changed state.cfi */
3017
3018                         if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3019                                 insn->cfi = prev_insn->cfi;
3020                                 nr_cfi_reused++;
3021                         } else {
3022                                 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3023                         }
3024                 }
3025
3026                 insn->visited |= visited;
3027
3028                 if (propagate_alt_cfi(file, insn))
3029                         return 1;
3030
3031                 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3032                         bool skip_orig = false;
3033
3034                         list_for_each_entry(alt, &insn->alts, list) {
3035                                 if (alt->skip_orig)
3036                                         skip_orig = true;
3037
3038                                 ret = validate_branch(file, func, alt->insn, state);
3039                                 if (ret) {
3040                                         if (backtrace)
3041                                                 BT_FUNC("(alt)", insn);
3042                                         return ret;
3043                                 }
3044                         }
3045
3046                         if (skip_orig)
3047                                 return 0;
3048                 }
3049
3050                 if (handle_insn_ops(insn, next_insn, &state))
3051                         return 1;
3052
3053                 switch (insn->type) {
3054
3055                 case INSN_RETURN:
3056                         if (sls && !insn->retpoline_safe &&
3057                             next_insn && next_insn->type != INSN_TRAP) {
3058                                 WARN_FUNC("missing int3 after ret",
3059                                           insn->sec, insn->offset);
3060                         }
3061                         return validate_return(func, insn, &state);
3062
3063                 case INSN_CALL:
3064                 case INSN_CALL_DYNAMIC:
3065                         ret = validate_call(insn, &state);
3066                         if (ret)
3067                                 return ret;
3068
3069                         if (!no_fp && func && !is_fentry_call(insn) &&
3070                             !has_valid_stack_frame(&state)) {
3071                                 WARN_FUNC("call without frame pointer save/setup",
3072                                           sec, insn->offset);
3073                                 return 1;
3074                         }
3075
3076                         if (dead_end_function(file, insn->call_dest))
3077                                 return 0;
3078
3079                         break;
3080
3081                 case INSN_JUMP_CONDITIONAL:
3082                 case INSN_JUMP_UNCONDITIONAL:
3083                         if (is_sibling_call(insn)) {
3084                                 ret = validate_sibling_call(insn, &state);
3085                                 if (ret)
3086                                         return ret;
3087
3088                         } else if (insn->jump_dest) {
3089                                 ret = validate_branch(file, func,
3090                                                       insn->jump_dest, state);
3091                                 if (ret) {
3092                                         if (backtrace)
3093                                                 BT_FUNC("(branch)", insn);
3094                                         return ret;
3095                                 }
3096                         }
3097
3098                         if (insn->type == INSN_JUMP_UNCONDITIONAL)
3099                                 return 0;
3100
3101                         break;
3102
3103                 case INSN_JUMP_DYNAMIC:
3104                         if (sls && !insn->retpoline_safe &&
3105                             next_insn && next_insn->type != INSN_TRAP) {
3106                                 WARN_FUNC("missing int3 after indirect jump",
3107                                           insn->sec, insn->offset);
3108                         }
3109
3110                         /* fallthrough */
3111                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3112                         if (is_sibling_call(insn)) {
3113                                 ret = validate_sibling_call(insn, &state);
3114                                 if (ret)
3115                                         return ret;
3116                         }
3117
3118                         if (insn->type == INSN_JUMP_DYNAMIC)
3119                                 return 0;
3120
3121                         break;
3122
3123                 case INSN_CONTEXT_SWITCH:
3124                         if (func && (!next_insn || !next_insn->hint)) {
3125                                 WARN_FUNC("unsupported instruction in callable function",
3126                                           sec, insn->offset);
3127                                 return 1;
3128                         }
3129                         return 0;
3130
3131                 case INSN_STAC:
3132                         if (state.uaccess) {
3133                                 WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
3134                                 return 1;
3135                         }
3136
3137                         state.uaccess = true;
3138                         break;
3139
3140                 case INSN_CLAC:
3141                         if (!state.uaccess && func) {
3142                                 WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
3143                                 return 1;
3144                         }
3145
3146                         if (func_uaccess_safe(func) && !state.uaccess_stack) {
3147                                 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
3148                                 return 1;
3149                         }
3150
3151                         state.uaccess = false;
3152                         break;
3153
3154                 case INSN_STD:
3155                         if (state.df) {
3156                                 WARN_FUNC("recursive STD", sec, insn->offset);
3157                                 return 1;
3158                         }
3159
3160                         state.df = true;
3161                         break;
3162
3163                 case INSN_CLD:
3164                         if (!state.df && func) {
3165                                 WARN_FUNC("redundant CLD", sec, insn->offset);
3166                                 return 1;
3167                         }
3168
3169                         state.df = false;
3170                         break;
3171
3172                 default:
3173                         break;
3174                 }
3175
3176                 if (insn->dead_end)
3177                         return 0;
3178
3179                 if (!next_insn) {
3180                         if (state.cfi.cfa.base == CFI_UNDEFINED)
3181                                 return 0;
3182                         WARN("%s: unexpected end of section", sec->name);
3183                         return 1;
3184                 }
3185
3186                 prev_insn = insn;
3187                 insn = next_insn;
3188         }
3189
3190         return 0;
3191 }
3192
3193 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3194 {
3195         struct instruction *insn;
3196         struct insn_state state;
3197         int ret, warnings = 0;
3198
3199         if (!file->hints)
3200                 return 0;
3201
3202         init_insn_state(&state, sec);
3203
3204         if (sec) {
3205                 insn = find_insn(file, sec, 0);
3206                 if (!insn)
3207                         return 0;
3208         } else {
3209                 insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3210         }
3211
3212         while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
3213                 if (insn->hint && !insn->visited) {
3214                         ret = validate_branch(file, insn->func, insn, state);
3215                         if (ret && backtrace)
3216                                 BT_FUNC("<=== (hint)", insn);
3217                         warnings += ret;
3218                 }
3219
3220                 insn = list_next_entry(insn, list);
3221         }
3222
3223         return warnings;
3224 }
3225
3226 static int validate_retpoline(struct objtool_file *file)
3227 {
3228         struct instruction *insn;
3229         int warnings = 0;
3230
3231         for_each_insn(file, insn) {
3232                 if (insn->type != INSN_JUMP_DYNAMIC &&
3233                     insn->type != INSN_CALL_DYNAMIC &&
3234                     insn->type != INSN_RETURN)
3235                         continue;
3236
3237                 if (insn->retpoline_safe)
3238                         continue;
3239
3240                 /*
3241                  * .init.text code is ran before userspace and thus doesn't
3242                  * strictly need retpolines, except for modules which are
3243                  * loaded late, they very much do need retpoline in their
3244                  * .init.text
3245                  */
3246                 if (!strcmp(insn->sec->name, ".init.text") && !module)
3247                         continue;
3248
3249                 if (insn->type == INSN_RETURN) {
3250                         WARN_FUNC("'naked' return found in RETPOLINE build",
3251                                   insn->sec, insn->offset);
3252                 } else {
3253                         WARN_FUNC("indirect %s found in RETPOLINE build",
3254                                   insn->sec, insn->offset,
3255                                   insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3256                 }
3257
3258                 warnings++;
3259         }
3260
3261         return warnings;
3262 }
3263
3264 static bool is_kasan_insn(struct instruction *insn)
3265 {
3266         return (insn->type == INSN_CALL &&
3267                 !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3268 }
3269
3270 static bool is_ubsan_insn(struct instruction *insn)
3271 {
3272         return (insn->type == INSN_CALL &&
3273                 !strcmp(insn->call_dest->name,
3274                         "__ubsan_handle_builtin_unreachable"));
3275 }
3276
3277 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
3278 {
3279         int i;
3280         struct instruction *prev_insn;
3281
3282         if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
3283                 return true;
3284
3285         /*
3286          * Ignore any unused exceptions.  This can happen when a whitelisted
3287          * function has an exception table entry.
3288          *
3289          * Also ignore alternative replacement instructions.  This can happen
3290          * when a whitelisted function uses one of the ALTERNATIVE macros.
3291          */
3292         if (!strcmp(insn->sec->name, ".fixup") ||
3293             !strcmp(insn->sec->name, ".altinstr_replacement") ||
3294             !strcmp(insn->sec->name, ".altinstr_aux"))
3295                 return true;
3296
3297         if (!insn->func)
3298                 return false;
3299
3300         /*
3301          * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
3302          * __builtin_unreachable().  The BUG() macro has an unreachable() after
3303          * the UD2, which causes GCC's undefined trap logic to emit another UD2
3304          * (or occasionally a JMP to UD2).
3305          *
3306          * It may also insert a UD2 after calling a __noreturn function.
3307          */
3308         prev_insn = list_prev_entry(insn, list);
3309         if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
3310             (insn->type == INSN_BUG ||
3311              (insn->type == INSN_JUMP_UNCONDITIONAL &&
3312               insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
3313                 return true;
3314
3315         /*
3316          * Check if this (or a subsequent) instruction is related to
3317          * CONFIG_UBSAN or CONFIG_KASAN.
3318          *
3319          * End the search at 5 instructions to avoid going into the weeds.
3320          */
3321         for (i = 0; i < 5; i++) {
3322
3323                 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
3324                         return true;
3325
3326                 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3327                         if (insn->jump_dest &&
3328                             insn->jump_dest->func == insn->func) {
3329                                 insn = insn->jump_dest;
3330                                 continue;
3331                         }
3332
3333                         break;
3334                 }
3335
3336                 if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
3337                         break;
3338
3339                 insn = list_next_entry(insn, list);
3340         }
3341
3342         return false;
3343 }
3344
3345 static int validate_symbol(struct objtool_file *file, struct section *sec,
3346                            struct symbol *sym, struct insn_state *state)
3347 {
3348         struct instruction *insn;
3349         int ret;
3350
3351         if (!sym->len) {
3352                 WARN("%s() is missing an ELF size annotation", sym->name);
3353                 return 1;
3354         }
3355
3356         if (sym->pfunc != sym || sym->alias != sym)
3357                 return 0;
3358
3359         insn = find_insn(file, sec, sym->offset);
3360         if (!insn || insn->ignore || insn->visited)
3361                 return 0;
3362
3363         state->uaccess = sym->uaccess_safe;
3364
3365         ret = validate_branch(file, insn->func, insn, *state);
3366         if (ret && backtrace)
3367                 BT_FUNC("<=== (sym)", insn);
3368         return ret;
3369 }
3370
3371 static int validate_section(struct objtool_file *file, struct section *sec)
3372 {
3373         struct insn_state state;
3374         struct symbol *func;
3375         int warnings = 0;
3376
3377         list_for_each_entry(func, &sec->symbol_list, list) {
3378                 if (func->type != STT_FUNC)
3379                         continue;
3380
3381                 init_insn_state(&state, sec);
3382                 set_func_state(&state.cfi);
3383
3384                 warnings += validate_symbol(file, sec, func, &state);
3385         }
3386
3387         return warnings;
3388 }
3389
3390 static int validate_vmlinux_functions(struct objtool_file *file)
3391 {
3392         struct section *sec;
3393         int warnings = 0;
3394
3395         sec = find_section_by_name(file->elf, ".noinstr.text");
3396         if (sec) {
3397                 warnings += validate_section(file, sec);
3398                 warnings += validate_unwind_hints(file, sec);
3399         }
3400
3401         sec = find_section_by_name(file->elf, ".entry.text");
3402         if (sec) {
3403                 warnings += validate_section(file, sec);
3404                 warnings += validate_unwind_hints(file, sec);
3405         }
3406
3407         return warnings;
3408 }
3409
3410 static int validate_functions(struct objtool_file *file)
3411 {
3412         struct section *sec;
3413         int warnings = 0;
3414
3415         for_each_sec(file, sec) {
3416                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
3417                         continue;
3418
3419                 warnings += validate_section(file, sec);
3420         }
3421
3422         return warnings;
3423 }
3424
3425 static int validate_reachable_instructions(struct objtool_file *file)
3426 {
3427         struct instruction *insn;
3428
3429         if (file->ignore_unreachables)
3430                 return 0;
3431
3432         for_each_insn(file, insn) {
3433                 if (insn->visited || ignore_unreachable_insn(file, insn))
3434                         continue;
3435
3436                 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
3437                 return 1;
3438         }
3439
3440         return 0;
3441 }
3442
3443 int check(struct objtool_file *file)
3444 {
3445         int ret, warnings = 0;
3446
3447         arch_initial_func_cfi_state(&initial_func_cfi);
3448         init_cfi_state(&init_cfi);
3449         init_cfi_state(&func_cfi);
3450         set_func_state(&func_cfi);
3451
3452         if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
3453                 goto out;
3454
3455         cfi_hash_add(&init_cfi);
3456         cfi_hash_add(&func_cfi);
3457
3458         ret = decode_sections(file);
3459         if (ret < 0)
3460                 goto out;
3461
3462         warnings += ret;
3463
3464         if (list_empty(&file->insn_list))
3465                 goto out;
3466
3467         if (vmlinux && !validate_dup) {
3468                 ret = validate_vmlinux_functions(file);
3469                 if (ret < 0)
3470                         goto out;
3471
3472                 warnings += ret;
3473                 goto out;
3474         }
3475
3476         if (retpoline) {
3477                 ret = validate_retpoline(file);
3478                 if (ret < 0)
3479                         return ret;
3480                 warnings += ret;
3481         }
3482
3483         ret = validate_functions(file);
3484         if (ret < 0)
3485                 goto out;
3486         warnings += ret;
3487
3488         ret = validate_unwind_hints(file, NULL);
3489         if (ret < 0)
3490                 goto out;
3491         warnings += ret;
3492
3493         if (!warnings) {
3494                 ret = validate_reachable_instructions(file);
3495                 if (ret < 0)
3496                         goto out;
3497                 warnings += ret;
3498         }
3499
3500         ret = create_static_call_sections(file);
3501         if (ret < 0)
3502                 goto out;
3503         warnings += ret;
3504
3505         if (retpoline) {
3506                 ret = create_retpoline_sites_sections(file);
3507                 if (ret < 0)
3508                         goto out;
3509                 warnings += ret;
3510
3511                 ret = create_return_sites_sections(file);
3512                 if (ret < 0)
3513                         goto out;
3514                 warnings += ret;
3515         }
3516
3517         if (mcount) {
3518                 ret = create_mcount_loc_sections(file);
3519                 if (ret < 0)
3520                         goto out;
3521                 warnings += ret;
3522         }
3523
3524         if (stats) {
3525                 printf("nr_insns_visited: %ld\n", nr_insns_visited);
3526                 printf("nr_cfi: %ld\n", nr_cfi);
3527                 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
3528                 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
3529         }
3530
3531 out:
3532         /*
3533          *  For now, don't fail the kernel build on fatal warnings.  These
3534          *  errors are still fairly common due to the growing matrix of
3535          *  supported toolchains and their recent pace of change.
3536          */
3537         return 0;
3538 }