scripts/sorttable.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * sorttable.h
   4  *
   5  * Added ORC unwind tables sort support and other updates:
   6  * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by:
   7  * Shile Zhang <shile.zhang@linux.alibaba.com>
   8  *
   9  * Copyright 2011 - 2012 Cavium, Inc.
  10  *
  11  * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by:
  12  * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
  13  *
  14  * Some of this code was taken out of recordmcount.h written by:
  15  *
  16  * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved.
  17  * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
  18  */
  19
  20 #undef extable_ent_size
  21 #undef compare_extable
  22 #undef do_sort
  23 #undef Elf_Addr
  24 #undef Elf_Ehdr
  25 #undef Elf_Shdr
  26 #undef Elf_Rel
  27 #undef Elf_Rela
  28 #undef Elf_Sym
  29 #undef ELF_R_SYM
  30 #undef Elf_r_sym
  31 #undef ELF_R_INFO
  32 #undef Elf_r_info
  33 #undef ELF_ST_BIND
  34 #undef ELF_ST_TYPE
  35 #undef fn_ELF_R_SYM
  36 #undef fn_ELF_R_INFO
  37 #undef uint_t
  38 #undef _r
  39 #undef _w
  40
  41 #ifdef SORTTABLE_64
  42 # define extable_ent_size       16
  43 # define compare_extable        compare_extable_64
  44 # define do_sort                do_sort_64
  45 # define Elf_Addr               Elf64_Addr
  46 # define Elf_Ehdr               Elf64_Ehdr
  47 # define Elf_Shdr               Elf64_Shdr
  48 # define Elf_Rel                Elf64_Rel
  49 # define Elf_Rela               Elf64_Rela
  50 # define Elf_Sym                Elf64_Sym
  51 # define ELF_R_SYM              ELF64_R_SYM
  52 # define Elf_r_sym              Elf64_r_sym
  53 # define ELF_R_INFO             ELF64_R_INFO
  54 # define Elf_r_info             Elf64_r_info
  55 # define ELF_ST_BIND            ELF64_ST_BIND
  56 # define ELF_ST_TYPE            ELF64_ST_TYPE
  57 # define fn_ELF_R_SYM           fn_ELF64_R_SYM
  58 # define fn_ELF_R_INFO          fn_ELF64_R_INFO
  59 # define uint_t                 uint64_t
  60 # define _r                     r8
  61 # define _w                     w8
  62 #else
  63 # define extable_ent_size       8
  64 # define compare_extable        compare_extable_32
  65 # define do_sort                do_sort_32
  66 # define Elf_Addr               Elf32_Addr
  67 # define Elf_Ehdr               Elf32_Ehdr
  68 # define Elf_Shdr               Elf32_Shdr
  69 # define Elf_Rel                Elf32_Rel
  70 # define Elf_Rela               Elf32_Rela
  71 # define Elf_Sym                Elf32_Sym
  72 # define ELF_R_SYM              ELF32_R_SYM
  73 # define Elf_r_sym              Elf32_r_sym
  74 # define ELF_R_INFO             ELF32_R_INFO
  75 # define Elf_r_info             Elf32_r_info
  76 # define ELF_ST_BIND            ELF32_ST_BIND
  77 # define ELF_ST_TYPE            ELF32_ST_TYPE
  78 # define fn_ELF_R_SYM           fn_ELF32_R_SYM
  79 # define fn_ELF_R_INFO          fn_ELF32_R_INFO
  80 # define uint_t                 uint32_t
  81 # define _r                     r
  82 # define _w                     w
  83 #endif
  84
  85 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
  86 /* ORC unwinder only support X86_64 */
  87 #include <errno.h>
  88 #include <pthread.h>
  89 #include <asm/orc_types.h>
  90
  91 #define ERRSTR_MAXSZ    256
  92
  93 char g_err[ERRSTR_MAXSZ];
  94 int *g_orc_ip_table;
  95 struct orc_entry *g_orc_table;
  96
  97 pthread_t orc_sort_thread;
  98
  99 static inline unsigned long orc_ip(const int *ip)
 100 {
 101         return (unsigned long)ip + *ip;
 102 }
 103
 104 static int orc_sort_cmp(const void *_a, const void *_b)
 105 {
 106         struct orc_entry *orc_a;
 107         const int *a = g_orc_ip_table + *(int *)_a;
 108         const int *b = g_orc_ip_table + *(int *)_b;
 109         unsigned long a_val = orc_ip(a);
 110         unsigned long b_val = orc_ip(b);
 111
 112         if (a_val > b_val)
 113                 return 1;
 114         if (a_val < b_val)
 115                 return -1;
 116
 117         /*
 118          * The "weak" section terminator entries need to always be on the left
 119          * to ensure the lookup code skips them in favor of real entries.
 120          * These terminator entries exist to handle any gaps created by
 121          * whitelisted .o files which didn't get objtool generation.
 122          */
 123         orc_a = g_orc_table + (a - g_orc_ip_table);
 124         return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1;
 125 }
 126
 127 static void *sort_orctable(void *arg)
 128 {
 129         int i;
 130         int *idxs = NULL;
 131         int *tmp_orc_ip_table = NULL;
 132         struct orc_entry *tmp_orc_table = NULL;
 133         unsigned int *orc_ip_size = (unsigned int *)arg;
 134         unsigned int num_entries = *orc_ip_size / sizeof(int);
 135         unsigned int orc_size = num_entries * sizeof(struct orc_entry);
 136
 137         idxs = (int *)malloc(*orc_ip_size);
 138         if (!idxs) {
 139                 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s",
 140                          strerror(errno));
 141                 pthread_exit(g_err);
 142         }
 143
 144         tmp_orc_ip_table = (int *)malloc(*orc_ip_size);
 145         if (!tmp_orc_ip_table) {
 146                 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s",
 147                          strerror(errno));
 148                 pthread_exit(g_err);
 149         }
 150
 151         tmp_orc_table = (struct orc_entry *)malloc(orc_size);
 152         if (!tmp_orc_table) {
 153                 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s",
 154                          strerror(errno));
 155                 pthread_exit(g_err);
 156         }
 157
 158         /* initialize indices array, convert ip_table to absolute address */
 159         for (i = 0; i < num_entries; i++) {
 160                 idxs[i] = i;
 161                 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int);
 162         }
 163         memcpy(tmp_orc_table, g_orc_table, orc_size);
 164
 165         qsort(idxs, num_entries, sizeof(int), orc_sort_cmp);
 166
 167         for (i = 0; i < num_entries; i++) {
 168                 if (idxs[i] == i)
 169                         continue;
 170
 171                 /* convert back to relative address */
 172                 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int);
 173                 g_orc_table[i] = tmp_orc_table[idxs[i]];
 174         }
 175
 176         free(idxs);
 177         free(tmp_orc_ip_table);
 178         free(tmp_orc_table);
 179         pthread_exit(NULL);
 180 }
 181 #endif
 182
 183 static int compare_extable(const void *a, const void *b)
 184 {
 185         Elf_Addr av = _r(a);
 186         Elf_Addr bv = _r(b);
 187
 188         if (av < bv)
 189                 return -1;
 190         if (av > bv)
 191                 return 1;
 192         return 0;
 193 }
 194
 195 static int do_sort(Elf_Ehdr *ehdr,
 196                    char const *const fname,
 197                    table_sort_t custom_sort)
 198 {
 199         int rc = -1;
 200         Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff));
 201         Elf_Shdr *strtab_sec = NULL;
 202         Elf_Shdr *symtab_sec = NULL;
 203         Elf_Shdr *extab_sec = NULL;
 204         Elf_Sym *sym;
 205         const Elf_Sym *symtab;
 206         Elf32_Word *symtab_shndx = NULL;
 207         Elf_Sym *sort_needed_sym = NULL;
 208         Elf_Shdr *sort_needed_sec;
 209         Elf_Rel *relocs = NULL;
 210         int relocs_size = 0;
 211         uint32_t *sort_needed_loc;
 212         const char *secstrings;
 213         const char *strtab;
 214         char *extab_image;
 215         int extab_index = 0;
 216         int i;
 217         int idx;
 218         unsigned int shnum;
 219         unsigned int shstrndx;
 220 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
 221         unsigned int orc_ip_size = 0;
 222         unsigned int orc_size = 0;
 223         unsigned int orc_num_entries = 0;
 224 #endif
 225
 226         shstrndx = r2(&ehdr->e_shstrndx);
 227         if (shstrndx == SHN_XINDEX)
 228                 shstrndx = r(&shdr[0].sh_link);
 229         secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset);
 230
 231         shnum = r2(&ehdr->e_shnum);
 232         if (shnum == SHN_UNDEF)
 233                 shnum = _r(&shdr[0].sh_size);
 234
 235         for (i = 0, s = shdr; s < shdr + shnum; i++, s++) {
 236                 idx = r(&s->sh_name);
 237                 if (!strcmp(secstrings + idx, "__ex_table")) {
 238                         extab_sec = s;
 239                         extab_index = i;
 240                 }
 241                 if (!strcmp(secstrings + idx, ".symtab"))
 242                         symtab_sec = s;
 243                 if (!strcmp(secstrings + idx, ".strtab"))
 244                         strtab_sec = s;
 245
 246                 if ((r(&s->sh_type) == SHT_REL ||
 247                      r(&s->sh_type) == SHT_RELA) &&
 248                     r(&s->sh_info) == extab_index) {
 249                         relocs = (void *)ehdr + _r(&s->sh_offset);
 250                         relocs_size = _r(&s->sh_size);
 251                 }
 252                 if (r(&s->sh_type) == SHT_SYMTAB_SHNDX)
 253                         symtab_shndx = (Elf32_Word *)((const char *)ehdr +
 254                                                       _r(&s->sh_offset));
 255
 256 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
 257                 /* locate the ORC unwind tables */
 258                 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) {
 259                         orc_ip_size = s->sh_size;
 260                         g_orc_ip_table = (int *)((void *)ehdr +
 261                                                    s->sh_offset);
 262                 }
 263                 if (!strcmp(secstrings + idx, ".orc_unwind")) {
 264                         orc_size = s->sh_size;
 265                         g_orc_table = (struct orc_entry *)((void *)ehdr +
 266                                                              s->sh_offset);
 267                 }
 268 #endif
 269         } /* for loop */
 270
 271 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
 272         if (!g_orc_ip_table || !g_orc_table) {
 273                 fprintf(stderr,
 274                         "incomplete ORC unwind tables in file: %s\n", fname);
 275                 goto out;
 276         }
 277
 278         orc_num_entries = orc_ip_size / sizeof(int);
 279         if (orc_ip_size % sizeof(int) != 0 ||
 280             orc_size % sizeof(struct orc_entry) != 0 ||
 281             orc_num_entries != orc_size / sizeof(struct orc_entry)) {
 282                 fprintf(stderr,
 283                         "inconsistent ORC unwind table entries in file: %s\n",
 284                         fname);
 285                 goto out;
 286         }
 287
 288         /* create thread to sort ORC unwind tables concurrently */
 289         if (pthread_create(&orc_sort_thread, NULL,
 290                            sort_orctable, &orc_ip_size)) {
 291                 fprintf(stderr,
 292                         "pthread_create orc_sort_thread failed '%s': %s\n",
 293                         strerror(errno), fname);
 294                 goto out;
 295         }
 296 #endif
 297         if (!extab_sec) {
 298                 fprintf(stderr, "no __ex_table in file: %s\n", fname);
 299                 goto out;
 300         }
 301
 302         if (!symtab_sec) {
 303                 fprintf(stderr, "no .symtab in file: %s\n", fname);
 304                 goto out;
 305         }
 306
 307         if (!strtab_sec) {
 308                 fprintf(stderr, "no .strtab in file: %s\n", fname);
 309                 goto out;
 310         }
 311
 312         extab_image = (void *)ehdr + _r(&extab_sec->sh_offset);
 313         strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset);
 314         symtab = (const Elf_Sym *)((const char *)ehdr +
 315                                                   _r(&symtab_sec->sh_offset));
 316
 317         if (custom_sort) {
 318                 custom_sort(extab_image, _r(&extab_sec->sh_size));
 319         } else {
 320                 int num_entries = _r(&extab_sec->sh_size) / extable_ent_size;
 321                 qsort(extab_image, num_entries,
 322                       extable_ent_size, compare_extable);
 323         }
 324
 325         /* If there were relocations, we no longer need them. */
 326         if (relocs)
 327                 memset(relocs, 0, relocs_size);
 328
 329         /* find the flag main_extable_sort_needed */
 330         for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset);
 331              sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym);
 332              sym++) {
 333                 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT)
 334                         continue;
 335                 if (!strcmp(strtab + r(&sym->st_name),
 336                             "main_extable_sort_needed")) {
 337                         sort_needed_sym = sym;
 338                         break;
 339                 }
 340         }
 341
 342         if (!sort_needed_sym) {
 343                 fprintf(stderr,
 344                         "no main_extable_sort_needed symbol in file: %s\n",
 345                         fname);
 346                 goto out;
 347         }
 348
 349         sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx),
 350                                              sort_needed_sym - symtab,
 351                                              symtab_shndx)];
 352         sort_needed_loc = (void *)ehdr +
 353                 _r(&sort_needed_sec->sh_offset) +
 354                 _r(&sort_needed_sym->st_value) -
 355                 _r(&sort_needed_sec->sh_addr);
 356
 357         /* extable has been sorted, clear the flag */
 358         w(0, sort_needed_loc);
 359         rc = 0;
 360
 361 out:
 362 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED)
 363         if (orc_sort_thread) {
 364                 void *retval = NULL;
 365                 /* wait for ORC tables sort done */
 366                 rc = pthread_join(orc_sort_thread, &retval);
 367                 if (rc)
 368                         fprintf(stderr,
 369                                 "pthread_join failed '%s': %s\n",
 370                                 strerror(errno), fname);
 371                 else if (retval) {
 372                         rc = -1;
 373                         fprintf(stderr,
 374                                 "failed to sort ORC tables '%s': %s\n",
 375                                 (char *)retval, fname);
 376                 }
 377         }
 378 #endif
 379         return rc;
 380 }