arch/x86/kernel/callthunks.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 #define pr_fmt(fmt) "callthunks: " fmt
   4
   5 #include <linux/debugfs.h>
   6 #include <linux/kallsyms.h>
   7 #include <linux/memory.h>
   8 #include <linux/moduleloader.h>
   9 #include <linux/static_call.h>
  10
  11 #include <asm/alternative.h>
  12 #include <asm/asm-offsets.h>
  13 #include <asm/cpu.h>
  14 #include <asm/ftrace.h>
  15 #include <asm/insn.h>
  16 #include <asm/kexec.h>
  17 #include <asm/nospec-branch.h>
  18 #include <asm/paravirt.h>
  19 #include <asm/sections.h>
  20 #include <asm/switch_to.h>
  21 #include <asm/sync_core.h>
  22 #include <asm/text-patching.h>
  23 #include <asm/xen/hypercall.h>
  24
  25 static int __initdata_or_module debug_callthunks;
  26
  27 #define prdbg(fmt, args...)                                     \
  28 do {                                                            \
  29         if (debug_callthunks)                                   \
  30                 printk(KERN_DEBUG pr_fmt(fmt), ##args);         \
  31 } while(0)
  32
  33 static int __init debug_thunks(char *str)
  34 {
  35         debug_callthunks = 1;
  36         return 1;
  37 }
  38 __setup("debug-callthunks", debug_thunks);
  39
  40 #ifdef CONFIG_CALL_THUNKS_DEBUG
  41 DEFINE_PER_CPU(u64, __x86_call_count);
  42 DEFINE_PER_CPU(u64, __x86_ret_count);
  43 DEFINE_PER_CPU(u64, __x86_stuffs_count);
  44 DEFINE_PER_CPU(u64, __x86_ctxsw_count);
  45 EXPORT_SYMBOL_GPL(__x86_ctxsw_count);
  46 EXPORT_SYMBOL_GPL(__x86_call_count);
  47 #endif
  48
  49 extern s32 __call_sites[], __call_sites_end[];
  50
  51 struct thunk_desc {
  52         void            *template;
  53         unsigned int    template_size;
  54 };
  55
  56 struct core_text {
  57         unsigned long   base;
  58         unsigned long   end;
  59         const char      *name;
  60 };
  61
  62 static bool thunks_initialized __ro_after_init;
  63
  64 static const struct core_text builtin_coretext = {
  65         .base = (unsigned long)_text,
  66         .end  = (unsigned long)_etext,
  67         .name = "builtin",
  68 };
  69
  70 asm (
  71         ".pushsection .rodata                           \n"
  72         ".global skl_call_thunk_template                \n"
  73         "skl_call_thunk_template:                       \n"
  74                 __stringify(INCREMENT_CALL_DEPTH)"      \n"
  75         ".global skl_call_thunk_tail                    \n"
  76         "skl_call_thunk_tail:                           \n"
  77         ".popsection                                    \n"
  78 );
  79
  80 extern u8 skl_call_thunk_template[];
  81 extern u8 skl_call_thunk_tail[];
  82
  83 #define SKL_TMPL_SIZE \
  84         ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
  85
  86 extern void error_entry(void);
  87 extern void xen_error_entry(void);
  88 extern void paranoid_entry(void);
  89
  90 static inline bool within_coretext(const struct core_text *ct, void *addr)
  91 {
  92         unsigned long p = (unsigned long)addr;
  93
  94         return ct->base <= p && p < ct->end;
  95 }
  96
  97 static inline bool within_module_coretext(void *addr)
  98 {
  99         bool ret = false;
 100
 101 #ifdef CONFIG_MODULES
 102         struct module *mod;
 103
 104         preempt_disable();
 105         mod = __module_address((unsigned long)addr);
 106         if (mod && within_module_core((unsigned long)addr, mod))
 107                 ret = true;
 108         preempt_enable();
 109 #endif
 110         return ret;
 111 }
 112
 113 static bool is_coretext(const struct core_text *ct, void *addr)
 114 {
 115         if (ct && within_coretext(ct, addr))
 116                 return true;
 117         if (within_coretext(&builtin_coretext, addr))
 118                 return true;
 119         return within_module_coretext(addr);
 120 }
 121
 122 static bool skip_addr(void *dest)
 123 {
 124         if (dest == error_entry)
 125                 return true;
 126         if (dest == paranoid_entry)
 127                 return true;
 128         if (dest == xen_error_entry)
 129                 return true;
 130         /* Does FILL_RSB... */
 131         if (dest == __switch_to_asm)
 132                 return true;
 133         /* Accounts directly */
 134         if (dest == ret_from_fork)
 135                 return true;
 136 #ifdef CONFIG_HOTPLUG_CPU
 137         if (dest == start_cpu0)
 138                 return true;
 139 #endif
 140 #ifdef CONFIG_FUNCTION_TRACER
 141         if (dest == __fentry__)
 142                 return true;
 143 #endif
 144 #ifdef CONFIG_KEXEC_CORE
 145         if (dest >= (void *)relocate_kernel &&
 146             dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
 147                 return true;
 148 #endif
 149 #ifdef CONFIG_XEN
 150         if (dest >= (void *)hypercall_page &&
 151             dest < (void*)hypercall_page + PAGE_SIZE)
 152                 return true;
 153 #endif
 154         return false;
 155 }
 156
 157 static __init_or_module void *call_get_dest(void *addr)
 158 {
 159         struct insn insn;
 160         void *dest;
 161         int ret;
 162
 163         ret = insn_decode_kernel(&insn, addr);
 164         if (ret)
 165                 return ERR_PTR(ret);
 166
 167         /* Patched out call? */
 168         if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
 169                 return NULL;
 170
 171         dest = addr + insn.length + insn.immediate.value;
 172         if (skip_addr(dest))
 173                 return NULL;
 174         return dest;
 175 }
 176
 177 static const u8 nops[] = {
 178         0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
 179         0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
 180         0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
 181         0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
 182 };
 183
 184 static void *patch_dest(void *dest, bool direct)
 185 {
 186         unsigned int tsize = SKL_TMPL_SIZE;
 187         u8 *pad = dest - tsize;
 188
 189         /* Already patched? */
 190         if (!bcmp(pad, skl_call_thunk_template, tsize))
 191                 return pad;
 192
 193         /* Ensure there are nops */
 194         if (bcmp(pad, nops, tsize)) {
 195                 pr_warn_once("Invalid padding area for %pS\n", dest);
 196                 return NULL;
 197         }
 198
 199         if (direct)
 200                 memcpy(pad, skl_call_thunk_template, tsize);
 201         else
 202                 text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true);
 203         return pad;
 204 }
 205
 206 static __init_or_module void patch_call(void *addr, const struct core_text *ct)
 207 {
 208         void *pad, *dest;
 209         u8 bytes[8];
 210
 211         if (!within_coretext(ct, addr))
 212                 return;
 213
 214         dest = call_get_dest(addr);
 215         if (!dest || WARN_ON_ONCE(IS_ERR(dest)))
 216                 return;
 217
 218         if (!is_coretext(ct, dest))
 219                 return;
 220
 221         pad = patch_dest(dest, within_coretext(ct, dest));
 222         if (!pad)
 223                 return;
 224
 225         prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
 226                 dest, dest, pad);
 227         __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
 228         text_poke_early(addr, bytes, CALL_INSN_SIZE);
 229 }
 230
 231 static __init_or_module void
 232 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
 233 {
 234         s32 *s;
 235
 236         for (s = start; s < end; s++)
 237                 patch_call((void *)s + *s, ct);
 238 }
 239
 240 static __init_or_module void
 241 patch_paravirt_call_sites(struct paravirt_patch_site *start,
 242                           struct paravirt_patch_site *end,
 243                           const struct core_text *ct)
 244 {
 245         struct paravirt_patch_site *p;
 246
 247         for (p = start; p < end; p++)
 248                 patch_call(p->instr, ct);
 249 }
 250
 251 static __init_or_module void
 252 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
 253 {
 254         prdbg("Patching call sites %s\n", ct->name);
 255         patch_call_sites(cs->call_start, cs->call_end, ct);
 256         patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct);
 257         prdbg("Patching call sites done%s\n", ct->name);
 258 }
 259
 260 void __init callthunks_patch_builtin_calls(void)
 261 {
 262         struct callthunk_sites cs = {
 263                 .call_start     = __call_sites,
 264                 .call_end       = __call_sites_end,
 265                 .pv_start       = __parainstructions,
 266                 .pv_end         = __parainstructions_end
 267         };
 268
 269         if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
 270                 return;
 271
 272         pr_info("Setting up call depth tracking\n");
 273         mutex_lock(&text_mutex);
 274         callthunks_setup(&cs, &builtin_coretext);
 275         static_call_force_reinit();
 276         thunks_initialized = true;
 277         mutex_unlock(&text_mutex);
 278 }
 279
 280 void *callthunks_translate_call_dest(void *dest)
 281 {
 282         void *target;
 283
 284         lockdep_assert_held(&text_mutex);
 285
 286         if (!thunks_initialized || skip_addr(dest))
 287                 return dest;
 288
 289         if (!is_coretext(NULL, dest))
 290                 return dest;
 291
 292         target = patch_dest(dest, false);
 293         return target ? : dest;
 294 }
 295
 296 bool is_callthunk(void *addr)
 297 {
 298         unsigned int tmpl_size = SKL_TMPL_SIZE;
 299         void *tmpl = skl_call_thunk_template;
 300         unsigned long dest;
 301
 302         dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
 303         if (!thunks_initialized || skip_addr((void *)dest))
 304                 return false;
 305
 306         return !bcmp((void *)(dest - tmpl_size), tmpl, tmpl_size);
 307 }
 308
 309 #ifdef CONFIG_BPF_JIT
 310 int x86_call_depth_emit_accounting(u8 **pprog, void *func)
 311 {
 312         unsigned int tmpl_size = SKL_TMPL_SIZE;
 313         void *tmpl = skl_call_thunk_template;
 314
 315         if (!thunks_initialized)
 316                 return 0;
 317
 318         /* Is function call target a thunk? */
 319         if (func && is_callthunk(func))
 320                 return 0;
 321
 322         memcpy(*pprog, tmpl, tmpl_size);
 323         *pprog += tmpl_size;
 324         return tmpl_size;
 325 }
 326 #endif
 327
 328 #ifdef CONFIG_MODULES
 329 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
 330                                             struct module *mod)
 331 {
 332         struct core_text ct = {
 333                 .base = (unsigned long)mod->core_layout.base,
 334                 .end  = (unsigned long)mod->core_layout.base + mod->core_layout.size,
 335                 .name = mod->name,
 336         };
 337
 338         if (!thunks_initialized)
 339                 return;
 340
 341         mutex_lock(&text_mutex);
 342         callthunks_setup(cs, &ct);
 343         mutex_unlock(&text_mutex);
 344 }
 345 #endif /* CONFIG_MODULES */
 346
 347 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
 348 static int callthunks_debug_show(struct seq_file *m, void *p)
 349 {
 350         unsigned long cpu = (unsigned long)m->private;
 351
 352         seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
 353                    per_cpu(__x86_call_count, cpu),
 354                    per_cpu(__x86_ret_count, cpu),
 355                    per_cpu(__x86_stuffs_count, cpu),
 356                    per_cpu(__x86_ctxsw_count, cpu));
 357         return 0;
 358 }
 359
 360 static int callthunks_debug_open(struct inode *inode, struct file *file)
 361 {
 362         return single_open(file, callthunks_debug_show, inode->i_private);
 363 }
 364
 365 static const struct file_operations dfs_ops = {
 366         .open           = callthunks_debug_open,
 367         .read           = seq_read,
 368         .llseek         = seq_lseek,
 369         .release        = single_release,
 370 };
 371
 372 static int __init callthunks_debugfs_init(void)
 373 {
 374         struct dentry *dir;
 375         unsigned long cpu;
 376
 377         dir = debugfs_create_dir("callthunks", NULL);
 378         for_each_possible_cpu(cpu) {
 379                 void *arg = (void *)cpu;
 380                 char name [10];
 381
 382                 sprintf(name, "cpu%lu", cpu);
 383                 debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
 384         }
 385         return 0;
 386 }
 387 __initcall(callthunks_debugfs_init);
 388 #endif