1 // SPDX-License-Identifier: GPL-2.0-only
3 #define pr_fmt(fmt) "callthunks: " fmt
5 #include <linux/debugfs.h>
6 #include <linux/kallsyms.h>
7 #include <linux/memory.h>
8 #include <linux/moduleloader.h>
9 #include <linux/static_call.h>
11 #include <asm/alternative.h>
12 #include <asm/asm-offsets.h>
14 #include <asm/ftrace.h>
16 #include <asm/kexec.h>
17 #include <asm/nospec-branch.h>
18 #include <asm/paravirt.h>
19 #include <asm/sections.h>
20 #include <asm/switch_to.h>
21 #include <asm/sync_core.h>
22 #include <asm/text-patching.h>
23 #include <asm/xen/hypercall.h>
25 static int __initdata_or_module debug_callthunks;
27 #define prdbg(fmt, args...) \
29 if (debug_callthunks) \
30 printk(KERN_DEBUG pr_fmt(fmt), ##args); \
33 static int __init debug_thunks(char *str)
38 __setup("debug-callthunks", debug_thunks);
40 #ifdef CONFIG_CALL_THUNKS_DEBUG
41 DEFINE_PER_CPU(u64, __x86_call_count);
42 DEFINE_PER_CPU(u64, __x86_ret_count);
43 DEFINE_PER_CPU(u64, __x86_stuffs_count);
44 DEFINE_PER_CPU(u64, __x86_ctxsw_count);
45 EXPORT_SYMBOL_GPL(__x86_ctxsw_count);
46 EXPORT_SYMBOL_GPL(__x86_call_count);
49 extern s32 __call_sites[], __call_sites_end[];
53 unsigned int template_size;
62 static bool thunks_initialized __ro_after_init;
64 static const struct core_text builtin_coretext = {
65 .base = (unsigned long)_text,
66 .end = (unsigned long)_etext,
71 ".pushsection .rodata \n"
72 ".global skl_call_thunk_template \n"
73 "skl_call_thunk_template: \n"
74 __stringify(INCREMENT_CALL_DEPTH)" \n"
75 ".global skl_call_thunk_tail \n"
76 "skl_call_thunk_tail: \n"
80 extern u8 skl_call_thunk_template[];
81 extern u8 skl_call_thunk_tail[];
83 #define SKL_TMPL_SIZE \
84 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
86 extern void error_entry(void);
87 extern void xen_error_entry(void);
88 extern void paranoid_entry(void);
90 static inline bool within_coretext(const struct core_text *ct, void *addr)
92 unsigned long p = (unsigned long)addr;
94 return ct->base <= p && p < ct->end;
97 static inline bool within_module_coretext(void *addr)
101 #ifdef CONFIG_MODULES
105 mod = __module_address((unsigned long)addr);
106 if (mod && within_module_core((unsigned long)addr, mod))
113 static bool is_coretext(const struct core_text *ct, void *addr)
115 if (ct && within_coretext(ct, addr))
117 if (within_coretext(&builtin_coretext, addr))
119 return within_module_coretext(addr);
122 static bool skip_addr(void *dest)
124 if (dest == error_entry)
126 if (dest == paranoid_entry)
128 if (dest == xen_error_entry)
130 /* Does FILL_RSB... */
131 if (dest == __switch_to_asm)
133 /* Accounts directly */
134 if (dest == ret_from_fork)
136 #ifdef CONFIG_HOTPLUG_CPU
137 if (dest == start_cpu0)
140 #ifdef CONFIG_FUNCTION_TRACER
141 if (dest == __fentry__)
144 #ifdef CONFIG_KEXEC_CORE
145 if (dest >= (void *)relocate_kernel &&
146 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
150 if (dest >= (void *)hypercall_page &&
151 dest < (void*)hypercall_page + PAGE_SIZE)
157 static __init_or_module void *call_get_dest(void *addr)
163 ret = insn_decode_kernel(&insn, addr);
167 /* Patched out call? */
168 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
171 dest = addr + insn.length + insn.immediate.value;
177 static const u8 nops[] = {
178 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
179 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
180 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
181 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
184 static void *patch_dest(void *dest, bool direct)
186 unsigned int tsize = SKL_TMPL_SIZE;
187 u8 *pad = dest - tsize;
189 /* Already patched? */
190 if (!bcmp(pad, skl_call_thunk_template, tsize))
193 /* Ensure there are nops */
194 if (bcmp(pad, nops, tsize)) {
195 pr_warn_once("Invalid padding area for %pS\n", dest);
200 memcpy(pad, skl_call_thunk_template, tsize);
202 text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true);
206 static __init_or_module void patch_call(void *addr, const struct core_text *ct)
211 if (!within_coretext(ct, addr))
214 dest = call_get_dest(addr);
215 if (!dest || WARN_ON_ONCE(IS_ERR(dest)))
218 if (!is_coretext(ct, dest))
221 pad = patch_dest(dest, within_coretext(ct, dest));
225 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
227 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
228 text_poke_early(addr, bytes, CALL_INSN_SIZE);
231 static __init_or_module void
232 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
236 for (s = start; s < end; s++)
237 patch_call((void *)s + *s, ct);
240 static __init_or_module void
241 patch_paravirt_call_sites(struct paravirt_patch_site *start,
242 struct paravirt_patch_site *end,
243 const struct core_text *ct)
245 struct paravirt_patch_site *p;
247 for (p = start; p < end; p++)
248 patch_call(p->instr, ct);
251 static __init_or_module void
252 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
254 prdbg("Patching call sites %s\n", ct->name);
255 patch_call_sites(cs->call_start, cs->call_end, ct);
256 patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct);
257 prdbg("Patching call sites done%s\n", ct->name);
260 void __init callthunks_patch_builtin_calls(void)
262 struct callthunk_sites cs = {
263 .call_start = __call_sites,
264 .call_end = __call_sites_end,
265 .pv_start = __parainstructions,
266 .pv_end = __parainstructions_end
269 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
272 pr_info("Setting up call depth tracking\n");
273 mutex_lock(&text_mutex);
274 callthunks_setup(&cs, &builtin_coretext);
275 static_call_force_reinit();
276 thunks_initialized = true;
277 mutex_unlock(&text_mutex);
280 void *callthunks_translate_call_dest(void *dest)
284 lockdep_assert_held(&text_mutex);
286 if (!thunks_initialized || skip_addr(dest))
289 if (!is_coretext(NULL, dest))
292 target = patch_dest(dest, false);
293 return target ? : dest;
296 bool is_callthunk(void *addr)
298 unsigned int tmpl_size = SKL_TMPL_SIZE;
299 void *tmpl = skl_call_thunk_template;
302 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
303 if (!thunks_initialized || skip_addr((void *)dest))
306 return !bcmp((void *)(dest - tmpl_size), tmpl, tmpl_size);
309 #ifdef CONFIG_BPF_JIT
310 int x86_call_depth_emit_accounting(u8 **pprog, void *func)
312 unsigned int tmpl_size = SKL_TMPL_SIZE;
313 void *tmpl = skl_call_thunk_template;
315 if (!thunks_initialized)
318 /* Is function call target a thunk? */
319 if (func && is_callthunk(func))
322 memcpy(*pprog, tmpl, tmpl_size);
328 #ifdef CONFIG_MODULES
329 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
332 struct core_text ct = {
333 .base = (unsigned long)mod->core_layout.base,
334 .end = (unsigned long)mod->core_layout.base + mod->core_layout.size,
338 if (!thunks_initialized)
341 mutex_lock(&text_mutex);
342 callthunks_setup(cs, &ct);
343 mutex_unlock(&text_mutex);
345 #endif /* CONFIG_MODULES */
347 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
348 static int callthunks_debug_show(struct seq_file *m, void *p)
350 unsigned long cpu = (unsigned long)m->private;
352 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
353 per_cpu(__x86_call_count, cpu),
354 per_cpu(__x86_ret_count, cpu),
355 per_cpu(__x86_stuffs_count, cpu),
356 per_cpu(__x86_ctxsw_count, cpu));
360 static int callthunks_debug_open(struct inode *inode, struct file *file)
362 return single_open(file, callthunks_debug_show, inode->i_private);
365 static const struct file_operations dfs_ops = {
366 .open = callthunks_debug_open,
369 .release = single_release,
372 static int __init callthunks_debugfs_init(void)
377 dir = debugfs_create_dir("callthunks", NULL);
378 for_each_possible_cpu(cpu) {
379 void *arg = (void *)cpu;
382 sprintf(name, "cpu%lu", cpu);
383 debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
387 __initcall(callthunks_debugfs_init);