1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_IRQ_STACK_H
3 #define _ASM_X86_IRQ_STACK_H
5 #include <linux/ptrace.h>
7 #include <asm/processor.h>
12 * Macro to inline switching to an interrupt stack and invoking function
13 * calls from there. The following rules apply:
17 * 1. Write the stack pointer into the top most place of the irq
18 * stack. This ensures that the various unwinders can link back to the
21 * 2. Switch the stack pointer to the top of the irq stack.
23 * 3. Invoke whatever needs to be done (@asm_call argument)
25 * 4. Pop the original stack pointer from the top of the irq stack
26 * which brings it back to the original stack where it left off.
28 * - Function invocation:
30 * To allow flexible usage of the macro, the actual function code including
31 * the store of the arguments in the call ABI registers is handed in via
32 * the @asm_call argument.
37 * The @tos variable holds a pointer to the top of the irq stack and
38 * _must_ be allocated in a non-callee saved register as this is a
39 * restriction coming from objtool.
41 * Note, that (tos) is both in input and output constraints to ensure
42 * that the compiler does not assume that R11 is left untouched in
43 * case this macro is used in some place where the per cpu interrupt
44 * stack pointer is used again afterwards
46 * - Function arguments:
47 * The function argument(s), if any, have to be defined in register
48 * variables at the place where this is invoked. Storing the
49 * argument(s) in the proper register(s) is part of the @asm_call
53 * The constraints have to be done very carefully because the compiler
54 * does not know about the assembly call.
57 * As documented already above the @tos variable is required to be in
58 * the output constraints to make the compiler aware that R11 cannot be
59 * reused after the asm() statement.
61 * For builds with CONFIG_UNWIND_FRAME_POINTER ASM_CALL_CONSTRAINT is
62 * required as well as this prevents certain creative GCC variants from
63 * misplacing the ASM code.
67 * Immediate, which tells the compiler that the function is referenced.
70 * Register. The actual register is defined by the variable declaration.
72 * - function arguments:
73 * The constraints are handed in via the 'argconstr' argument list. They
74 * describe the register arguments which are used in @asm_call.
77 * Function calls can clobber anything except the callee-saved
78 * registers. Tell the compiler.
80 #define call_on_stack(stack, func, asm_call, argconstr...) \
82 register void *tos asm("r11"); \
84 tos = ((void *)(stack)); \
86 asm_inline volatile( \
87 "movq %%rsp, (%[tos]) \n" \
88 "movq %[tos], %%rsp \n" \
94 : "+r" (tos), ASM_CALL_CONSTRAINT \
95 : [__func] "i" (func), [tos] "r" (tos) argconstr \
96 : "cc", "rax", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10", \
101 #define ASM_CALL_ARG0 \
102 "call %P[__func] \n" \
105 #define ASM_CALL_ARG1 \
106 "movq %[arg1], %%rdi \n" \
109 #define ASM_CALL_ARG2 \
110 "movq %[arg2], %%rsi \n" \
113 #define ASM_CALL_ARG3 \
114 "movq %[arg3], %%rdx \n" \
117 #define call_on_irqstack(func, asm_call, argconstr...) \
118 call_on_stack(__this_cpu_read(hardirq_stack_ptr), \
119 func, asm_call, argconstr)
121 /* Macros to assert type correctness for run_*_on_irqstack macros */
122 #define assert_function_type(func, proto) \
123 static_assert(__builtin_types_compatible_p(typeof(&func), proto))
125 #define assert_arg_type(arg, proto) \
126 static_assert(__builtin_types_compatible_p(typeof(arg), proto))
129 * Macro to invoke system vector and device interrupt C handlers.
131 #define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...) \
134 * User mode entry and interrupt on the irq stack do not \
135 * switch stacks. If from user mode the task stack is empty. \
137 if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) { \
143 * Mark the irq stack inuse _before_ and unmark _after_ \
144 * switching stacks. Interrupts are disabled in both \
145 * places. Invoke the stack switch macro with the call \
146 * sequence which matches the above direct invocation. \
148 __this_cpu_write(hardirq_stack_inuse, true); \
149 call_on_irqstack(func, asm_call, constr); \
150 __this_cpu_write(hardirq_stack_inuse, false); \
155 * Function call sequence for __call_on_irqstack() for system vectors.
157 * Note that irq_enter_rcu() and irq_exit_rcu() do not use the input
158 * mechanism because these functions are global and cannot be optimized out
159 * when compiling a particular source file which uses one of these macros.
161 * The argument (regs) does not need to be pushed or stashed in a callee
162 * saved register to be safe vs. the irq_enter_rcu() call because the
163 * clobbers already prevent the compiler from storing it in a callee
164 * clobbered register. As the compiler has to preserve @regs for the final
165 * call to idtentry_exit() anyway, it's likely that it does not cause extra
166 * effort for this asm magic.
168 #define ASM_CALL_SYSVEC \
169 "call irq_enter_rcu \n" \
171 "call irq_exit_rcu \n"
173 #define SYSVEC_CONSTRAINTS , [arg1] "r" (regs)
175 #define run_sysvec_on_irqstack_cond(func, regs) \
177 assert_function_type(func, void (*)(struct pt_regs *)); \
178 assert_arg_type(regs, struct pt_regs *); \
180 call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC, \
181 SYSVEC_CONSTRAINTS, regs); \
185 * As in ASM_CALL_SYSVEC above the clobbers force the compiler to store
186 * @regs and @vector in callee saved registers.
188 #define ASM_CALL_IRQ \
189 "call irq_enter_rcu \n" \
191 "call irq_exit_rcu \n"
193 #define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" ((unsigned long)vector)
195 #define run_irq_on_irqstack_cond(func, regs, vector) \
197 assert_function_type(func, void (*)(struct pt_regs *, u32)); \
198 assert_arg_type(regs, struct pt_regs *); \
199 assert_arg_type(vector, u32); \
201 call_on_irqstack_cond(func, regs, ASM_CALL_IRQ, \
202 IRQ_CONSTRAINTS, regs, vector); \
205 #ifndef CONFIG_PREEMPT_RT
207 * Macro to invoke __do_softirq on the irq stack. This is only called from
208 * task context when bottom halves are about to be reenabled and soft
209 * interrupts are pending to be processed. The interrupt stack cannot be in
212 #define do_softirq_own_stack() \
214 __this_cpu_write(hardirq_stack_inuse, true); \
215 call_on_irqstack(__do_softirq, ASM_CALL_ARG0); \
216 __this_cpu_write(hardirq_stack_inuse, false); \
221 #else /* CONFIG_X86_64 */
222 /* System vector handlers always run on the stack they interrupted. */
223 #define run_sysvec_on_irqstack_cond(func, regs) \
230 /* Switches to the irq stack within func() */
231 #define run_irq_on_irqstack_cond(func, regs, vector) \
234 func(regs, vector); \
238 #endif /* !CONFIG_X86_64 */