1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/context_tracking.h>
4 #include <linux/entry-common.h>
5 #include <linux/livepatch.h>
6 #include <linux/audit.h>
8 #define CREATE_TRACE_POINTS
9 #include <trace/events/syscalls.h>
12 * enter_from_user_mode - Establish state when coming from user mode
14 * Syscall/interrupt entry disables interrupts, but user mode is traced as
15 * interrupts enabled. Also with NO_HZ_FULL RCU might be idle.
17 * 1) Tell lockdep that interrupts are disabled
18 * 2) Invoke context tracking if enabled to reactivate RCU
19 * 3) Trace interrupts off state
21 static __always_inline void enter_from_user_mode(struct pt_regs *regs)
23 arch_check_user_regs(regs);
24 lockdep_hardirqs_off(CALLER_ADDR0);
26 CT_WARN_ON(ct_state() != CONTEXT_USER);
29 instrumentation_begin();
30 trace_hardirqs_off_finish();
31 instrumentation_end();
34 static inline void syscall_enter_audit(struct pt_regs *regs, long syscall)
36 if (unlikely(audit_context())) {
37 unsigned long args[6];
39 syscall_get_arguments(current, regs, args);
40 audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]);
44 static long syscall_trace_enter(struct pt_regs *regs, long syscall,
45 unsigned long ti_work)
50 if (ti_work & (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU)) {
51 ret = arch_syscall_enter_tracehook(regs);
52 if (ret || (ti_work & _TIF_SYSCALL_EMU))
56 /* Do seccomp after ptrace, to catch any tracer changes. */
57 if (ti_work & _TIF_SECCOMP) {
58 ret = __secure_computing(NULL);
63 /* Either of the above might have changed the syscall number */
64 syscall = syscall_get_nr(current, regs);
66 if (unlikely(ti_work & _TIF_SYSCALL_TRACEPOINT))
67 trace_sys_enter(regs, syscall);
69 syscall_enter_audit(regs, syscall);
71 return ret ? : syscall;
74 static __always_inline long
75 __syscall_enter_from_user_work(struct pt_regs *regs, long syscall)
77 unsigned long ti_work;
79 ti_work = READ_ONCE(current_thread_info()->flags);
80 if (ti_work & SYSCALL_ENTER_WORK)
81 syscall = syscall_trace_enter(regs, syscall, ti_work);
86 long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)
88 return __syscall_enter_from_user_work(regs, syscall);
91 noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
95 enter_from_user_mode(regs);
97 instrumentation_begin();
99 ret = __syscall_enter_from_user_work(regs, syscall);
100 instrumentation_end();
105 noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs)
107 enter_from_user_mode(regs);
108 instrumentation_begin();
110 instrumentation_end();
114 * exit_to_user_mode - Fixup state when exiting to user mode
116 * Syscall/interupt exit enables interrupts, but the kernel state is
117 * interrupts disabled when this is invoked. Also tell RCU about it.
119 * 1) Trace interrupts on state
120 * 2) Invoke context tracking if enabled to adjust RCU state
121 * 3) Invoke architecture specific last minute exit code, e.g. speculation
123 * 4) Tell lockdep that interrupts are enabled
125 static __always_inline void exit_to_user_mode(void)
127 instrumentation_begin();
128 trace_hardirqs_on_prepare();
129 lockdep_hardirqs_on_prepare(CALLER_ADDR0);
130 instrumentation_end();
133 arch_exit_to_user_mode();
134 lockdep_hardirqs_on(CALLER_ADDR0);
137 /* Workaround to allow gradual conversion of architecture code */
138 void __weak arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) { }
140 static void handle_signal_work(struct pt_regs *regs, unsigned long ti_work)
142 if (ti_work & _TIF_NOTIFY_SIGNAL)
143 tracehook_notify_signal();
145 arch_do_signal_or_restart(regs, ti_work & _TIF_SIGPENDING);
148 static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
149 unsigned long ti_work)
152 * Before returning to user space ensure that all pending work
153 * items have been completed.
155 while (ti_work & EXIT_TO_USER_MODE_WORK) {
157 local_irq_enable_exit_to_user(ti_work);
159 if (ti_work & _TIF_NEED_RESCHED)
162 if (ti_work & _TIF_UPROBE)
163 uprobe_notify_resume(regs);
165 if (ti_work & _TIF_PATCH_PENDING)
166 klp_update_patch_state(current);
168 if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
169 handle_signal_work(regs, ti_work);
171 if (ti_work & _TIF_NOTIFY_RESUME) {
172 tracehook_notify_resume(regs);
173 rseq_handle_notify_resume(NULL, regs);
176 /* Architecture specific TIF work */
177 arch_exit_to_user_mode_work(regs, ti_work);
180 * Disable interrupts and reevaluate the work flags as they
181 * might have changed while interrupts and preemption was
184 local_irq_disable_exit_to_user();
185 ti_work = READ_ONCE(current_thread_info()->flags);
188 /* Return the latest work state for arch_exit_to_user_mode() */
192 static void exit_to_user_mode_prepare(struct pt_regs *regs)
194 unsigned long ti_work = READ_ONCE(current_thread_info()->flags);
196 lockdep_assert_irqs_disabled();
198 if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
199 ti_work = exit_to_user_mode_loop(regs, ti_work);
201 arch_exit_to_user_mode_prepare(regs, ti_work);
203 /* Ensure that the address limit is intact and no locks are held */
204 addr_limit_user_check();
205 lockdep_assert_irqs_disabled();
209 #ifndef _TIF_SINGLESTEP
210 static inline bool report_single_step(unsigned long ti_work)
216 * If TIF_SYSCALL_EMU is set, then the only reason to report is when
217 * TIF_SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall
218 * instruction has been already reported in syscall_enter_from_user_mode().
220 #define SYSEMU_STEP (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU)
222 static inline bool report_single_step(unsigned long ti_work)
224 return (ti_work & SYSEMU_STEP) == _TIF_SINGLESTEP;
228 static void syscall_exit_work(struct pt_regs *regs, unsigned long ti_work)
232 audit_syscall_exit(regs);
234 if (ti_work & _TIF_SYSCALL_TRACEPOINT)
235 trace_sys_exit(regs, syscall_get_return_value(current, regs));
237 step = report_single_step(ti_work);
238 if (step || ti_work & _TIF_SYSCALL_TRACE)
239 arch_syscall_exit_tracehook(regs, step);
243 * Syscall specific exit to user mode preparation. Runs with interrupts
246 static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)
248 u32 cached_flags = READ_ONCE(current_thread_info()->flags);
249 unsigned long nr = syscall_get_nr(current, regs);
251 CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
253 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
254 if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
261 * Do one-time syscall specific work. If these work items are
262 * enabled, we want to run them exactly once per syscall exit with
263 * interrupts enabled.
265 if (unlikely(cached_flags & SYSCALL_EXIT_WORK))
266 syscall_exit_work(regs, cached_flags);
269 __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs)
271 instrumentation_begin();
272 syscall_exit_to_user_mode_prepare(regs);
273 local_irq_disable_exit_to_user();
274 exit_to_user_mode_prepare(regs);
275 instrumentation_end();
279 noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)
281 enter_from_user_mode(regs);
284 noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs)
286 instrumentation_begin();
287 exit_to_user_mode_prepare(regs);
288 instrumentation_end();
292 noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
294 irqentry_state_t ret = {
298 if (user_mode(regs)) {
299 irqentry_enter_from_user_mode(regs);
304 * If this entry hit the idle task invoke rcu_irq_enter() whether
305 * RCU is watching or not.
307 * Interupts can nest when the first interrupt invokes softirq
308 * processing on return which enables interrupts.
310 * Scheduler ticks in the idle task can mark quiescent state and
311 * terminate a grace period, if and only if the timer interrupt is
312 * not nested into another interrupt.
314 * Checking for rcu_is_watching() here would prevent the nesting
315 * interrupt to invoke rcu_irq_enter(). If that nested interrupt is
316 * the tick then rcu_flavor_sched_clock_irq() would wrongfully
317 * assume that it is the first interupt and eventually claim
318 * quiescient state and end grace periods prematurely.
320 * Unconditionally invoke rcu_irq_enter() so RCU state stays
323 * TINY_RCU does not support EQS, so let the compiler eliminate
324 * this part when enabled.
326 if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
328 * If RCU is not watching then the same careful
329 * sequence vs. lockdep and tracing is required
330 * as in irqentry_enter_from_user_mode().
332 lockdep_hardirqs_off(CALLER_ADDR0);
334 instrumentation_begin();
335 trace_hardirqs_off_finish();
336 instrumentation_end();
343 * If RCU is watching then RCU only wants to check whether it needs
344 * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick()
345 * already contains a warning when RCU is not watching, so no point
346 * in having another one here.
348 lockdep_hardirqs_off(CALLER_ADDR0);
349 instrumentation_begin();
350 rcu_irq_enter_check_tick();
351 trace_hardirqs_off_finish();
352 instrumentation_end();
357 void irqentry_exit_cond_resched(void)
359 if (!preempt_count()) {
360 /* Sanity check RCU and thread stack */
361 rcu_irq_exit_check_preempt();
362 if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
363 WARN_ON_ONCE(!on_thread_stack());
365 preempt_schedule_irq();
369 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
371 lockdep_assert_irqs_disabled();
373 /* Check whether this returns to user mode */
374 if (user_mode(regs)) {
375 irqentry_exit_to_user_mode(regs);
376 } else if (!regs_irqs_disabled(regs)) {
378 * If RCU was not watching on entry this needs to be done
379 * carefully and needs the same ordering of lockdep/tracing
380 * and RCU as the return to user mode path.
382 if (state.exit_rcu) {
383 instrumentation_begin();
384 /* Tell the tracer that IRET will enable interrupts */
385 trace_hardirqs_on_prepare();
386 lockdep_hardirqs_on_prepare(CALLER_ADDR0);
387 instrumentation_end();
389 lockdep_hardirqs_on(CALLER_ADDR0);
393 instrumentation_begin();
394 if (IS_ENABLED(CONFIG_PREEMPTION))
395 irqentry_exit_cond_resched();
396 /* Covers both tracing and lockdep */
398 instrumentation_end();
401 * IRQ flags state is correct already. Just tell RCU if it
402 * was not watching on entry.