1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra
5 * Provides a framework for enqueueing and running callbacks from hardirq
6 * context. The enqueueing is NMI-safe.
10 #include <linux/kernel.h>
11 #include <linux/export.h>
12 #include <linux/irq_work.h>
13 #include <linux/percpu.h>
14 #include <linux/hardirq.h>
15 #include <linux/irqflags.h>
16 #include <linux/sched.h>
17 #include <linux/tick.h>
18 #include <linux/cpu.h>
19 #include <linux/notifier.h>
20 #include <linux/smp.h>
21 #include <linux/smpboot.h>
22 #include <linux/interrupt.h>
23 #include <asm/processor.h>
26 static DEFINE_PER_CPU(struct llist_head, raised_list);
27 static DEFINE_PER_CPU(struct llist_head, lazy_list);
28 static DEFINE_PER_CPU(struct task_struct *, irq_workd);
30 static void wake_irq_workd(void)
32 struct task_struct *tsk = __this_cpu_read(irq_workd);
34 if (!llist_empty(this_cpu_ptr(&lazy_list)) && tsk)
39 static void irq_work_wake(struct irq_work *entry)
44 static DEFINE_PER_CPU(struct irq_work, irq_work_wakeup) =
45 IRQ_WORK_INIT_HARD(irq_work_wake);
48 static int irq_workd_should_run(unsigned int cpu)
50 return !llist_empty(this_cpu_ptr(&lazy_list));
54 * Claim the entry so that no one else will poke at it.
56 static bool irq_work_claim(struct irq_work *work)
60 oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->flags);
62 * If the work is already pending, no need to raise the IPI.
63 * The pairing atomic_fetch_andnot() in irq_work_run() makes sure
64 * everything we did before is visible.
66 if (oflags & IRQ_WORK_PENDING)
71 void __weak arch_irq_work_raise(void)
74 * Lame architectures will get the timer tick callback
78 /* Enqueue on current CPU, work must already be claimed and preempt disabled */
79 static void __irq_work_queue_local(struct irq_work *work)
81 struct llist_head *list;
82 bool rt_lazy_work = false;
83 bool lazy_work = false;
86 work_flags = atomic_read(&work->flags);
87 if (work_flags & IRQ_WORK_LAZY)
89 else if (IS_ENABLED(CONFIG_PREEMPT_RT) &&
90 !(work_flags & IRQ_WORK_HARD_IRQ))
93 if (lazy_work || rt_lazy_work)
94 list = this_cpu_ptr(&lazy_list);
96 list = this_cpu_ptr(&raised_list);
98 if (!llist_add(&work->llnode, list))
101 /* If the work is "lazy", handle it from next tick if any */
102 if (!lazy_work || tick_nohz_tick_stopped())
103 arch_irq_work_raise();
106 /* Enqueue the irq work @work on the current CPU */
107 bool irq_work_queue(struct irq_work *work)
109 /* Only queue if not already pending */
110 if (!irq_work_claim(work))
113 /* Queue the entry and raise the IPI if needed. */
115 __irq_work_queue_local(work);
120 EXPORT_SYMBOL_GPL(irq_work_queue);
123 * Enqueue the irq_work @work on @cpu unless it's already pending
126 * Can be re-enqueued while the callback is still in progress.
128 bool irq_work_queue_on(struct irq_work *work, int cpu)
131 return irq_work_queue(work);
133 #else /* CONFIG_SMP: */
134 /* All work should have been flushed before going offline */
135 WARN_ON_ONCE(cpu_is_offline(cpu));
137 /* Only queue if not already pending */
138 if (!irq_work_claim(work))
142 if (cpu != smp_processor_id()) {
143 /* Arch remote IPI send/receive backend aren't NMI safe */
144 WARN_ON_ONCE(in_nmi());
147 * On PREEMPT_RT the items which are not marked as
148 * IRQ_WORK_HARD_IRQ are added to the lazy list and a HARD work
149 * item is used on the remote CPU to wake the thread.
151 if (IS_ENABLED(CONFIG_PREEMPT_RT) &&
152 !(atomic_read(&work->flags) & IRQ_WORK_HARD_IRQ)) {
154 if (!llist_add(&work->llnode, &per_cpu(lazy_list, cpu)))
157 work = &per_cpu(irq_work_wakeup, cpu);
158 if (!irq_work_claim(work))
162 __smp_call_single_queue(cpu, &work->llnode);
164 __irq_work_queue_local(work);
170 #endif /* CONFIG_SMP */
174 bool irq_work_needs_cpu(void)
176 struct llist_head *raised, *lazy;
178 raised = this_cpu_ptr(&raised_list);
179 lazy = this_cpu_ptr(&lazy_list);
181 if (llist_empty(raised) && llist_empty(lazy))
184 /* All work should have been flushed before going offline */
185 WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
190 void irq_work_single(void *arg)
192 struct irq_work *work = arg;
196 * Clear the PENDING bit, after this point the @work
198 * Make it immediately visible so that other CPUs trying
199 * to claim that work don't rely on us to handle their data
200 * while we are in the middle of the func.
202 flags = atomic_fetch_andnot(IRQ_WORK_PENDING, &work->flags);
204 lockdep_irq_work_enter(work);
206 lockdep_irq_work_exit(work);
208 * Clear the BUSY bit and return to the free state if
209 * no-one else claimed it meanwhile.
211 flags &= ~IRQ_WORK_PENDING;
212 (void)atomic_cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY);
214 if ((IS_ENABLED(CONFIG_PREEMPT_RT) && !irq_work_is_hard(work)) ||
215 !arch_irq_work_has_interrupt())
216 rcuwait_wake_up(&work->irqwait);
219 static void irq_work_run_list(struct llist_head *list)
221 struct irq_work *work, *tmp;
222 struct llist_node *llnode;
225 * On PREEMPT_RT IRQ-work which is not marked as HARD will be processed
226 * in a per-CPU thread in preemptible context. Only the items which are
227 * marked as IRQ_WORK_HARD_IRQ will be processed in hardirq context.
229 BUG_ON(!irqs_disabled() && !IS_ENABLED(CONFIG_PREEMPT_RT));
231 if (llist_empty(list))
234 llnode = llist_del_all(list);
235 llist_for_each_entry_safe(work, tmp, llnode, llnode)
236 irq_work_single(work);
240 * hotplug calls this through:
241 * hotplug_cfd() -> flush_smp_call_function_queue()
243 void irq_work_run(void)
245 irq_work_run_list(this_cpu_ptr(&raised_list));
246 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
247 irq_work_run_list(this_cpu_ptr(&lazy_list));
251 EXPORT_SYMBOL_GPL(irq_work_run);
253 void irq_work_tick(void)
255 struct llist_head *raised = this_cpu_ptr(&raised_list);
257 if (!llist_empty(raised) && !arch_irq_work_has_interrupt())
258 irq_work_run_list(raised);
260 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
261 irq_work_run_list(this_cpu_ptr(&lazy_list));
267 * Synchronize against the irq_work @entry, ensures the entry is not
270 void irq_work_sync(struct irq_work *work)
272 lockdep_assert_irqs_enabled();
275 if ((IS_ENABLED(CONFIG_PREEMPT_RT) && !irq_work_is_hard(work)) ||
276 !arch_irq_work_has_interrupt()) {
277 rcuwait_wait_event(&work->irqwait, !irq_work_is_busy(work),
278 TASK_UNINTERRUPTIBLE);
282 while (atomic_read(&work->flags) & IRQ_WORK_BUSY)
285 EXPORT_SYMBOL_GPL(irq_work_sync);
287 static void run_irq_workd(unsigned int cpu)
289 irq_work_run_list(this_cpu_ptr(&lazy_list));
292 static void irq_workd_setup(unsigned int cpu)
294 sched_set_fifo_low(current);
297 static struct smp_hotplug_thread irqwork_threads = {
299 .setup = irq_workd_setup,
300 .thread_should_run = irq_workd_should_run,
301 .thread_fn = run_irq_workd,
302 .thread_comm = "irq_work/%u",
305 static __init int irq_work_init_threads(void)
307 if (IS_ENABLED(CONFIG_PREEMPT_RT))
308 BUG_ON(smpboot_register_percpu_thread(&irqwork_threads));
311 early_initcall(irq_work_init_threads);