1 // SPDX-License-Identifier: GPL-2.0-only
3 * linux/kernel/softirq.c
5 * Copyright (C) 1992 Linus Torvalds
7 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 #include <linux/export.h>
13 #include <linux/kernel_stat.h>
14 #include <linux/interrupt.h>
15 #include <linux/init.h>
16 #include <linux/local_lock.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/ftrace.h>
25 #include <linux/smp.h>
26 #include <linux/smpboot.h>
27 #include <linux/tick.h>
28 #include <linux/irq.h>
29 #include <linux/wait_bit.h>
31 #define CREATE_TRACE_POINTS
32 #include <trace/events/irq.h>
35 - No shared variables, all the data are CPU local.
36 - If a softirq needs serialization, let it serialize itself
38 - Even if softirq is serialized, only local cpu is marked for
39 execution. Hence, we get something sort of weak cpu binding.
40 Though it is still not clear, will it result in better locality
44 - NET RX softirq. It is multithreaded and does not require
45 any global serialization.
46 - NET TX softirq. It kicks software netdevice queues, hence
47 it is logically serialized per device, but this serialization
48 is invisible to common code.
49 - Tasklets: serialized wrt itself.
52 #ifndef __ARCH_IRQ_STAT
53 DEFINE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
54 EXPORT_PER_CPU_SYMBOL(irq_stat);
57 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
59 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
61 const char * const softirq_to_name[NR_SOFTIRQS] = {
62 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
63 "TASKLET", "SCHED", "HRTIMER", "RCU"
67 * we cannot loop indefinitely here to avoid userspace starvation,
68 * but we also don't want to introduce a worst case 1/HZ latency
69 * to the pending events, so lets the scheduler to balance
70 * the softirq load for us.
72 static void wakeup_softirqd(void)
74 /* Interrupts are disabled: no need to stop preemption */
75 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
77 if (tsk && tsk->state != TASK_RUNNING)
82 * If ksoftirqd is scheduled, we do not want to process pending softirqs
83 * right now. Let ksoftirqd handle this at its own rate, to get fairness,
84 * unless we're doing some of the synchronous softirqs.
86 #define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ))
87 static bool ksoftirqd_running(unsigned long pending)
89 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
91 if (pending & SOFTIRQ_NOW_MASK)
93 return tsk && (tsk->state == TASK_RUNNING) &&
94 !__kthread_should_park(tsk);
97 #ifdef CONFIG_TRACE_IRQFLAGS
98 DEFINE_PER_CPU(int, hardirqs_enabled);
99 DEFINE_PER_CPU(int, hardirq_context);
100 EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
101 EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
105 * SOFTIRQ_OFFSET usage:
107 * On !RT kernels 'count' is the preempt counter, on RT kernels this applies
108 * to a per CPU counter and to task::softirqs_disabled_cnt.
110 * - count is changed by SOFTIRQ_OFFSET on entering or leaving softirq
113 * - count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
114 * on local_bh_disable or local_bh_enable.
116 * This lets us distinguish between whether we are currently processing
117 * softirq and whether we just have bh disabled.
119 #ifdef CONFIG_PREEMPT_RT
122 * RT accounts for BH disabled sections in task::softirqs_disabled_cnt and
123 * also in per CPU softirq_ctrl::cnt. This is necessary to allow tasks in a
124 * softirq disabled section to be preempted.
126 * The per task counter is used for softirq_count(), in_softirq() and
127 * in_serving_softirqs() because these counts are only valid when the task
128 * holding softirq_ctrl::lock is running.
130 * The per CPU counter prevents pointless wakeups of ksoftirqd in case that
131 * the task which is in a softirq disabled section is preempted or blocks.
133 struct softirq_ctrl {
138 static DEFINE_PER_CPU(struct softirq_ctrl, softirq_ctrl) = {
139 .lock = INIT_LOCAL_LOCK(softirq_ctrl.lock),
143 * local_bh_blocked() - Check for idle whether BH processing is blocked
145 * Returns false if the per CPU softirq::cnt is 0 otherwise true.
147 * This is invoked from the idle task to guard against false positive
148 * softirq pending warnings, which would happen when the task which holds
149 * softirq_ctrl::lock was the only running task on the CPU and blocks on
152 bool local_bh_blocked(void)
154 return __this_cpu_read(softirq_ctrl.cnt) != 0;
157 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
162 WARN_ON_ONCE(in_hardirq());
164 /* First entry of a task into a BH disabled section? */
165 if (!current->softirq_disable_cnt) {
167 local_lock(&softirq_ctrl.lock);
168 /* Required to meet the RCU bottomhalf requirements. */
171 DEBUG_LOCKS_WARN_ON(this_cpu_read(softirq_ctrl.cnt));
176 * Track the per CPU softirq disabled state. On RT this is per CPU
177 * state to allow preemption of bottom half disabled sections.
179 newcnt = __this_cpu_add_return(softirq_ctrl.cnt, cnt);
181 * Reflect the result in the task state to prevent recursion on the
182 * local lock and to make softirq_count() & al work.
184 current->softirq_disable_cnt = newcnt;
186 if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && newcnt == cnt) {
187 raw_local_irq_save(flags);
188 lockdep_softirqs_off(ip);
189 raw_local_irq_restore(flags);
192 EXPORT_SYMBOL(__local_bh_disable_ip);
194 static void __local_bh_enable(unsigned int cnt, bool unlock)
199 DEBUG_LOCKS_WARN_ON(current->softirq_disable_cnt !=
200 this_cpu_read(softirq_ctrl.cnt));
202 if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && softirq_count() == cnt) {
203 raw_local_irq_save(flags);
204 lockdep_softirqs_on(_RET_IP_);
205 raw_local_irq_restore(flags);
208 newcnt = __this_cpu_sub_return(softirq_ctrl.cnt, cnt);
209 current->softirq_disable_cnt = newcnt;
211 if (!newcnt && unlock) {
213 local_unlock(&softirq_ctrl.lock);
217 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
219 bool preempt_on = preemptible();
224 WARN_ON_ONCE(in_irq());
225 lockdep_assert_irqs_enabled();
227 local_irq_save(flags);
228 curcnt = __this_cpu_read(softirq_ctrl.cnt);
231 * If this is not reenabling soft interrupts, no point in trying to
237 pending = local_softirq_pending();
238 if (!pending || ksoftirqd_running(pending))
242 * If this was called from non preemptible context, wake up the
251 * Adjust softirq count to SOFTIRQ_OFFSET which makes
252 * in_serving_softirq() become true.
254 cnt = SOFTIRQ_OFFSET;
255 __local_bh_enable(cnt, false);
259 __local_bh_enable(cnt, preempt_on);
260 local_irq_restore(flags);
262 EXPORT_SYMBOL(__local_bh_enable_ip);
265 * Invoked from ksoftirqd_run() outside of the interrupt disabled section
266 * to acquire the per CPU local lock for reentrancy protection.
268 static inline void ksoftirqd_run_begin(void)
270 __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
274 /* Counterpart to ksoftirqd_run_begin() */
275 static inline void ksoftirqd_run_end(void)
277 __local_bh_enable(SOFTIRQ_OFFSET, true);
278 WARN_ON_ONCE(in_interrupt());
282 static inline void softirq_handle_begin(void) { }
283 static inline void softirq_handle_end(void) { }
285 static inline bool should_wake_ksoftirqd(void)
287 return !this_cpu_read(softirq_ctrl.cnt);
290 static inline void invoke_softirq(void)
292 if (should_wake_ksoftirqd())
296 #else /* CONFIG_PREEMPT_RT */
299 * This one is for softirq.c-internal use, where hardirqs are disabled
302 #ifdef CONFIG_TRACE_IRQFLAGS
303 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
307 WARN_ON_ONCE(in_irq());
309 raw_local_irq_save(flags);
311 * The preempt tracer hooks into preempt_count_add and will break
312 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
313 * is set and before current->softirq_enabled is cleared.
314 * We must manually increment preempt_count here and manually
315 * call the trace_preempt_off later.
317 __preempt_count_add(cnt);
319 * Were softirqs turned off above:
321 if (softirq_count() == (cnt & SOFTIRQ_MASK))
322 lockdep_softirqs_off(ip);
323 raw_local_irq_restore(flags);
325 if (preempt_count() == cnt) {
326 #ifdef CONFIG_DEBUG_PREEMPT
327 current->preempt_disable_ip = get_lock_parent_ip();
329 trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip());
332 EXPORT_SYMBOL(__local_bh_disable_ip);
333 #endif /* CONFIG_TRACE_IRQFLAGS */
335 static void __local_bh_enable(unsigned int cnt)
337 lockdep_assert_irqs_disabled();
339 if (preempt_count() == cnt)
340 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
342 if (softirq_count() == (cnt & SOFTIRQ_MASK))
343 lockdep_softirqs_on(_RET_IP_);
345 __preempt_count_sub(cnt);
349 * Special-case - softirqs can safely be enabled by __do_softirq(),
350 * without processing still-pending softirqs:
352 void _local_bh_enable(void)
354 WARN_ON_ONCE(in_irq());
355 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
357 EXPORT_SYMBOL(_local_bh_enable);
359 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
361 WARN_ON_ONCE(in_irq());
362 lockdep_assert_irqs_enabled();
363 #ifdef CONFIG_TRACE_IRQFLAGS
367 * Are softirqs going to be turned on now:
369 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
370 lockdep_softirqs_on(ip);
372 * Keep preemption disabled until we are done with
373 * softirq processing:
375 preempt_count_sub(cnt - 1);
377 if (unlikely(!in_interrupt() && local_softirq_pending())) {
379 * Run softirq if any pending. And do it in its own stack
380 * as we may be calling this deep in a task call stack already.
386 #ifdef CONFIG_TRACE_IRQFLAGS
389 preempt_check_resched();
391 EXPORT_SYMBOL(__local_bh_enable_ip);
393 static inline void softirq_handle_begin(void)
395 __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
398 static inline void softirq_handle_end(void)
400 __local_bh_enable(SOFTIRQ_OFFSET);
401 WARN_ON_ONCE(in_interrupt());
404 static inline void ksoftirqd_run_begin(void)
409 static inline void ksoftirqd_run_end(void)
414 static inline bool should_wake_ksoftirqd(void)
419 static inline void invoke_softirq(void)
421 if (ksoftirqd_running(local_softirq_pending()))
424 if (!force_irqthreads) {
425 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
427 * We can safely execute softirq on the current stack if
428 * it is the irq stack, because it should be near empty
434 * Otherwise, irq_exit() is called on the task stack that can
435 * be potentially deep already. So call softirq in its own stack
436 * to prevent from any overrun.
438 do_softirq_own_stack();
445 asmlinkage __visible void do_softirq(void)
453 local_irq_save(flags);
455 pending = local_softirq_pending();
457 if (pending && !ksoftirqd_running(pending))
458 do_softirq_own_stack();
460 local_irq_restore(flags);
463 #endif /* !CONFIG_PREEMPT_RT */
466 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
467 * but break the loop if need_resched() is set or after 2 ms.
468 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
469 * certain cases, such as stop_machine(), jiffies may cease to
470 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
471 * well to make sure we eventually return from this method.
473 * These limits have been established via experimentation.
474 * The two things to balance is latency against fairness -
475 * we want to handle softirqs as soon as possible, but they
476 * should not be able to lock up the box.
478 #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
479 #define MAX_SOFTIRQ_RESTART 10
481 #ifdef CONFIG_TRACE_IRQFLAGS
483 * When we run softirqs from irq_exit() and thus on the hardirq stack we need
484 * to keep the lockdep irq context tracking as tight as possible in order to
485 * not miss-qualify lock contexts and miss possible deadlocks.
488 static inline bool lockdep_softirq_start(void)
490 bool in_hardirq = false;
492 if (lockdep_hardirq_context()) {
494 lockdep_hardirq_exit();
497 lockdep_softirq_enter();
502 static inline void lockdep_softirq_end(bool in_hardirq)
504 lockdep_softirq_exit();
507 lockdep_hardirq_enter();
510 static inline bool lockdep_softirq_start(void) { return false; }
511 static inline void lockdep_softirq_end(bool in_hardirq) { }
514 asmlinkage __visible void __softirq_entry __do_softirq(void)
516 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
517 unsigned long old_flags = current->flags;
518 int max_restart = MAX_SOFTIRQ_RESTART;
519 struct softirq_action *h;
525 * Mask out PF_MEMALLOC as the current task context is borrowed for the
526 * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
527 * again if the socket is related to swapping.
529 current->flags &= ~PF_MEMALLOC;
531 pending = local_softirq_pending();
533 softirq_handle_begin();
534 in_hardirq = lockdep_softirq_start();
535 account_softirq_enter(current);
538 /* Reset the pending bitmask before enabling irqs */
539 set_softirq_pending(0);
545 while ((softirq_bit = ffs(pending))) {
549 h += softirq_bit - 1;
551 vec_nr = h - softirq_vec;
552 prev_count = preempt_count();
554 kstat_incr_softirqs_this_cpu(vec_nr);
556 trace_softirq_entry(vec_nr);
558 trace_softirq_exit(vec_nr);
559 if (unlikely(prev_count != preempt_count())) {
560 pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
561 vec_nr, softirq_to_name[vec_nr], h->action,
562 prev_count, preempt_count());
563 preempt_count_set(prev_count);
566 pending >>= softirq_bit;
569 if (!IS_ENABLED(CONFIG_PREEMPT_RT) &&
570 __this_cpu_read(ksoftirqd) == current)
575 pending = local_softirq_pending();
577 if (time_before(jiffies, end) && !need_resched() &&
584 account_softirq_exit(current);
585 lockdep_softirq_end(in_hardirq);
586 softirq_handle_end();
587 current_restore_flags(old_flags, PF_MEMALLOC);
591 * irq_enter_rcu - Enter an interrupt context with RCU watching
593 void irq_enter_rcu(void)
597 if (is_idle_task(current) && (irq_count() == HARDIRQ_OFFSET))
600 account_hardirq_enter(current);
604 * irq_enter - Enter an interrupt context including RCU update
612 static inline void tick_irq_exit(void)
614 #ifdef CONFIG_NO_HZ_COMMON
615 int cpu = smp_processor_id();
617 /* Make sure that timer wheel updates are propagated */
618 if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
620 tick_nohz_irq_exit();
625 static inline void __irq_exit_rcu(void)
627 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
630 lockdep_assert_irqs_disabled();
632 account_hardirq_exit(current);
633 preempt_count_sub(HARDIRQ_OFFSET);
634 if (!in_interrupt() && local_softirq_pending())
641 * irq_exit_rcu() - Exit an interrupt context without updating RCU
643 * Also processes softirqs if needed and possible.
645 void irq_exit_rcu(void)
649 lockdep_hardirq_exit();
653 * irq_exit - Exit an interrupt context, update RCU and lockdep
655 * Also processes softirqs if needed and possible.
662 lockdep_hardirq_exit();
666 * This function must run with irqs disabled!
668 inline void raise_softirq_irqoff(unsigned int nr)
670 __raise_softirq_irqoff(nr);
673 * If we're in an interrupt or softirq, we're done
674 * (this also catches softirq-disabled code). We will
675 * actually run the softirq once we return from
676 * the irq or softirq.
678 * Otherwise we wake up ksoftirqd to make sure we
679 * schedule the softirq soon.
681 if (!in_interrupt() && should_wake_ksoftirqd())
685 void raise_softirq(unsigned int nr)
689 local_irq_save(flags);
690 raise_softirq_irqoff(nr);
691 local_irq_restore(flags);
694 void __raise_softirq_irqoff(unsigned int nr)
696 lockdep_assert_irqs_disabled();
697 trace_softirq_raise(nr);
698 or_softirq_pending(1UL << nr);
701 void open_softirq(int nr, void (*action)(struct softirq_action *))
703 softirq_vec[nr].action = action;
709 struct tasklet_head {
710 struct tasklet_struct *head;
711 struct tasklet_struct **tail;
714 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
715 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
717 static void __tasklet_schedule_common(struct tasklet_struct *t,
718 struct tasklet_head __percpu *headp,
719 unsigned int softirq_nr)
721 struct tasklet_head *head;
724 local_irq_save(flags);
725 head = this_cpu_ptr(headp);
728 head->tail = &(t->next);
729 raise_softirq_irqoff(softirq_nr);
730 local_irq_restore(flags);
733 void __tasklet_schedule(struct tasklet_struct *t)
735 __tasklet_schedule_common(t, &tasklet_vec,
738 EXPORT_SYMBOL(__tasklet_schedule);
740 void __tasklet_hi_schedule(struct tasklet_struct *t)
742 __tasklet_schedule_common(t, &tasklet_hi_vec,
745 EXPORT_SYMBOL(__tasklet_hi_schedule);
747 static inline bool tasklet_clear_sched(struct tasklet_struct *t)
749 if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) {
750 wake_up_var(&t->state);
757 static void tasklet_action_common(struct softirq_action *a,
758 struct tasklet_head *tl_head,
759 unsigned int softirq_nr)
761 struct tasklet_struct *list;
764 list = tl_head->head;
765 tl_head->head = NULL;
766 tl_head->tail = &tl_head->head;
770 struct tasklet_struct *t = list;
774 if (tasklet_trylock(t)) {
775 if (!atomic_read(&t->count)) {
776 if (!tasklet_clear_sched(t))
791 tl_head->tail = &t->next;
792 __raise_softirq_irqoff(softirq_nr);
797 static __latent_entropy void tasklet_action(struct softirq_action *a)
799 tasklet_action_common(a, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
802 static __latent_entropy void tasklet_hi_action(struct softirq_action *a)
804 tasklet_action_common(a, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
807 void tasklet_setup(struct tasklet_struct *t,
808 void (*callback)(struct tasklet_struct *))
812 atomic_set(&t->count, 0);
813 t->callback = callback;
814 t->use_callback = true;
817 EXPORT_SYMBOL(tasklet_setup);
819 void tasklet_init(struct tasklet_struct *t,
820 void (*func)(unsigned long), unsigned long data)
824 atomic_set(&t->count, 0);
826 t->use_callback = false;
829 EXPORT_SYMBOL(tasklet_init);
831 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
833 * Do not use in new code. Waiting for tasklets from atomic contexts is
834 * error prone and should be avoided.
836 void tasklet_unlock_spin_wait(struct tasklet_struct *t)
838 while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
839 if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
841 * Prevent a live lock when current preempted soft
842 * interrupt processing or prevents ksoftirqd from
843 * running. If the tasklet runs on a different CPU
844 * then this has no effect other than doing the BH
845 * disable/enable dance for nothing.
854 EXPORT_SYMBOL(tasklet_unlock_spin_wait);
857 void tasklet_kill(struct tasklet_struct *t)
860 pr_notice("Attempt to kill tasklet from interrupt\n");
862 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
863 wait_var_event(&t->state, !test_bit(TASKLET_STATE_SCHED, &t->state));
865 tasklet_unlock_wait(t);
866 tasklet_clear_sched(t);
868 EXPORT_SYMBOL(tasklet_kill);
870 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
871 void tasklet_unlock(struct tasklet_struct *t)
873 smp_mb__before_atomic();
874 clear_bit(TASKLET_STATE_RUN, &t->state);
875 smp_mb__after_atomic();
876 wake_up_var(&t->state);
878 EXPORT_SYMBOL_GPL(tasklet_unlock);
880 void tasklet_unlock_wait(struct tasklet_struct *t)
882 wait_var_event(&t->state, !test_bit(TASKLET_STATE_RUN, &t->state));
884 EXPORT_SYMBOL_GPL(tasklet_unlock_wait);
887 void __init softirq_init(void)
891 for_each_possible_cpu(cpu) {
892 per_cpu(tasklet_vec, cpu).tail =
893 &per_cpu(tasklet_vec, cpu).head;
894 per_cpu(tasklet_hi_vec, cpu).tail =
895 &per_cpu(tasklet_hi_vec, cpu).head;
898 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
899 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
902 static int ksoftirqd_should_run(unsigned int cpu)
904 return local_softirq_pending();
907 static void run_ksoftirqd(unsigned int cpu)
909 ksoftirqd_run_begin();
910 if (local_softirq_pending()) {
912 * We can safely run softirq on inline stack, as we are not deep
913 * in the task stack here.
923 #ifdef CONFIG_HOTPLUG_CPU
925 * tasklet_kill_immediate is called to remove a tasklet which can already be
926 * scheduled for execution on @cpu.
928 * Unlike tasklet_kill, this function removes the tasklet
929 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
931 * When this function is called, @cpu must be in the CPU_DEAD state.
933 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
935 struct tasklet_struct **i;
937 BUG_ON(cpu_online(cpu));
938 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
940 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
943 /* CPU is dead, so no lock needed. */
944 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
947 /* If this was the tail element, move the tail ptr */
949 per_cpu(tasklet_vec, cpu).tail = i;
956 static int takeover_tasklets(unsigned int cpu)
958 /* CPU is dead, so no lock needed. */
961 /* Find end, append list for that CPU. */
962 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
963 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
964 __this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
965 per_cpu(tasklet_vec, cpu).head = NULL;
966 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
968 raise_softirq_irqoff(TASKLET_SOFTIRQ);
970 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
971 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
972 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
973 per_cpu(tasklet_hi_vec, cpu).head = NULL;
974 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
976 raise_softirq_irqoff(HI_SOFTIRQ);
982 #define takeover_tasklets NULL
983 #endif /* CONFIG_HOTPLUG_CPU */
985 static struct smp_hotplug_thread softirq_threads = {
987 .thread_should_run = ksoftirqd_should_run,
988 .thread_fn = run_ksoftirqd,
989 .thread_comm = "ksoftirqd/%u",
992 static __init int spawn_ksoftirqd(void)
994 cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
996 BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
1000 early_initcall(spawn_ksoftirqd);
1003 * [ These __weak aliases are kept in a separate compilation unit, so that
1004 * GCC does not inline them incorrectly. ]
1007 int __init __weak early_irq_init(void)
1012 int __init __weak arch_probe_nr_irqs(void)
1014 return NR_IRQS_LEGACY;
1017 int __init __weak arch_early_irq_init(void)
1022 unsigned int __weak arch_dynirq_lower_bound(unsigned int from)