Merge git://git.kernel.org/pub/scm/linux/kernel/git/steve/gfs2-3.0-fixes
[platform/adaptation/renesas_rcar/renesas_kernel.git] / kernel / softirq.c
1 /*
2  *      linux/kernel/softirq.c
3  *
4  *      Copyright (C) 1992 Linus Torvalds
5  *
6  *      Distribute under GPLv2.
7  *
8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9  *
10  *      Remote softirq infrastructure is by Jens Axboe.
11  */
12
13 #include <linux/export.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/ftrace.h>
25 #include <linux/smp.h>
26 #include <linux/smpboot.h>
27 #include <linux/tick.h>
28
29 #define CREATE_TRACE_POINTS
30 #include <trace/events/irq.h>
31
32 #include <asm/irq.h>
33 /*
34    - No shared variables, all the data are CPU local.
35    - If a softirq needs serialization, let it serialize itself
36      by its own spinlocks.
37    - Even if softirq is serialized, only local cpu is marked for
38      execution. Hence, we get something sort of weak cpu binding.
39      Though it is still not clear, will it result in better locality
40      or will not.
41
42    Examples:
43    - NET RX softirq. It is multithreaded and does not require
44      any global serialization.
45    - NET TX softirq. It kicks software netdevice queues, hence
46      it is logically serialized per device, but this serialization
47      is invisible to common code.
48    - Tasklets: serialized wrt itself.
49  */
50
51 #ifndef __ARCH_IRQ_STAT
52 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
53 EXPORT_SYMBOL(irq_stat);
54 #endif
55
56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
57
58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
59
60 char *softirq_to_name[NR_SOFTIRQS] = {
61         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
62         "TASKLET", "SCHED", "HRTIMER", "RCU"
63 };
64
65 /*
66  * we cannot loop indefinitely here to avoid userspace starvation,
67  * but we also don't want to introduce a worst case 1/HZ latency
68  * to the pending events, so lets the scheduler to balance
69  * the softirq load for us.
70  */
71 static void wakeup_softirqd(void)
72 {
73         /* Interrupts are disabled: no need to stop preemption */
74         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
75
76         if (tsk && tsk->state != TASK_RUNNING)
77                 wake_up_process(tsk);
78 }
79
80 /*
81  * preempt_count and SOFTIRQ_OFFSET usage:
82  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
83  *   softirq processing.
84  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
85  *   on local_bh_disable or local_bh_enable.
86  * This lets us distinguish between whether we are currently processing
87  * softirq and whether we just have bh disabled.
88  */
89
90 /*
91  * This one is for softirq.c-internal use,
92  * where hardirqs are disabled legitimately:
93  */
94 #ifdef CONFIG_TRACE_IRQFLAGS
95 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
96 {
97         unsigned long flags;
98
99         WARN_ON_ONCE(in_irq());
100
101         raw_local_irq_save(flags);
102         /*
103          * The preempt tracer hooks into add_preempt_count and will break
104          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
105          * is set and before current->softirq_enabled is cleared.
106          * We must manually increment preempt_count here and manually
107          * call the trace_preempt_off later.
108          */
109         preempt_count() += cnt;
110         /*
111          * Were softirqs turned off above:
112          */
113         if (softirq_count() == cnt)
114                 trace_softirqs_off(ip);
115         raw_local_irq_restore(flags);
116
117         if (preempt_count() == cnt)
118                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
119 }
120 #else /* !CONFIG_TRACE_IRQFLAGS */
121 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
122 {
123         add_preempt_count(cnt);
124         barrier();
125 }
126 #endif /* CONFIG_TRACE_IRQFLAGS */
127
128 void local_bh_disable(void)
129 {
130         __local_bh_disable(_RET_IP_, SOFTIRQ_DISABLE_OFFSET);
131 }
132
133 EXPORT_SYMBOL(local_bh_disable);
134
135 static void __local_bh_enable(unsigned int cnt)
136 {
137         WARN_ON_ONCE(in_irq());
138         WARN_ON_ONCE(!irqs_disabled());
139
140         if (softirq_count() == cnt)
141                 trace_softirqs_on(_RET_IP_);
142         sub_preempt_count(cnt);
143 }
144
145 /*
146  * Special-case - softirqs can safely be enabled in
147  * cond_resched_softirq(), or by __do_softirq(),
148  * without processing still-pending softirqs:
149  */
150 void _local_bh_enable(void)
151 {
152         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
153 }
154
155 EXPORT_SYMBOL(_local_bh_enable);
156
157 static inline void _local_bh_enable_ip(unsigned long ip)
158 {
159         WARN_ON_ONCE(in_irq() || irqs_disabled());
160 #ifdef CONFIG_TRACE_IRQFLAGS
161         local_irq_disable();
162 #endif
163         /*
164          * Are softirqs going to be turned on now:
165          */
166         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
167                 trace_softirqs_on(ip);
168         /*
169          * Keep preemption disabled until we are done with
170          * softirq processing:
171          */
172         sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
173
174         if (unlikely(!in_interrupt() && local_softirq_pending()))
175                 do_softirq();
176
177         dec_preempt_count();
178 #ifdef CONFIG_TRACE_IRQFLAGS
179         local_irq_enable();
180 #endif
181         preempt_check_resched();
182 }
183
184 void local_bh_enable(void)
185 {
186         _local_bh_enable_ip(_RET_IP_);
187 }
188 EXPORT_SYMBOL(local_bh_enable);
189
190 void local_bh_enable_ip(unsigned long ip)
191 {
192         _local_bh_enable_ip(ip);
193 }
194 EXPORT_SYMBOL(local_bh_enable_ip);
195
196 /*
197  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
198  * but break the loop if need_resched() is set or after 2 ms.
199  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
200  * certain cases, such as stop_machine(), jiffies may cease to
201  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
202  * well to make sure we eventually return from this method.
203  *
204  * These limits have been established via experimentation.
205  * The two things to balance is latency against fairness -
206  * we want to handle softirqs as soon as possible, but they
207  * should not be able to lock up the box.
208  */
209 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
210 #define MAX_SOFTIRQ_RESTART 10
211
212 asmlinkage void __do_softirq(void)
213 {
214         struct softirq_action *h;
215         __u32 pending;
216         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
217         int cpu;
218         unsigned long old_flags = current->flags;
219         int max_restart = MAX_SOFTIRQ_RESTART;
220
221         /*
222          * Mask out PF_MEMALLOC s current task context is borrowed for the
223          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
224          * again if the socket is related to swap
225          */
226         current->flags &= ~PF_MEMALLOC;
227
228         pending = local_softirq_pending();
229         account_irq_enter_time(current);
230
231         __local_bh_disable(_RET_IP_, SOFTIRQ_OFFSET);
232         lockdep_softirq_enter();
233
234         cpu = smp_processor_id();
235 restart:
236         /* Reset the pending bitmask before enabling irqs */
237         set_softirq_pending(0);
238
239         local_irq_enable();
240
241         h = softirq_vec;
242
243         do {
244                 if (pending & 1) {
245                         unsigned int vec_nr = h - softirq_vec;
246                         int prev_count = preempt_count();
247
248                         kstat_incr_softirqs_this_cpu(vec_nr);
249
250                         trace_softirq_entry(vec_nr);
251                         h->action(h);
252                         trace_softirq_exit(vec_nr);
253                         if (unlikely(prev_count != preempt_count())) {
254                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
255                                        "with preempt_count %08x,"
256                                        " exited with %08x?\n", vec_nr,
257                                        softirq_to_name[vec_nr], h->action,
258                                        prev_count, preempt_count());
259                                 preempt_count() = prev_count;
260                         }
261
262                         rcu_bh_qs(cpu);
263                 }
264                 h++;
265                 pending >>= 1;
266         } while (pending);
267
268         local_irq_disable();
269
270         pending = local_softirq_pending();
271         if (pending) {
272                 if (time_before(jiffies, end) && !need_resched() &&
273                     --max_restart)
274                         goto restart;
275
276                 wakeup_softirqd();
277         }
278
279         lockdep_softirq_exit();
280
281         account_irq_exit_time(current);
282         __local_bh_enable(SOFTIRQ_OFFSET);
283         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
284 }
285
286 #ifndef __ARCH_HAS_DO_SOFTIRQ
287
288 asmlinkage void do_softirq(void)
289 {
290         __u32 pending;
291         unsigned long flags;
292
293         if (in_interrupt())
294                 return;
295
296         local_irq_save(flags);
297
298         pending = local_softirq_pending();
299
300         if (pending)
301                 __do_softirq();
302
303         local_irq_restore(flags);
304 }
305
306 #endif
307
308 /*
309  * Enter an interrupt context.
310  */
311 void irq_enter(void)
312 {
313         int cpu = smp_processor_id();
314
315         rcu_irq_enter();
316         if (is_idle_task(current) && !in_interrupt()) {
317                 /*
318                  * Prevent raise_softirq from needlessly waking up ksoftirqd
319                  * here, as softirq will be serviced on return from interrupt.
320                  */
321                 local_bh_disable();
322                 tick_check_idle(cpu);
323                 _local_bh_enable();
324         }
325
326         __irq_enter();
327 }
328
329 static inline void invoke_softirq(void)
330 {
331         if (!force_irqthreads)
332                 __do_softirq();
333         else
334                 wakeup_softirqd();
335 }
336
337 static inline void tick_irq_exit(void)
338 {
339 #ifdef CONFIG_NO_HZ_COMMON
340         int cpu = smp_processor_id();
341
342         /* Make sure that timer wheel updates are propagated */
343         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
344                 if (!in_interrupt())
345                         tick_nohz_irq_exit();
346         }
347 #endif
348 }
349
350 /*
351  * Exit an interrupt context. Process softirqs if needed and possible:
352  */
353 void irq_exit(void)
354 {
355 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
356         local_irq_disable();
357 #else
358         WARN_ON_ONCE(!irqs_disabled());
359 #endif
360
361         account_irq_exit_time(current);
362         trace_hardirq_exit();
363         sub_preempt_count(HARDIRQ_OFFSET);
364         if (!in_interrupt() && local_softirq_pending())
365                 invoke_softirq();
366
367         tick_irq_exit();
368         rcu_irq_exit();
369 }
370
371 /*
372  * This function must run with irqs disabled!
373  */
374 inline void raise_softirq_irqoff(unsigned int nr)
375 {
376         __raise_softirq_irqoff(nr);
377
378         /*
379          * If we're in an interrupt or softirq, we're done
380          * (this also catches softirq-disabled code). We will
381          * actually run the softirq once we return from
382          * the irq or softirq.
383          *
384          * Otherwise we wake up ksoftirqd to make sure we
385          * schedule the softirq soon.
386          */
387         if (!in_interrupt())
388                 wakeup_softirqd();
389 }
390
391 void raise_softirq(unsigned int nr)
392 {
393         unsigned long flags;
394
395         local_irq_save(flags);
396         raise_softirq_irqoff(nr);
397         local_irq_restore(flags);
398 }
399
400 void __raise_softirq_irqoff(unsigned int nr)
401 {
402         trace_softirq_raise(nr);
403         or_softirq_pending(1UL << nr);
404 }
405
406 void open_softirq(int nr, void (*action)(struct softirq_action *))
407 {
408         softirq_vec[nr].action = action;
409 }
410
411 /*
412  * Tasklets
413  */
414 struct tasklet_head
415 {
416         struct tasklet_struct *head;
417         struct tasklet_struct **tail;
418 };
419
420 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
421 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
422
423 void __tasklet_schedule(struct tasklet_struct *t)
424 {
425         unsigned long flags;
426
427         local_irq_save(flags);
428         t->next = NULL;
429         *__this_cpu_read(tasklet_vec.tail) = t;
430         __this_cpu_write(tasklet_vec.tail, &(t->next));
431         raise_softirq_irqoff(TASKLET_SOFTIRQ);
432         local_irq_restore(flags);
433 }
434
435 EXPORT_SYMBOL(__tasklet_schedule);
436
437 void __tasklet_hi_schedule(struct tasklet_struct *t)
438 {
439         unsigned long flags;
440
441         local_irq_save(flags);
442         t->next = NULL;
443         *__this_cpu_read(tasklet_hi_vec.tail) = t;
444         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
445         raise_softirq_irqoff(HI_SOFTIRQ);
446         local_irq_restore(flags);
447 }
448
449 EXPORT_SYMBOL(__tasklet_hi_schedule);
450
451 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
452 {
453         BUG_ON(!irqs_disabled());
454
455         t->next = __this_cpu_read(tasklet_hi_vec.head);
456         __this_cpu_write(tasklet_hi_vec.head, t);
457         __raise_softirq_irqoff(HI_SOFTIRQ);
458 }
459
460 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
461
462 static void tasklet_action(struct softirq_action *a)
463 {
464         struct tasklet_struct *list;
465
466         local_irq_disable();
467         list = __this_cpu_read(tasklet_vec.head);
468         __this_cpu_write(tasklet_vec.head, NULL);
469         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
470         local_irq_enable();
471
472         while (list) {
473                 struct tasklet_struct *t = list;
474
475                 list = list->next;
476
477                 if (tasklet_trylock(t)) {
478                         if (!atomic_read(&t->count)) {
479                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
480                                         BUG();
481                                 t->func(t->data);
482                                 tasklet_unlock(t);
483                                 continue;
484                         }
485                         tasklet_unlock(t);
486                 }
487
488                 local_irq_disable();
489                 t->next = NULL;
490                 *__this_cpu_read(tasklet_vec.tail) = t;
491                 __this_cpu_write(tasklet_vec.tail, &(t->next));
492                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
493                 local_irq_enable();
494         }
495 }
496
497 static void tasklet_hi_action(struct softirq_action *a)
498 {
499         struct tasklet_struct *list;
500
501         local_irq_disable();
502         list = __this_cpu_read(tasklet_hi_vec.head);
503         __this_cpu_write(tasklet_hi_vec.head, NULL);
504         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
505         local_irq_enable();
506
507         while (list) {
508                 struct tasklet_struct *t = list;
509
510                 list = list->next;
511
512                 if (tasklet_trylock(t)) {
513                         if (!atomic_read(&t->count)) {
514                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
515                                         BUG();
516                                 t->func(t->data);
517                                 tasklet_unlock(t);
518                                 continue;
519                         }
520                         tasklet_unlock(t);
521                 }
522
523                 local_irq_disable();
524                 t->next = NULL;
525                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
526                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
527                 __raise_softirq_irqoff(HI_SOFTIRQ);
528                 local_irq_enable();
529         }
530 }
531
532
533 void tasklet_init(struct tasklet_struct *t,
534                   void (*func)(unsigned long), unsigned long data)
535 {
536         t->next = NULL;
537         t->state = 0;
538         atomic_set(&t->count, 0);
539         t->func = func;
540         t->data = data;
541 }
542
543 EXPORT_SYMBOL(tasklet_init);
544
545 void tasklet_kill(struct tasklet_struct *t)
546 {
547         if (in_interrupt())
548                 printk("Attempt to kill tasklet from interrupt\n");
549
550         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
551                 do {
552                         yield();
553                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
554         }
555         tasklet_unlock_wait(t);
556         clear_bit(TASKLET_STATE_SCHED, &t->state);
557 }
558
559 EXPORT_SYMBOL(tasklet_kill);
560
561 /*
562  * tasklet_hrtimer
563  */
564
565 /*
566  * The trampoline is called when the hrtimer expires. It schedules a tasklet
567  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
568  * hrtimer callback, but from softirq context.
569  */
570 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
571 {
572         struct tasklet_hrtimer *ttimer =
573                 container_of(timer, struct tasklet_hrtimer, timer);
574
575         tasklet_hi_schedule(&ttimer->tasklet);
576         return HRTIMER_NORESTART;
577 }
578
579 /*
580  * Helper function which calls the hrtimer callback from
581  * tasklet/softirq context
582  */
583 static void __tasklet_hrtimer_trampoline(unsigned long data)
584 {
585         struct tasklet_hrtimer *ttimer = (void *)data;
586         enum hrtimer_restart restart;
587
588         restart = ttimer->function(&ttimer->timer);
589         if (restart != HRTIMER_NORESTART)
590                 hrtimer_restart(&ttimer->timer);
591 }
592
593 /**
594  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
595  * @ttimer:      tasklet_hrtimer which is initialized
596  * @function:    hrtimer callback function which gets called from softirq context
597  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
598  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
599  */
600 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
601                           enum hrtimer_restart (*function)(struct hrtimer *),
602                           clockid_t which_clock, enum hrtimer_mode mode)
603 {
604         hrtimer_init(&ttimer->timer, which_clock, mode);
605         ttimer->timer.function = __hrtimer_tasklet_trampoline;
606         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
607                      (unsigned long)ttimer);
608         ttimer->function = function;
609 }
610 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
611
612 /*
613  * Remote softirq bits
614  */
615
616 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
617 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
618
619 static void __local_trigger(struct call_single_data *cp, int softirq)
620 {
621         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
622
623         list_add_tail(&cp->list, head);
624
625         /* Trigger the softirq only if the list was previously empty.  */
626         if (head->next == &cp->list)
627                 raise_softirq_irqoff(softirq);
628 }
629
630 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
631 static void remote_softirq_receive(void *data)
632 {
633         struct call_single_data *cp = data;
634         unsigned long flags;
635         int softirq;
636
637         softirq = *(int *)cp->info;
638         local_irq_save(flags);
639         __local_trigger(cp, softirq);
640         local_irq_restore(flags);
641 }
642
643 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
644 {
645         if (cpu_online(cpu)) {
646                 cp->func = remote_softirq_receive;
647                 cp->info = &softirq;
648                 cp->flags = 0;
649
650                 __smp_call_function_single(cpu, cp, 0);
651                 return 0;
652         }
653         return 1;
654 }
655 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
656 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
657 {
658         return 1;
659 }
660 #endif
661
662 /**
663  * __send_remote_softirq - try to schedule softirq work on a remote cpu
664  * @cp: private SMP call function data area
665  * @cpu: the remote cpu
666  * @this_cpu: the currently executing cpu
667  * @softirq: the softirq for the work
668  *
669  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
670  * done, the work is instead queued up on the local cpu.
671  *
672  * Interrupts must be disabled.
673  */
674 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
675 {
676         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
677                 __local_trigger(cp, softirq);
678 }
679 EXPORT_SYMBOL(__send_remote_softirq);
680
681 /**
682  * send_remote_softirq - try to schedule softirq work on a remote cpu
683  * @cp: private SMP call function data area
684  * @cpu: the remote cpu
685  * @softirq: the softirq for the work
686  *
687  * Like __send_remote_softirq except that disabling interrupts and
688  * computing the current cpu is done for the caller.
689  */
690 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
691 {
692         unsigned long flags;
693         int this_cpu;
694
695         local_irq_save(flags);
696         this_cpu = smp_processor_id();
697         __send_remote_softirq(cp, cpu, this_cpu, softirq);
698         local_irq_restore(flags);
699 }
700 EXPORT_SYMBOL(send_remote_softirq);
701
702 static int remote_softirq_cpu_notify(struct notifier_block *self,
703                                                unsigned long action, void *hcpu)
704 {
705         /*
706          * If a CPU goes away, splice its entries to the current CPU
707          * and trigger a run of the softirq
708          */
709         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
710                 int cpu = (unsigned long) hcpu;
711                 int i;
712
713                 local_irq_disable();
714                 for (i = 0; i < NR_SOFTIRQS; i++) {
715                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
716                         struct list_head *local_head;
717
718                         if (list_empty(head))
719                                 continue;
720
721                         local_head = &__get_cpu_var(softirq_work_list[i]);
722                         list_splice_init(head, local_head);
723                         raise_softirq_irqoff(i);
724                 }
725                 local_irq_enable();
726         }
727
728         return NOTIFY_OK;
729 }
730
731 static struct notifier_block remote_softirq_cpu_notifier = {
732         .notifier_call  = remote_softirq_cpu_notify,
733 };
734
735 void __init softirq_init(void)
736 {
737         int cpu;
738
739         for_each_possible_cpu(cpu) {
740                 int i;
741
742                 per_cpu(tasklet_vec, cpu).tail =
743                         &per_cpu(tasklet_vec, cpu).head;
744                 per_cpu(tasklet_hi_vec, cpu).tail =
745                         &per_cpu(tasklet_hi_vec, cpu).head;
746                 for (i = 0; i < NR_SOFTIRQS; i++)
747                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
748         }
749
750         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
751
752         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
753         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
754 }
755
756 static int ksoftirqd_should_run(unsigned int cpu)
757 {
758         return local_softirq_pending();
759 }
760
761 static void run_ksoftirqd(unsigned int cpu)
762 {
763         local_irq_disable();
764         if (local_softirq_pending()) {
765                 __do_softirq();
766                 rcu_note_context_switch(cpu);
767                 local_irq_enable();
768                 cond_resched();
769                 return;
770         }
771         local_irq_enable();
772 }
773
774 #ifdef CONFIG_HOTPLUG_CPU
775 /*
776  * tasklet_kill_immediate is called to remove a tasklet which can already be
777  * scheduled for execution on @cpu.
778  *
779  * Unlike tasklet_kill, this function removes the tasklet
780  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
781  *
782  * When this function is called, @cpu must be in the CPU_DEAD state.
783  */
784 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
785 {
786         struct tasklet_struct **i;
787
788         BUG_ON(cpu_online(cpu));
789         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
790
791         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
792                 return;
793
794         /* CPU is dead, so no lock needed. */
795         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
796                 if (*i == t) {
797                         *i = t->next;
798                         /* If this was the tail element, move the tail ptr */
799                         if (*i == NULL)
800                                 per_cpu(tasklet_vec, cpu).tail = i;
801                         return;
802                 }
803         }
804         BUG();
805 }
806
807 static void takeover_tasklets(unsigned int cpu)
808 {
809         /* CPU is dead, so no lock needed. */
810         local_irq_disable();
811
812         /* Find end, append list for that CPU. */
813         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
814                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
815                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
816                 per_cpu(tasklet_vec, cpu).head = NULL;
817                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
818         }
819         raise_softirq_irqoff(TASKLET_SOFTIRQ);
820
821         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
822                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
823                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
824                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
825                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
826         }
827         raise_softirq_irqoff(HI_SOFTIRQ);
828
829         local_irq_enable();
830 }
831 #endif /* CONFIG_HOTPLUG_CPU */
832
833 static int cpu_callback(struct notifier_block *nfb,
834                                   unsigned long action,
835                                   void *hcpu)
836 {
837         switch (action) {
838 #ifdef CONFIG_HOTPLUG_CPU
839         case CPU_DEAD:
840         case CPU_DEAD_FROZEN:
841                 takeover_tasklets((unsigned long)hcpu);
842                 break;
843 #endif /* CONFIG_HOTPLUG_CPU */
844         }
845         return NOTIFY_OK;
846 }
847
848 static struct notifier_block cpu_nfb = {
849         .notifier_call = cpu_callback
850 };
851
852 static struct smp_hotplug_thread softirq_threads = {
853         .store                  = &ksoftirqd,
854         .thread_should_run      = ksoftirqd_should_run,
855         .thread_fn              = run_ksoftirqd,
856         .thread_comm            = "ksoftirqd/%u",
857 };
858
859 static __init int spawn_ksoftirqd(void)
860 {
861         register_cpu_notifier(&cpu_nfb);
862
863         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
864
865         return 0;
866 }
867 early_initcall(spawn_ksoftirqd);
868
869 /*
870  * [ These __weak aliases are kept in a separate compilation unit, so that
871  *   GCC does not inline them incorrectly. ]
872  */
873
874 int __init __weak early_irq_init(void)
875 {
876         return 0;
877 }
878
879 #ifdef CONFIG_GENERIC_HARDIRQS
880 int __init __weak arch_probe_nr_irqs(void)
881 {
882         return NR_IRQS_LEGACY;
883 }
884
885 int __init __weak arch_early_irq_init(void)
886 {
887         return 0;
888 }
889 #endif