kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/export.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/tick.h>
  27
  28 #define CREATE_TRACE_POINTS
  29 #include <trace/events/irq.h>
  30
  31 #include <asm/irq.h>
  32 /*
  33    - No shared variables, all the data are CPU local.
  34    - If a softirq needs serialization, let it serialize itself
  35      by its own spinlocks.
  36    - Even if softirq is serialized, only local cpu is marked for
  37      execution. Hence, we get something sort of weak cpu binding.
  38      Though it is still not clear, will it result in better locality
  39      or will not.
  40
  41    Examples:
  42    - NET RX softirq. It is multithreaded and does not require
  43      any global serialization.
  44    - NET TX softirq. It kicks software netdevice queues, hence
  45      it is logically serialized per device, but this serialization
  46      is invisible to common code.
  47    - Tasklets: serialized wrt itself.
  48  */
  49
  50 #ifndef __ARCH_IRQ_STAT
  51 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  52 EXPORT_SYMBOL(irq_stat);
  53 #endif
  54
  55 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  56
  57 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  58
  59 char *softirq_to_name[NR_SOFTIRQS] = {
  60         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  61         "TASKLET", "SCHED", "HRTIMER", "RCU"
  62 };
  63
  64 /*
  65  * we cannot loop indefinitely here to avoid userspace starvation,
  66  * but we also don't want to introduce a worst case 1/HZ latency
  67  * to the pending events, so lets the scheduler to balance
  68  * the softirq load for us.
  69  */
  70 static void wakeup_softirqd(void)
  71 {
  72         /* Interrupts are disabled: no need to stop preemption */
  73         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  74
  75         if (tsk && tsk->state != TASK_RUNNING)
  76                 wake_up_process(tsk);
  77 }
  78
  79 /*
  80  * preempt_count and SOFTIRQ_OFFSET usage:
  81  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  82  *   softirq processing.
  83  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  84  *   on local_bh_disable or local_bh_enable.
  85  * This lets us distinguish between whether we are currently processing
  86  * softirq and whether we just have bh disabled.
  87  */
  88
  89 /*
  90  * This one is for softirq.c-internal use,
  91  * where hardirqs are disabled legitimately:
  92  */
  93 #ifdef CONFIG_TRACE_IRQFLAGS
  94 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  95 {
  96         unsigned long flags;
  97
  98         WARN_ON_ONCE(in_irq());
  99
 100         raw_local_irq_save(flags);
 101         /*
 102          * The preempt tracer hooks into add_preempt_count and will break
 103          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 104          * is set and before current->softirq_enabled is cleared.
 105          * We must manually increment preempt_count here and manually
 106          * call the trace_preempt_off later.
 107          */
 108         preempt_count() += cnt;
 109         /*
 110          * Were softirqs turned off above:
 111          */
 112         if (softirq_count() == cnt)
 113                 trace_softirqs_off(ip);
 114         raw_local_irq_restore(flags);
 115
 116         if (preempt_count() == cnt)
 117                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 118 }
 119 #else /* !CONFIG_TRACE_IRQFLAGS */
 120 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 121 {
 122         add_preempt_count(cnt);
 123         barrier();
 124 }
 125 #endif /* CONFIG_TRACE_IRQFLAGS */
 126
 127 void local_bh_disable(void)
 128 {
 129         __local_bh_disable((unsigned long)__builtin_return_address(0),
 130                                 SOFTIRQ_DISABLE_OFFSET);
 131 }
 132
 133 EXPORT_SYMBOL(local_bh_disable);
 134
 135 static void __local_bh_enable(unsigned int cnt)
 136 {
 137         WARN_ON_ONCE(in_irq());
 138         WARN_ON_ONCE(!irqs_disabled());
 139
 140         if (softirq_count() == cnt)
 141                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 142         sub_preempt_count(cnt);
 143 }
 144
 145 /*
 146  * Special-case - softirqs can safely be enabled in
 147  * cond_resched_softirq(), or by __do_softirq(),
 148  * without processing still-pending softirqs:
 149  */
 150 void _local_bh_enable(void)
 151 {
 152         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 153 }
 154
 155 EXPORT_SYMBOL(_local_bh_enable);
 156
 157 static inline void _local_bh_enable_ip(unsigned long ip)
 158 {
 159         WARN_ON_ONCE(in_irq() || irqs_disabled());
 160 #ifdef CONFIG_TRACE_IRQFLAGS
 161         local_irq_disable();
 162 #endif
 163         /*
 164          * Are softirqs going to be turned on now:
 165          */
 166         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 167                 trace_softirqs_on(ip);
 168         /*
 169          * Keep preemption disabled until we are done with
 170          * softirq processing:
 171          */
 172         sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 173
 174         if (unlikely(!in_interrupt() && local_softirq_pending()))
 175                 do_softirq();
 176
 177         dec_preempt_count();
 178 #ifdef CONFIG_TRACE_IRQFLAGS
 179         local_irq_enable();
 180 #endif
 181         preempt_check_resched();
 182 }
 183
 184 void local_bh_enable(void)
 185 {
 186         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 187 }
 188 EXPORT_SYMBOL(local_bh_enable);
 189
 190 void local_bh_enable_ip(unsigned long ip)
 191 {
 192         _local_bh_enable_ip(ip);
 193 }
 194 EXPORT_SYMBOL(local_bh_enable_ip);
 195
 196 /*
 197  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 198  * and we fall back to softirqd after that.
 199  *
 200  * This number has been established via experimentation.
 201  * The two things to balance is latency against fairness -
 202  * we want to handle softirqs as soon as possible, but they
 203  * should not be able to lock up the box.
 204  */
 205 #define MAX_SOFTIRQ_RESTART 10
 206
 207 asmlinkage void __do_softirq(void)
 208 {
 209         struct softirq_action *h;
 210         __u32 pending;
 211         int max_restart = MAX_SOFTIRQ_RESTART;
 212         int cpu;
 213         unsigned long old_flags = current->flags;
 214
 215         /*
 216          * Mask out PF_MEMALLOC s current task context is borrowed for the
 217          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 218          * again if the socket is related to swap
 219          */
 220         current->flags &= ~PF_MEMALLOC;
 221
 222         pending = local_softirq_pending();
 223         account_system_vtime(current);
 224
 225         __local_bh_disable((unsigned long)__builtin_return_address(0),
 226                                 SOFTIRQ_OFFSET);
 227         lockdep_softirq_enter();
 228
 229         cpu = smp_processor_id();
 230 restart:
 231         /* Reset the pending bitmask before enabling irqs */
 232         set_softirq_pending(0);
 233
 234         local_irq_enable();
 235
 236         h = softirq_vec;
 237
 238         do {
 239                 if (pending & 1) {
 240                         unsigned int vec_nr = h - softirq_vec;
 241                         int prev_count = preempt_count();
 242
 243                         kstat_incr_softirqs_this_cpu(vec_nr);
 244
 245                         trace_softirq_entry(vec_nr);
 246                         h->action(h);
 247                         trace_softirq_exit(vec_nr);
 248                         if (unlikely(prev_count != preempt_count())) {
 249                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 250                                        "with preempt_count %08x,"
 251                                        " exited with %08x?\n", vec_nr,
 252                                        softirq_to_name[vec_nr], h->action,
 253                                        prev_count, preempt_count());
 254                                 preempt_count() = prev_count;
 255                         }
 256
 257                         rcu_bh_qs(cpu);
 258                 }
 259                 h++;
 260                 pending >>= 1;
 261         } while (pending);
 262
 263         local_irq_disable();
 264
 265         pending = local_softirq_pending();
 266         if (pending && --max_restart)
 267                 goto restart;
 268
 269         if (pending)
 270                 wakeup_softirqd();
 271
 272         lockdep_softirq_exit();
 273
 274         account_system_vtime(current);
 275         __local_bh_enable(SOFTIRQ_OFFSET);
 276         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 277 }
 278
 279 #ifndef __ARCH_HAS_DO_SOFTIRQ
 280
 281 asmlinkage void do_softirq(void)
 282 {
 283         __u32 pending;
 284         unsigned long flags;
 285
 286         if (in_interrupt())
 287                 return;
 288
 289         local_irq_save(flags);
 290
 291         pending = local_softirq_pending();
 292
 293         if (pending)
 294                 __do_softirq();
 295
 296         local_irq_restore(flags);
 297 }
 298
 299 #endif
 300
 301 /*
 302  * Enter an interrupt context.
 303  */
 304 void irq_enter(void)
 305 {
 306         int cpu = smp_processor_id();
 307
 308         rcu_irq_enter();
 309         if (is_idle_task(current) && !in_interrupt()) {
 310                 /*
 311                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 312                  * here, as softirq will be serviced on return from interrupt.
 313                  */
 314                 local_bh_disable();
 315                 tick_check_idle(cpu);
 316                 _local_bh_enable();
 317         }
 318
 319         __irq_enter();
 320 }
 321
 322 static inline void invoke_softirq(void)
 323 {
 324         if (!force_irqthreads) {
 325 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 326                 __do_softirq();
 327 #else
 328                 do_softirq();
 329 #endif
 330         } else {
 331                 __local_bh_disable((unsigned long)__builtin_return_address(0),
 332                                 SOFTIRQ_OFFSET);
 333                 wakeup_softirqd();
 334                 __local_bh_enable(SOFTIRQ_OFFSET);
 335         }
 336 }
 337
 338 /*
 339  * Exit an interrupt context. Process softirqs if needed and possible:
 340  */
 341 void irq_exit(void)
 342 {
 343         account_system_vtime(current);
 344         trace_hardirq_exit();
 345         sub_preempt_count(IRQ_EXIT_OFFSET);
 346         if (!in_interrupt() && local_softirq_pending())
 347                 invoke_softirq();
 348
 349 #ifdef CONFIG_NO_HZ
 350         /* Make sure that timer wheel updates are propagated */
 351         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 352                 tick_nohz_irq_exit();
 353 #endif
 354         rcu_irq_exit();
 355         sched_preempt_enable_no_resched();
 356 }
 357
 358 /*
 359  * This function must run with irqs disabled!
 360  */
 361 inline void raise_softirq_irqoff(unsigned int nr)
 362 {
 363         __raise_softirq_irqoff(nr);
 364
 365         /*
 366          * If we're in an interrupt or softirq, we're done
 367          * (this also catches softirq-disabled code). We will
 368          * actually run the softirq once we return from
 369          * the irq or softirq.
 370          *
 371          * Otherwise we wake up ksoftirqd to make sure we
 372          * schedule the softirq soon.
 373          */
 374         if (!in_interrupt())
 375                 wakeup_softirqd();
 376 }
 377
 378 void raise_softirq(unsigned int nr)
 379 {
 380         unsigned long flags;
 381
 382         local_irq_save(flags);
 383         raise_softirq_irqoff(nr);
 384         local_irq_restore(flags);
 385 }
 386
 387 void __raise_softirq_irqoff(unsigned int nr)
 388 {
 389         trace_softirq_raise(nr);
 390         or_softirq_pending(1UL << nr);
 391 }
 392
 393 void open_softirq(int nr, void (*action)(struct softirq_action *))
 394 {
 395         softirq_vec[nr].action = action;
 396 }
 397
 398 /*
 399  * Tasklets
 400  */
 401 struct tasklet_head
 402 {
 403         struct tasklet_struct *head;
 404         struct tasklet_struct **tail;
 405 };
 406
 407 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 408 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 409
 410 void __tasklet_schedule(struct tasklet_struct *t)
 411 {
 412         unsigned long flags;
 413
 414         local_irq_save(flags);
 415         t->next = NULL;
 416         *__this_cpu_read(tasklet_vec.tail) = t;
 417         __this_cpu_write(tasklet_vec.tail, &(t->next));
 418         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 419         local_irq_restore(flags);
 420 }
 421
 422 EXPORT_SYMBOL(__tasklet_schedule);
 423
 424 void __tasklet_hi_schedule(struct tasklet_struct *t)
 425 {
 426         unsigned long flags;
 427
 428         local_irq_save(flags);
 429         t->next = NULL;
 430         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 431         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 432         raise_softirq_irqoff(HI_SOFTIRQ);
 433         local_irq_restore(flags);
 434 }
 435
 436 EXPORT_SYMBOL(__tasklet_hi_schedule);
 437
 438 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 439 {
 440         BUG_ON(!irqs_disabled());
 441
 442         t->next = __this_cpu_read(tasklet_hi_vec.head);
 443         __this_cpu_write(tasklet_hi_vec.head, t);
 444         __raise_softirq_irqoff(HI_SOFTIRQ);
 445 }
 446
 447 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 448
 449 static void tasklet_action(struct softirq_action *a)
 450 {
 451         struct tasklet_struct *list;
 452
 453         local_irq_disable();
 454         list = __this_cpu_read(tasklet_vec.head);
 455         __this_cpu_write(tasklet_vec.head, NULL);
 456         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 457         local_irq_enable();
 458
 459         while (list) {
 460                 struct tasklet_struct *t = list;
 461
 462                 list = list->next;
 463
 464                 if (tasklet_trylock(t)) {
 465                         if (!atomic_read(&t->count)) {
 466                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 467                                         BUG();
 468                                 t->func(t->data);
 469                                 tasklet_unlock(t);
 470                                 continue;
 471                         }
 472                         tasklet_unlock(t);
 473                 }
 474
 475                 local_irq_disable();
 476                 t->next = NULL;
 477                 *__this_cpu_read(tasklet_vec.tail) = t;
 478                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 479                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 480                 local_irq_enable();
 481         }
 482 }
 483
 484 static void tasklet_hi_action(struct softirq_action *a)
 485 {
 486         struct tasklet_struct *list;
 487
 488         local_irq_disable();
 489         list = __this_cpu_read(tasklet_hi_vec.head);
 490         __this_cpu_write(tasklet_hi_vec.head, NULL);
 491         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 492         local_irq_enable();
 493
 494         while (list) {
 495                 struct tasklet_struct *t = list;
 496
 497                 list = list->next;
 498
 499                 if (tasklet_trylock(t)) {
 500                         if (!atomic_read(&t->count)) {
 501                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 502                                         BUG();
 503                                 t->func(t->data);
 504                                 tasklet_unlock(t);
 505                                 continue;
 506                         }
 507                         tasklet_unlock(t);
 508                 }
 509
 510                 local_irq_disable();
 511                 t->next = NULL;
 512                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 513                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 514                 __raise_softirq_irqoff(HI_SOFTIRQ);
 515                 local_irq_enable();
 516         }
 517 }
 518
 519
 520 void tasklet_init(struct tasklet_struct *t,
 521                   void (*func)(unsigned long), unsigned long data)
 522 {
 523         t->next = NULL;
 524         t->state = 0;
 525         atomic_set(&t->count, 0);
 526         t->func = func;
 527         t->data = data;
 528 }
 529
 530 EXPORT_SYMBOL(tasklet_init);
 531
 532 void tasklet_kill(struct tasklet_struct *t)
 533 {
 534         if (in_interrupt())
 535                 printk("Attempt to kill tasklet from interrupt\n");
 536
 537         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 538                 do {
 539                         yield();
 540                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 541         }
 542         tasklet_unlock_wait(t);
 543         clear_bit(TASKLET_STATE_SCHED, &t->state);
 544 }
 545
 546 EXPORT_SYMBOL(tasklet_kill);
 547
 548 /*
 549  * tasklet_hrtimer
 550  */
 551
 552 /*
 553  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 554  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 555  * hrtimer callback, but from softirq context.
 556  */
 557 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 558 {
 559         struct tasklet_hrtimer *ttimer =
 560                 container_of(timer, struct tasklet_hrtimer, timer);
 561
 562         tasklet_hi_schedule(&ttimer->tasklet);
 563         return HRTIMER_NORESTART;
 564 }
 565
 566 /*
 567  * Helper function which calls the hrtimer callback from
 568  * tasklet/softirq context
 569  */
 570 static void __tasklet_hrtimer_trampoline(unsigned long data)
 571 {
 572         struct tasklet_hrtimer *ttimer = (void *)data;
 573         enum hrtimer_restart restart;
 574
 575         restart = ttimer->function(&ttimer->timer);
 576         if (restart != HRTIMER_NORESTART)
 577                 hrtimer_restart(&ttimer->timer);
 578 }
 579
 580 /**
 581  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 582  * @ttimer:      tasklet_hrtimer which is initialized
 583  * @function:    hrtimer callback function which gets called from softirq context
 584  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 585  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 586  */
 587 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 588                           enum hrtimer_restart (*function)(struct hrtimer *),
 589                           clockid_t which_clock, enum hrtimer_mode mode)
 590 {
 591         hrtimer_init(&ttimer->timer, which_clock, mode);
 592         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 593         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 594                      (unsigned long)ttimer);
 595         ttimer->function = function;
 596 }
 597 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 598
 599 /*
 600  * Remote softirq bits
 601  */
 602
 603 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 604 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 605
 606 static void __local_trigger(struct call_single_data *cp, int softirq)
 607 {
 608         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 609
 610         list_add_tail(&cp->list, head);
 611
 612         /* Trigger the softirq only if the list was previously empty.  */
 613         if (head->next == &cp->list)
 614                 raise_softirq_irqoff(softirq);
 615 }
 616
 617 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 618 static void remote_softirq_receive(void *data)
 619 {
 620         struct call_single_data *cp = data;
 621         unsigned long flags;
 622         int softirq;
 623
 624         softirq = cp->priv;
 625
 626         local_irq_save(flags);
 627         __local_trigger(cp, softirq);
 628         local_irq_restore(flags);
 629 }
 630
 631 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 632 {
 633         if (cpu_online(cpu)) {
 634                 cp->func = remote_softirq_receive;
 635                 cp->info = cp;
 636                 cp->flags = 0;
 637                 cp->priv = softirq;
 638
 639                 __smp_call_function_single(cpu, cp, 0);
 640                 return 0;
 641         }
 642         return 1;
 643 }
 644 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 645 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 646 {
 647         return 1;
 648 }
 649 #endif
 650
 651 /**
 652  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 653  * @cp: private SMP call function data area
 654  * @cpu: the remote cpu
 655  * @this_cpu: the currently executing cpu
 656  * @softirq: the softirq for the work
 657  *
 658  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 659  * done, the work is instead queued up on the local cpu.
 660  *
 661  * Interrupts must be disabled.
 662  */
 663 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 664 {
 665         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 666                 __local_trigger(cp, softirq);
 667 }
 668 EXPORT_SYMBOL(__send_remote_softirq);
 669
 670 /**
 671  * send_remote_softirq - try to schedule softirq work on a remote cpu
 672  * @cp: private SMP call function data area
 673  * @cpu: the remote cpu
 674  * @softirq: the softirq for the work
 675  *
 676  * Like __send_remote_softirq except that disabling interrupts and
 677  * computing the current cpu is done for the caller.
 678  */
 679 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 680 {
 681         unsigned long flags;
 682         int this_cpu;
 683
 684         local_irq_save(flags);
 685         this_cpu = smp_processor_id();
 686         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 687         local_irq_restore(flags);
 688 }
 689 EXPORT_SYMBOL(send_remote_softirq);
 690
 691 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 692                                                unsigned long action, void *hcpu)
 693 {
 694         /*
 695          * If a CPU goes away, splice its entries to the current CPU
 696          * and trigger a run of the softirq
 697          */
 698         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 699                 int cpu = (unsigned long) hcpu;
 700                 int i;
 701
 702                 local_irq_disable();
 703                 for (i = 0; i < NR_SOFTIRQS; i++) {
 704                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 705                         struct list_head *local_head;
 706
 707                         if (list_empty(head))
 708                                 continue;
 709
 710                         local_head = &__get_cpu_var(softirq_work_list[i]);
 711                         list_splice_init(head, local_head);
 712                         raise_softirq_irqoff(i);
 713                 }
 714                 local_irq_enable();
 715         }
 716
 717         return NOTIFY_OK;
 718 }
 719
 720 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 721         .notifier_call  = remote_softirq_cpu_notify,
 722 };
 723
 724 void __init softirq_init(void)
 725 {
 726         int cpu;
 727
 728         for_each_possible_cpu(cpu) {
 729                 int i;
 730
 731                 per_cpu(tasklet_vec, cpu).tail =
 732                         &per_cpu(tasklet_vec, cpu).head;
 733                 per_cpu(tasklet_hi_vec, cpu).tail =
 734                         &per_cpu(tasklet_hi_vec, cpu).head;
 735                 for (i = 0; i < NR_SOFTIRQS; i++)
 736                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 737         }
 738
 739         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 740
 741         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 742         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 743 }
 744
 745 static int run_ksoftirqd(void * __bind_cpu)
 746 {
 747         set_current_state(TASK_INTERRUPTIBLE);
 748
 749         while (!kthread_should_stop()) {
 750                 preempt_disable();
 751                 if (!local_softirq_pending()) {
 752                         schedule_preempt_disabled();
 753                 }
 754
 755                 __set_current_state(TASK_RUNNING);
 756
 757                 while (local_softirq_pending()) {
 758                         /* Preempt disable stops cpu going offline.
 759                            If already offline, we'll be on wrong CPU:
 760                            don't process */
 761                         if (cpu_is_offline((long)__bind_cpu))
 762                                 goto wait_to_die;
 763                         local_irq_disable();
 764                         if (local_softirq_pending())
 765                                 __do_softirq();
 766                         local_irq_enable();
 767                         sched_preempt_enable_no_resched();
 768                         cond_resched();
 769                         preempt_disable();
 770                         rcu_note_context_switch((long)__bind_cpu);
 771                 }
 772                 preempt_enable();
 773                 set_current_state(TASK_INTERRUPTIBLE);
 774         }
 775         __set_current_state(TASK_RUNNING);
 776         return 0;
 777
 778 wait_to_die:
 779         preempt_enable();
 780         /* Wait for kthread_stop */
 781         set_current_state(TASK_INTERRUPTIBLE);
 782         while (!kthread_should_stop()) {
 783                 schedule();
 784                 set_current_state(TASK_INTERRUPTIBLE);
 785         }
 786         __set_current_state(TASK_RUNNING);
 787         return 0;
 788 }
 789
 790 #ifdef CONFIG_HOTPLUG_CPU
 791 /*
 792  * tasklet_kill_immediate is called to remove a tasklet which can already be
 793  * scheduled for execution on @cpu.
 794  *
 795  * Unlike tasklet_kill, this function removes the tasklet
 796  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 797  *
 798  * When this function is called, @cpu must be in the CPU_DEAD state.
 799  */
 800 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 801 {
 802         struct tasklet_struct **i;
 803
 804         BUG_ON(cpu_online(cpu));
 805         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 806
 807         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 808                 return;
 809
 810         /* CPU is dead, so no lock needed. */
 811         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 812                 if (*i == t) {
 813                         *i = t->next;
 814                         /* If this was the tail element, move the tail ptr */
 815                         if (*i == NULL)
 816                                 per_cpu(tasklet_vec, cpu).tail = i;
 817                         return;
 818                 }
 819         }
 820         BUG();
 821 }
 822
 823 static void takeover_tasklets(unsigned int cpu)
 824 {
 825         /* CPU is dead, so no lock needed. */
 826         local_irq_disable();
 827
 828         /* Find end, append list for that CPU. */
 829         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 830                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 831                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 832                 per_cpu(tasklet_vec, cpu).head = NULL;
 833                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 834         }
 835         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 836
 837         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 838                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 839                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 840                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 841                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 842         }
 843         raise_softirq_irqoff(HI_SOFTIRQ);
 844
 845         local_irq_enable();
 846 }
 847 #endif /* CONFIG_HOTPLUG_CPU */
 848
 849 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 850                                   unsigned long action,
 851                                   void *hcpu)
 852 {
 853         int hotcpu = (unsigned long)hcpu;
 854         struct task_struct *p;
 855
 856         switch (action) {
 857         case CPU_UP_PREPARE:
 858         case CPU_UP_PREPARE_FROZEN:
 859                 p = kthread_create_on_node(run_ksoftirqd,
 860                                            hcpu,
 861                                            cpu_to_node(hotcpu),
 862                                            "ksoftirqd/%d", hotcpu);
 863                 if (IS_ERR(p)) {
 864                         printk("ksoftirqd for %i failed\n", hotcpu);
 865                         return notifier_from_errno(PTR_ERR(p));
 866                 }
 867                 kthread_bind(p, hotcpu);
 868                 per_cpu(ksoftirqd, hotcpu) = p;
 869                 break;
 870         case CPU_ONLINE:
 871         case CPU_ONLINE_FROZEN:
 872                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 873                 break;
 874 #ifdef CONFIG_HOTPLUG_CPU
 875         case CPU_UP_CANCELED:
 876         case CPU_UP_CANCELED_FROZEN:
 877                 if (!per_cpu(ksoftirqd, hotcpu))
 878                         break;
 879                 /* Unbind so it can run.  Fall thru. */
 880                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 881                              cpumask_any(cpu_online_mask));
 882         case CPU_DEAD:
 883         case CPU_DEAD_FROZEN: {
 884                 static const struct sched_param param = {
 885                         .sched_priority = MAX_RT_PRIO-1
 886                 };
 887
 888                 p = per_cpu(ksoftirqd, hotcpu);
 889                 per_cpu(ksoftirqd, hotcpu) = NULL;
 890                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
 891                 kthread_stop(p);
 892                 takeover_tasklets(hotcpu);
 893                 break;
 894         }
 895 #endif /* CONFIG_HOTPLUG_CPU */
 896         }
 897         return NOTIFY_OK;
 898 }
 899
 900 static struct notifier_block __cpuinitdata cpu_nfb = {
 901         .notifier_call = cpu_callback
 902 };
 903
 904 static __init int spawn_ksoftirqd(void)
 905 {
 906         void *cpu = (void *)(long)smp_processor_id();
 907         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 908
 909         BUG_ON(err != NOTIFY_OK);
 910         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 911         register_cpu_notifier(&cpu_nfb);
 912         return 0;
 913 }
 914 early_initcall(spawn_ksoftirqd);
 915
 916 /*
 917  * [ These __weak aliases are kept in a separate compilation unit, so that
 918  *   GCC does not inline them incorrectly. ]
 919  */
 920
 921 int __init __weak early_irq_init(void)
 922 {
 923         return 0;
 924 }
 925
 926 #ifdef CONFIG_GENERIC_HARDIRQS
 927 int __init __weak arch_probe_nr_irqs(void)
 928 {
 929         return NR_IRQS_LEGACY;
 930 }
 931
 932 int __init __weak arch_early_irq_init(void)
 933 {
 934         return 0;
 935 }
 936 #endif