kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/export.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/smpboot.h>
  27 #include <linux/tick.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include <trace/events/irq.h>
  31
  32 #include <asm/irq.h>
  33 /*
  34    - No shared variables, all the data are CPU local.
  35    - If a softirq needs serialization, let it serialize itself
  36      by its own spinlocks.
  37    - Even if softirq is serialized, only local cpu is marked for
  38      execution. Hence, we get something sort of weak cpu binding.
  39      Though it is still not clear, will it result in better locality
  40      or will not.
  41
  42    Examples:
  43    - NET RX softirq. It is multithreaded and does not require
  44      any global serialization.
  45    - NET TX softirq. It kicks software netdevice queues, hence
  46      it is logically serialized per device, but this serialization
  47      is invisible to common code.
  48    - Tasklets: serialized wrt itself.
  49  */
  50
  51 #ifndef __ARCH_IRQ_STAT
  52 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  53 EXPORT_SYMBOL(irq_stat);
  54 #endif
  55
  56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60 char *softirq_to_name[NR_SOFTIRQS] = {
  61         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  62         "TASKLET", "SCHED", "HRTIMER", "RCU"
  63 };
  64
  65 /*
  66  * we cannot loop indefinitely here to avoid userspace starvation,
  67  * but we also don't want to introduce a worst case 1/HZ latency
  68  * to the pending events, so lets the scheduler to balance
  69  * the softirq load for us.
  70  */
  71 static void wakeup_softirqd(void)
  72 {
  73         /* Interrupts are disabled: no need to stop preemption */
  74         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76         if (tsk && tsk->state != TASK_RUNNING)
  77                 wake_up_process(tsk);
  78 }
  79
  80 /*
  81  * preempt_count and SOFTIRQ_OFFSET usage:
  82  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  83  *   softirq processing.
  84  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  85  *   on local_bh_disable or local_bh_enable.
  86  * This lets us distinguish between whether we are currently processing
  87  * softirq and whether we just have bh disabled.
  88  */
  89
  90 /*
  91  * This one is for softirq.c-internal use,
  92  * where hardirqs are disabled legitimately:
  93  */
  94 #ifdef CONFIG_TRACE_IRQFLAGS
  95 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  96 {
  97         unsigned long flags;
  98
  99         WARN_ON_ONCE(in_irq());
 100
 101         raw_local_irq_save(flags);
 102         /*
 103          * The preempt tracer hooks into add_preempt_count and will break
 104          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 105          * is set and before current->softirq_enabled is cleared.
 106          * We must manually increment preempt_count here and manually
 107          * call the trace_preempt_off later.
 108          */
 109         preempt_count() += cnt;
 110         /*
 111          * Were softirqs turned off above:
 112          */
 113         if (softirq_count() == cnt)
 114                 trace_softirqs_off(ip);
 115         raw_local_irq_restore(flags);
 116
 117         if (preempt_count() == cnt)
 118                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 119 }
 120 #else /* !CONFIG_TRACE_IRQFLAGS */
 121 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 122 {
 123         add_preempt_count(cnt);
 124         barrier();
 125 }
 126 #endif /* CONFIG_TRACE_IRQFLAGS */
 127
 128 void local_bh_disable(void)
 129 {
 130         __local_bh_disable((unsigned long)__builtin_return_address(0),
 131                                 SOFTIRQ_DISABLE_OFFSET);
 132 }
 133
 134 EXPORT_SYMBOL(local_bh_disable);
 135
 136 static void __local_bh_enable(unsigned int cnt)
 137 {
 138         WARN_ON_ONCE(in_irq());
 139         WARN_ON_ONCE(!irqs_disabled());
 140
 141         if (softirq_count() == cnt)
 142                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 143         sub_preempt_count(cnt);
 144 }
 145
 146 /*
 147  * Special-case - softirqs can safely be enabled in
 148  * cond_resched_softirq(), or by __do_softirq(),
 149  * without processing still-pending softirqs:
 150  */
 151 void _local_bh_enable(void)
 152 {
 153         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 154 }
 155
 156 EXPORT_SYMBOL(_local_bh_enable);
 157
 158 static inline void _local_bh_enable_ip(unsigned long ip)
 159 {
 160         WARN_ON_ONCE(in_irq() || irqs_disabled());
 161 #ifdef CONFIG_TRACE_IRQFLAGS
 162         local_irq_disable();
 163 #endif
 164         /*
 165          * Are softirqs going to be turned on now:
 166          */
 167         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 168                 trace_softirqs_on(ip);
 169         /*
 170          * Keep preemption disabled until we are done with
 171          * softirq processing:
 172          */
 173         sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 174
 175         if (unlikely(!in_interrupt() && local_softirq_pending()))
 176                 do_softirq();
 177
 178         dec_preempt_count();
 179 #ifdef CONFIG_TRACE_IRQFLAGS
 180         local_irq_enable();
 181 #endif
 182         preempt_check_resched();
 183 }
 184
 185 void local_bh_enable(void)
 186 {
 187         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 188 }
 189 EXPORT_SYMBOL(local_bh_enable);
 190
 191 void local_bh_enable_ip(unsigned long ip)
 192 {
 193         _local_bh_enable_ip(ip);
 194 }
 195 EXPORT_SYMBOL(local_bh_enable_ip);
 196
 197 /*
 198  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 199  * but break the loop if need_resched() is set or after 2 ms.
 200  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 201  * certain cases, such as stop_machine(), jiffies may cease to
 202  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 203  * well to make sure we eventually return from this method.
 204  *
 205  * These limits have been established via experimentation.
 206  * The two things to balance is latency against fairness -
 207  * we want to handle softirqs as soon as possible, but they
 208  * should not be able to lock up the box.
 209  */
 210 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 211 #define MAX_SOFTIRQ_RESTART 10
 212
 213 asmlinkage void __do_softirq(void)
 214 {
 215         struct softirq_action *h;
 216         __u32 pending;
 217         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 218         int cpu;
 219         unsigned long old_flags = current->flags;
 220         int max_restart = MAX_SOFTIRQ_RESTART;
 221
 222         /*
 223          * Mask out PF_MEMALLOC s current task context is borrowed for the
 224          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 225          * again if the socket is related to swap
 226          */
 227         current->flags &= ~PF_MEMALLOC;
 228
 229         pending = local_softirq_pending();
 230         account_irq_enter_time(current);
 231
 232         __local_bh_disable((unsigned long)__builtin_return_address(0),
 233                                 SOFTIRQ_OFFSET);
 234         lockdep_softirq_enter();
 235
 236         cpu = smp_processor_id();
 237 restart:
 238         /* Reset the pending bitmask before enabling irqs */
 239         set_softirq_pending(0);
 240
 241         local_irq_enable();
 242
 243         h = softirq_vec;
 244
 245         do {
 246                 if (pending & 1) {
 247                         unsigned int vec_nr = h - softirq_vec;
 248                         int prev_count = preempt_count();
 249
 250                         kstat_incr_softirqs_this_cpu(vec_nr);
 251
 252                         trace_softirq_entry(vec_nr);
 253                         h->action(h);
 254                         trace_softirq_exit(vec_nr);
 255                         if (unlikely(prev_count != preempt_count())) {
 256                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 257                                        "with preempt_count %08x,"
 258                                        " exited with %08x?\n", vec_nr,
 259                                        softirq_to_name[vec_nr], h->action,
 260                                        prev_count, preempt_count());
 261                                 preempt_count() = prev_count;
 262                         }
 263
 264                         rcu_bh_qs(cpu);
 265                 }
 266                 h++;
 267                 pending >>= 1;
 268         } while (pending);
 269
 270         local_irq_disable();
 271
 272         pending = local_softirq_pending();
 273         if (pending) {
 274                 if (time_before(jiffies, end) && !need_resched() &&
 275                     --max_restart)
 276                         goto restart;
 277
 278                 wakeup_softirqd();
 279         }
 280
 281         lockdep_softirq_exit();
 282
 283         account_irq_exit_time(current);
 284         __local_bh_enable(SOFTIRQ_OFFSET);
 285         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 286 }
 287
 288 #ifndef __ARCH_HAS_DO_SOFTIRQ
 289
 290 asmlinkage void do_softirq(void)
 291 {
 292         __u32 pending;
 293         unsigned long flags;
 294
 295         if (in_interrupt())
 296                 return;
 297
 298         local_irq_save(flags);
 299
 300         pending = local_softirq_pending();
 301
 302         if (pending)
 303                 __do_softirq();
 304
 305         local_irq_restore(flags);
 306 }
 307
 308 #endif
 309
 310 /*
 311  * Enter an interrupt context.
 312  */
 313 void irq_enter(void)
 314 {
 315         int cpu = smp_processor_id();
 316
 317         rcu_irq_enter();
 318         if (is_idle_task(current) && !in_interrupt()) {
 319                 /*
 320                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 321                  * here, as softirq will be serviced on return from interrupt.
 322                  */
 323                 local_bh_disable();
 324                 tick_check_idle(cpu);
 325                 _local_bh_enable();
 326         }
 327
 328         __irq_enter();
 329 }
 330
 331 static inline void invoke_softirq(void)
 332 {
 333         if (!force_irqthreads)
 334                 __do_softirq();
 335         else
 336                 wakeup_softirqd();
 337 }
 338
 339 static inline void tick_irq_exit(void)
 340 {
 341 #ifdef CONFIG_NO_HZ_COMMON
 342         int cpu = smp_processor_id();
 343
 344         /* Make sure that timer wheel updates are propagated */
 345         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 346                 if (!in_interrupt())
 347                         tick_nohz_irq_exit();
 348         }
 349 #endif
 350 }
 351
 352 /*
 353  * Exit an interrupt context. Process softirqs if needed and possible:
 354  */
 355 void irq_exit(void)
 356 {
 357 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 358         local_irq_disable();
 359 #else
 360         WARN_ON_ONCE(!irqs_disabled());
 361 #endif
 362
 363         account_irq_exit_time(current);
 364         trace_hardirq_exit();
 365         sub_preempt_count(HARDIRQ_OFFSET);
 366         if (!in_interrupt() && local_softirq_pending())
 367                 invoke_softirq();
 368
 369         tick_irq_exit();
 370         rcu_irq_exit();
 371 }
 372
 373 /*
 374  * This function must run with irqs disabled!
 375  */
 376 inline void raise_softirq_irqoff(unsigned int nr)
 377 {
 378         __raise_softirq_irqoff(nr);
 379
 380         /*
 381          * If we're in an interrupt or softirq, we're done
 382          * (this also catches softirq-disabled code). We will
 383          * actually run the softirq once we return from
 384          * the irq or softirq.
 385          *
 386          * Otherwise we wake up ksoftirqd to make sure we
 387          * schedule the softirq soon.
 388          */
 389         if (!in_interrupt())
 390                 wakeup_softirqd();
 391 }
 392
 393 void raise_softirq(unsigned int nr)
 394 {
 395         unsigned long flags;
 396
 397         local_irq_save(flags);
 398         raise_softirq_irqoff(nr);
 399         local_irq_restore(flags);
 400 }
 401
 402 void __raise_softirq_irqoff(unsigned int nr)
 403 {
 404         trace_softirq_raise(nr);
 405         or_softirq_pending(1UL << nr);
 406 }
 407
 408 void open_softirq(int nr, void (*action)(struct softirq_action *))
 409 {
 410         softirq_vec[nr].action = action;
 411 }
 412
 413 /*
 414  * Tasklets
 415  */
 416 struct tasklet_head
 417 {
 418         struct tasklet_struct *head;
 419         struct tasklet_struct **tail;
 420 };
 421
 422 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 423 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 424
 425 void __tasklet_schedule(struct tasklet_struct *t)
 426 {
 427         unsigned long flags;
 428
 429         local_irq_save(flags);
 430         t->next = NULL;
 431         *__this_cpu_read(tasklet_vec.tail) = t;
 432         __this_cpu_write(tasklet_vec.tail, &(t->next));
 433         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 434         local_irq_restore(flags);
 435 }
 436
 437 EXPORT_SYMBOL(__tasklet_schedule);
 438
 439 void __tasklet_hi_schedule(struct tasklet_struct *t)
 440 {
 441         unsigned long flags;
 442
 443         local_irq_save(flags);
 444         t->next = NULL;
 445         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 446         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 447         raise_softirq_irqoff(HI_SOFTIRQ);
 448         local_irq_restore(flags);
 449 }
 450
 451 EXPORT_SYMBOL(__tasklet_hi_schedule);
 452
 453 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 454 {
 455         BUG_ON(!irqs_disabled());
 456
 457         t->next = __this_cpu_read(tasklet_hi_vec.head);
 458         __this_cpu_write(tasklet_hi_vec.head, t);
 459         __raise_softirq_irqoff(HI_SOFTIRQ);
 460 }
 461
 462 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 463
 464 static void tasklet_action(struct softirq_action *a)
 465 {
 466         struct tasklet_struct *list;
 467
 468         local_irq_disable();
 469         list = __this_cpu_read(tasklet_vec.head);
 470         __this_cpu_write(tasklet_vec.head, NULL);
 471         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 472         local_irq_enable();
 473
 474         while (list) {
 475                 struct tasklet_struct *t = list;
 476
 477                 list = list->next;
 478
 479                 if (tasklet_trylock(t)) {
 480                         if (!atomic_read(&t->count)) {
 481                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 482                                         BUG();
 483                                 t->func(t->data);
 484                                 tasklet_unlock(t);
 485                                 continue;
 486                         }
 487                         tasklet_unlock(t);
 488                 }
 489
 490                 local_irq_disable();
 491                 t->next = NULL;
 492                 *__this_cpu_read(tasklet_vec.tail) = t;
 493                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 494                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 495                 local_irq_enable();
 496         }
 497 }
 498
 499 static void tasklet_hi_action(struct softirq_action *a)
 500 {
 501         struct tasklet_struct *list;
 502
 503         local_irq_disable();
 504         list = __this_cpu_read(tasklet_hi_vec.head);
 505         __this_cpu_write(tasklet_hi_vec.head, NULL);
 506         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 507         local_irq_enable();
 508
 509         while (list) {
 510                 struct tasklet_struct *t = list;
 511
 512                 list = list->next;
 513
 514                 if (tasklet_trylock(t)) {
 515                         if (!atomic_read(&t->count)) {
 516                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 517                                         BUG();
 518                                 t->func(t->data);
 519                                 tasklet_unlock(t);
 520                                 continue;
 521                         }
 522                         tasklet_unlock(t);
 523                 }
 524
 525                 local_irq_disable();
 526                 t->next = NULL;
 527                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 528                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 529                 __raise_softirq_irqoff(HI_SOFTIRQ);
 530                 local_irq_enable();
 531         }
 532 }
 533
 534
 535 void tasklet_init(struct tasklet_struct *t,
 536                   void (*func)(unsigned long), unsigned long data)
 537 {
 538         t->next = NULL;
 539         t->state = 0;
 540         atomic_set(&t->count, 0);
 541         t->func = func;
 542         t->data = data;
 543 }
 544
 545 EXPORT_SYMBOL(tasklet_init);
 546
 547 void tasklet_kill(struct tasklet_struct *t)
 548 {
 549         if (in_interrupt())
 550                 printk("Attempt to kill tasklet from interrupt\n");
 551
 552         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 553                 do {
 554                         yield();
 555                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 556         }
 557         tasklet_unlock_wait(t);
 558         clear_bit(TASKLET_STATE_SCHED, &t->state);
 559 }
 560
 561 EXPORT_SYMBOL(tasklet_kill);
 562
 563 /*
 564  * tasklet_hrtimer
 565  */
 566
 567 /*
 568  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 569  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 570  * hrtimer callback, but from softirq context.
 571  */
 572 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 573 {
 574         struct tasklet_hrtimer *ttimer =
 575                 container_of(timer, struct tasklet_hrtimer, timer);
 576
 577         tasklet_hi_schedule(&ttimer->tasklet);
 578         return HRTIMER_NORESTART;
 579 }
 580
 581 /*
 582  * Helper function which calls the hrtimer callback from
 583  * tasklet/softirq context
 584  */
 585 static void __tasklet_hrtimer_trampoline(unsigned long data)
 586 {
 587         struct tasklet_hrtimer *ttimer = (void *)data;
 588         enum hrtimer_restart restart;
 589
 590         restart = ttimer->function(&ttimer->timer);
 591         if (restart != HRTIMER_NORESTART)
 592                 hrtimer_restart(&ttimer->timer);
 593 }
 594
 595 /**
 596  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 597  * @ttimer:      tasklet_hrtimer which is initialized
 598  * @function:    hrtimer callback function which gets called from softirq context
 599  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 600  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 601  */
 602 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 603                           enum hrtimer_restart (*function)(struct hrtimer *),
 604                           clockid_t which_clock, enum hrtimer_mode mode)
 605 {
 606         hrtimer_init(&ttimer->timer, which_clock, mode);
 607         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 608         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 609                      (unsigned long)ttimer);
 610         ttimer->function = function;
 611 }
 612 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 613
 614 /*
 615  * Remote softirq bits
 616  */
 617
 618 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 619 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 620
 621 static void __local_trigger(struct call_single_data *cp, int softirq)
 622 {
 623         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 624
 625         list_add_tail(&cp->list, head);
 626
 627         /* Trigger the softirq only if the list was previously empty.  */
 628         if (head->next == &cp->list)
 629                 raise_softirq_irqoff(softirq);
 630 }
 631
 632 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 633 static void remote_softirq_receive(void *data)
 634 {
 635         struct call_single_data *cp = data;
 636         unsigned long flags;
 637         int softirq;
 638
 639         softirq = *(int *)cp->info;
 640         local_irq_save(flags);
 641         __local_trigger(cp, softirq);
 642         local_irq_restore(flags);
 643 }
 644
 645 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 646 {
 647         if (cpu_online(cpu)) {
 648                 cp->func = remote_softirq_receive;
 649                 cp->info = &softirq;
 650                 cp->flags = 0;
 651
 652                 __smp_call_function_single(cpu, cp, 0);
 653                 return 0;
 654         }
 655         return 1;
 656 }
 657 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 658 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 659 {
 660         return 1;
 661 }
 662 #endif
 663
 664 /**
 665  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 666  * @cp: private SMP call function data area
 667  * @cpu: the remote cpu
 668  * @this_cpu: the currently executing cpu
 669  * @softirq: the softirq for the work
 670  *
 671  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 672  * done, the work is instead queued up on the local cpu.
 673  *
 674  * Interrupts must be disabled.
 675  */
 676 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 677 {
 678         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 679                 __local_trigger(cp, softirq);
 680 }
 681 EXPORT_SYMBOL(__send_remote_softirq);
 682
 683 /**
 684  * send_remote_softirq - try to schedule softirq work on a remote cpu
 685  * @cp: private SMP call function data area
 686  * @cpu: the remote cpu
 687  * @softirq: the softirq for the work
 688  *
 689  * Like __send_remote_softirq except that disabling interrupts and
 690  * computing the current cpu is done for the caller.
 691  */
 692 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 693 {
 694         unsigned long flags;
 695         int this_cpu;
 696
 697         local_irq_save(flags);
 698         this_cpu = smp_processor_id();
 699         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 700         local_irq_restore(flags);
 701 }
 702 EXPORT_SYMBOL(send_remote_softirq);
 703
 704 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 705                                                unsigned long action, void *hcpu)
 706 {
 707         /*
 708          * If a CPU goes away, splice its entries to the current CPU
 709          * and trigger a run of the softirq
 710          */
 711         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 712                 int cpu = (unsigned long) hcpu;
 713                 int i;
 714
 715                 local_irq_disable();
 716                 for (i = 0; i < NR_SOFTIRQS; i++) {
 717                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 718                         struct list_head *local_head;
 719
 720                         if (list_empty(head))
 721                                 continue;
 722
 723                         local_head = &__get_cpu_var(softirq_work_list[i]);
 724                         list_splice_init(head, local_head);
 725                         raise_softirq_irqoff(i);
 726                 }
 727                 local_irq_enable();
 728         }
 729
 730         return NOTIFY_OK;
 731 }
 732
 733 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 734         .notifier_call  = remote_softirq_cpu_notify,
 735 };
 736
 737 void __init softirq_init(void)
 738 {
 739         int cpu;
 740
 741         for_each_possible_cpu(cpu) {
 742                 int i;
 743
 744                 per_cpu(tasklet_vec, cpu).tail =
 745                         &per_cpu(tasklet_vec, cpu).head;
 746                 per_cpu(tasklet_hi_vec, cpu).tail =
 747                         &per_cpu(tasklet_hi_vec, cpu).head;
 748                 for (i = 0; i < NR_SOFTIRQS; i++)
 749                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 750         }
 751
 752         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 753
 754         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 755         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 756 }
 757
 758 static int ksoftirqd_should_run(unsigned int cpu)
 759 {
 760         return local_softirq_pending();
 761 }
 762
 763 static void run_ksoftirqd(unsigned int cpu)
 764 {
 765         local_irq_disable();
 766         if (local_softirq_pending()) {
 767                 __do_softirq();
 768                 rcu_note_context_switch(cpu);
 769                 local_irq_enable();
 770                 cond_resched();
 771                 return;
 772         }
 773         local_irq_enable();
 774 }
 775
 776 #ifdef CONFIG_HOTPLUG_CPU
 777 /*
 778  * tasklet_kill_immediate is called to remove a tasklet which can already be
 779  * scheduled for execution on @cpu.
 780  *
 781  * Unlike tasklet_kill, this function removes the tasklet
 782  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 783  *
 784  * When this function is called, @cpu must be in the CPU_DEAD state.
 785  */
 786 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 787 {
 788         struct tasklet_struct **i;
 789
 790         BUG_ON(cpu_online(cpu));
 791         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 792
 793         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 794                 return;
 795
 796         /* CPU is dead, so no lock needed. */
 797         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 798                 if (*i == t) {
 799                         *i = t->next;
 800                         /* If this was the tail element, move the tail ptr */
 801                         if (*i == NULL)
 802                                 per_cpu(tasklet_vec, cpu).tail = i;
 803                         return;
 804                 }
 805         }
 806         BUG();
 807 }
 808
 809 static void takeover_tasklets(unsigned int cpu)
 810 {
 811         /* CPU is dead, so no lock needed. */
 812         local_irq_disable();
 813
 814         /* Find end, append list for that CPU. */
 815         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 816                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 817                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 818                 per_cpu(tasklet_vec, cpu).head = NULL;
 819                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 820         }
 821         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 822
 823         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 824                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 825                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 826                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 827                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 828         }
 829         raise_softirq_irqoff(HI_SOFTIRQ);
 830
 831         local_irq_enable();
 832 }
 833 #endif /* CONFIG_HOTPLUG_CPU */
 834
 835 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 836                                   unsigned long action,
 837                                   void *hcpu)
 838 {
 839         switch (action) {
 840 #ifdef CONFIG_HOTPLUG_CPU
 841         case CPU_DEAD:
 842         case CPU_DEAD_FROZEN:
 843                 takeover_tasklets((unsigned long)hcpu);
 844                 break;
 845 #endif /* CONFIG_HOTPLUG_CPU */
 846         }
 847         return NOTIFY_OK;
 848 }
 849
 850 static struct notifier_block __cpuinitdata cpu_nfb = {
 851         .notifier_call = cpu_callback
 852 };
 853
 854 static struct smp_hotplug_thread softirq_threads = {
 855         .store                  = &ksoftirqd,
 856         .thread_should_run      = ksoftirqd_should_run,
 857         .thread_fn              = run_ksoftirqd,
 858         .thread_comm            = "ksoftirqd/%u",
 859 };
 860
 861 static __init int spawn_ksoftirqd(void)
 862 {
 863         register_cpu_notifier(&cpu_nfb);
 864
 865         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 866
 867         return 0;
 868 }
 869 early_initcall(spawn_ksoftirqd);
 870
 871 /*
 872  * [ These __weak aliases are kept in a separate compilation unit, so that
 873  *   GCC does not inline them incorrectly. ]
 874  */
 875
 876 int __init __weak early_irq_init(void)
 877 {
 878         return 0;
 879 }
 880
 881 #ifdef CONFIG_GENERIC_HARDIRQS
 882 int __init __weak arch_probe_nr_irqs(void)
 883 {
 884         return NR_IRQS_LEGACY;
 885 }
 886
 887 int __init __weak arch_early_irq_init(void)
 888 {
 889         return 0;
 890 }
 891 #endif