kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  */
  10
  11 #include <linux/export.h>
  12 #include <linux/kernel_stat.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/init.h>
  15 #include <linux/mm.h>
  16 #include <linux/notifier.h>
  17 #include <linux/percpu.h>
  18 #include <linux/cpu.h>
  19 #include <linux/freezer.h>
  20 #include <linux/kthread.h>
  21 #include <linux/rcupdate.h>
  22 #include <linux/ftrace.h>
  23 #include <linux/smp.h>
  24 #include <linux/smpboot.h>
  25 #include <linux/tick.h>
  26
  27 #define CREATE_TRACE_POINTS
  28 #include <trace/events/irq.h>
  29
  30 /*
  31    - No shared variables, all the data are CPU local.
  32    - If a softirq needs serialization, let it serialize itself
  33      by its own spinlocks.
  34    - Even if softirq is serialized, only local cpu is marked for
  35      execution. Hence, we get something sort of weak cpu binding.
  36      Though it is still not clear, will it result in better locality
  37      or will not.
  38
  39    Examples:
  40    - NET RX softirq. It is multithreaded and does not require
  41      any global serialization.
  42    - NET TX softirq. It kicks software netdevice queues, hence
  43      it is logically serialized per device, but this serialization
  44      is invisible to common code.
  45    - Tasklets: serialized wrt itself.
  46  */
  47
  48 #ifndef __ARCH_IRQ_STAT
  49 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  50 EXPORT_SYMBOL(irq_stat);
  51 #endif
  52
  53 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  54
  55 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  56
  57 char *softirq_to_name[NR_SOFTIRQS] = {
  58         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  59         "TASKLET", "SCHED", "HRTIMER", "RCU"
  60 };
  61
  62 /*
  63  * we cannot loop indefinitely here to avoid userspace starvation,
  64  * but we also don't want to introduce a worst case 1/HZ latency
  65  * to the pending events, so lets the scheduler to balance
  66  * the softirq load for us.
  67  */
  68 static void wakeup_softirqd(void)
  69 {
  70         /* Interrupts are disabled: no need to stop preemption */
  71         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  72
  73         if (tsk && tsk->state != TASK_RUNNING)
  74                 wake_up_process(tsk);
  75 }
  76
  77 /*
  78  * preempt_count and SOFTIRQ_OFFSET usage:
  79  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  80  *   softirq processing.
  81  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  82  *   on local_bh_disable or local_bh_enable.
  83  * This lets us distinguish between whether we are currently processing
  84  * softirq and whether we just have bh disabled.
  85  */
  86
  87 /*
  88  * This one is for softirq.c-internal use,
  89  * where hardirqs are disabled legitimately:
  90  */
  91 #ifdef CONFIG_TRACE_IRQFLAGS
  92 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
  93 {
  94         unsigned long flags;
  95
  96         WARN_ON_ONCE(in_irq());
  97
  98         raw_local_irq_save(flags);
  99         /*
 100          * The preempt tracer hooks into preempt_count_add and will break
 101          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 102          * is set and before current->softirq_enabled is cleared.
 103          * We must manually increment preempt_count here and manually
 104          * call the trace_preempt_off later.
 105          */
 106         __preempt_count_add(cnt);
 107         /*
 108          * Were softirqs turned off above:
 109          */
 110         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 111                 trace_softirqs_off(ip);
 112         raw_local_irq_restore(flags);
 113
 114         if (preempt_count() == cnt)
 115                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 116 }
 117 EXPORT_SYMBOL(__local_bh_disable_ip);
 118 #endif /* CONFIG_TRACE_IRQFLAGS */
 119
 120 static void __local_bh_enable(unsigned int cnt)
 121 {
 122         WARN_ON_ONCE(!irqs_disabled());
 123
 124         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 125                 trace_softirqs_on(_RET_IP_);
 126         preempt_count_sub(cnt);
 127 }
 128
 129 /*
 130  * Special-case - softirqs can safely be enabled in
 131  * cond_resched_softirq(), or by __do_softirq(),
 132  * without processing still-pending softirqs:
 133  */
 134 void _local_bh_enable(void)
 135 {
 136         WARN_ON_ONCE(in_irq());
 137         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 138 }
 139
 140 EXPORT_SYMBOL(_local_bh_enable);
 141
 142 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 143 {
 144         WARN_ON_ONCE(in_irq() || irqs_disabled());
 145 #ifdef CONFIG_TRACE_IRQFLAGS
 146         local_irq_disable();
 147 #endif
 148         /*
 149          * Are softirqs going to be turned on now:
 150          */
 151         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 152                 trace_softirqs_on(ip);
 153         /*
 154          * Keep preemption disabled until we are done with
 155          * softirq processing:
 156          */
 157         preempt_count_sub(cnt - 1);
 158
 159         if (unlikely(!in_interrupt() && local_softirq_pending())) {
 160                 /*
 161                  * Run softirq if any pending. And do it in its own stack
 162                  * as we may be calling this deep in a task call stack already.
 163                  */
 164                 do_softirq();
 165         }
 166
 167         preempt_count_dec();
 168 #ifdef CONFIG_TRACE_IRQFLAGS
 169         local_irq_enable();
 170 #endif
 171         preempt_check_resched();
 172 }
 173 EXPORT_SYMBOL(__local_bh_enable_ip);
 174
 175 /*
 176  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 177  * but break the loop if need_resched() is set or after 2 ms.
 178  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 179  * certain cases, such as stop_machine(), jiffies may cease to
 180  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 181  * well to make sure we eventually return from this method.
 182  *
 183  * These limits have been established via experimentation.
 184  * The two things to balance is latency against fairness -
 185  * we want to handle softirqs as soon as possible, but they
 186  * should not be able to lock up the box.
 187  */
 188 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 189 #define MAX_SOFTIRQ_RESTART 10
 190
 191 #ifdef CONFIG_TRACE_IRQFLAGS
 192 /*
 193  * When we run softirqs from irq_exit() and thus on the hardirq stack we need
 194  * to keep the lockdep irq context tracking as tight as possible in order to
 195  * not miss-qualify lock contexts and miss possible deadlocks.
 196  */
 197
 198 static inline bool lockdep_softirq_start(void)
 199 {
 200         bool in_hardirq = false;
 201
 202         if (trace_hardirq_context(current)) {
 203                 in_hardirq = true;
 204                 trace_hardirq_exit();
 205         }
 206
 207         lockdep_softirq_enter();
 208
 209         return in_hardirq;
 210 }
 211
 212 static inline void lockdep_softirq_end(bool in_hardirq)
 213 {
 214         lockdep_softirq_exit();
 215
 216         if (in_hardirq)
 217                 trace_hardirq_enter();
 218 }
 219 #else
 220 static inline bool lockdep_softirq_start(void) { return false; }
 221 static inline void lockdep_softirq_end(bool in_hardirq) { }
 222 #endif
 223
 224 asmlinkage void __do_softirq(void)
 225 {
 226         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 227         unsigned long old_flags = current->flags;
 228         int max_restart = MAX_SOFTIRQ_RESTART;
 229         struct softirq_action *h;
 230         bool in_hardirq;
 231         __u32 pending;
 232         int cpu;
 233
 234         /*
 235          * Mask out PF_MEMALLOC s current task context is borrowed for the
 236          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 237          * again if the socket is related to swap
 238          */
 239         current->flags &= ~PF_MEMALLOC;
 240
 241         pending = local_softirq_pending();
 242         account_irq_enter_time(current);
 243
 244         __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 245         in_hardirq = lockdep_softirq_start();
 246
 247         cpu = smp_processor_id();
 248 restart:
 249         /* Reset the pending bitmask before enabling irqs */
 250         set_softirq_pending(0);
 251
 252         local_irq_enable();
 253
 254         h = softirq_vec;
 255
 256         do {
 257                 if (pending & 1) {
 258                         unsigned int vec_nr = h - softirq_vec;
 259                         int prev_count = preempt_count();
 260
 261                         kstat_incr_softirqs_this_cpu(vec_nr);
 262
 263                         trace_softirq_entry(vec_nr);
 264                         h->action(h);
 265                         trace_softirq_exit(vec_nr);
 266                         if (unlikely(prev_count != preempt_count())) {
 267                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 268                                        "with preempt_count %08x,"
 269                                        " exited with %08x?\n", vec_nr,
 270                                        softirq_to_name[vec_nr], h->action,
 271                                        prev_count, preempt_count());
 272                                 preempt_count_set(prev_count);
 273                         }
 274
 275                         rcu_bh_qs(cpu);
 276                 }
 277                 h++;
 278                 pending >>= 1;
 279         } while (pending);
 280
 281         local_irq_disable();
 282
 283         pending = local_softirq_pending();
 284         if (pending) {
 285                 if (time_before(jiffies, end) && !need_resched() &&
 286                     --max_restart)
 287                         goto restart;
 288
 289                 wakeup_softirqd();
 290         }
 291
 292         lockdep_softirq_end(in_hardirq);
 293         account_irq_exit_time(current);
 294         __local_bh_enable(SOFTIRQ_OFFSET);
 295         WARN_ON_ONCE(in_interrupt());
 296         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 297 }
 298
 299 asmlinkage void do_softirq(void)
 300 {
 301         __u32 pending;
 302         unsigned long flags;
 303
 304         if (in_interrupt())
 305                 return;
 306
 307         local_irq_save(flags);
 308
 309         pending = local_softirq_pending();
 310
 311         if (pending)
 312                 do_softirq_own_stack();
 313
 314         local_irq_restore(flags);
 315 }
 316
 317 /*
 318  * Enter an interrupt context.
 319  */
 320 void irq_enter(void)
 321 {
 322         rcu_irq_enter();
 323         if (is_idle_task(current) && !in_interrupt()) {
 324                 /*
 325                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 326                  * here, as softirq will be serviced on return from interrupt.
 327                  */
 328                 local_bh_disable();
 329                 tick_check_idle();
 330                 _local_bh_enable();
 331         }
 332
 333         __irq_enter();
 334 }
 335
 336 static inline void invoke_softirq(void)
 337 {
 338         if (!force_irqthreads) {
 339 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 340                 /*
 341                  * We can safely execute softirq on the current stack if
 342                  * it is the irq stack, because it should be near empty
 343                  * at this stage.
 344                  */
 345                 __do_softirq();
 346 #else
 347                 /*
 348                  * Otherwise, irq_exit() is called on the task stack that can
 349                  * be potentially deep already. So call softirq in its own stack
 350                  * to prevent from any overrun.
 351                  */
 352                 do_softirq_own_stack();
 353 #endif
 354         } else {
 355                 wakeup_softirqd();
 356         }
 357 }
 358
 359 static inline void tick_irq_exit(void)
 360 {
 361 #ifdef CONFIG_NO_HZ_COMMON
 362         int cpu = smp_processor_id();
 363
 364         /* Make sure that timer wheel updates are propagated */
 365         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 366                 if (!in_interrupt())
 367                         tick_nohz_irq_exit();
 368         }
 369 #endif
 370 }
 371
 372 /*
 373  * Exit an interrupt context. Process softirqs if needed and possible:
 374  */
 375 void irq_exit(void)
 376 {
 377 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 378         local_irq_disable();
 379 #else
 380         WARN_ON_ONCE(!irqs_disabled());
 381 #endif
 382
 383         account_irq_exit_time(current);
 384         preempt_count_sub(HARDIRQ_OFFSET);
 385         if (!in_interrupt() && local_softirq_pending())
 386                 invoke_softirq();
 387
 388         tick_irq_exit();
 389         rcu_irq_exit();
 390         trace_hardirq_exit(); /* must be last! */
 391 }
 392
 393 /*
 394  * This function must run with irqs disabled!
 395  */
 396 inline void raise_softirq_irqoff(unsigned int nr)
 397 {
 398         __raise_softirq_irqoff(nr);
 399
 400         /*
 401          * If we're in an interrupt or softirq, we're done
 402          * (this also catches softirq-disabled code). We will
 403          * actually run the softirq once we return from
 404          * the irq or softirq.
 405          *
 406          * Otherwise we wake up ksoftirqd to make sure we
 407          * schedule the softirq soon.
 408          */
 409         if (!in_interrupt())
 410                 wakeup_softirqd();
 411 }
 412
 413 void raise_softirq(unsigned int nr)
 414 {
 415         unsigned long flags;
 416
 417         local_irq_save(flags);
 418         raise_softirq_irqoff(nr);
 419         local_irq_restore(flags);
 420 }
 421
 422 void __raise_softirq_irqoff(unsigned int nr)
 423 {
 424         trace_softirq_raise(nr);
 425         or_softirq_pending(1UL << nr);
 426 }
 427
 428 void open_softirq(int nr, void (*action)(struct softirq_action *))
 429 {
 430         softirq_vec[nr].action = action;
 431 }
 432
 433 /*
 434  * Tasklets
 435  */
 436 struct tasklet_head
 437 {
 438         struct tasklet_struct *head;
 439         struct tasklet_struct **tail;
 440 };
 441
 442 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 443 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 444
 445 void __tasklet_schedule(struct tasklet_struct *t)
 446 {
 447         unsigned long flags;
 448
 449         local_irq_save(flags);
 450         t->next = NULL;
 451         *__this_cpu_read(tasklet_vec.tail) = t;
 452         __this_cpu_write(tasklet_vec.tail, &(t->next));
 453         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 454         local_irq_restore(flags);
 455 }
 456
 457 EXPORT_SYMBOL(__tasklet_schedule);
 458
 459 void __tasklet_hi_schedule(struct tasklet_struct *t)
 460 {
 461         unsigned long flags;
 462
 463         local_irq_save(flags);
 464         t->next = NULL;
 465         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 466         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 467         raise_softirq_irqoff(HI_SOFTIRQ);
 468         local_irq_restore(flags);
 469 }
 470
 471 EXPORT_SYMBOL(__tasklet_hi_schedule);
 472
 473 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 474 {
 475         BUG_ON(!irqs_disabled());
 476
 477         t->next = __this_cpu_read(tasklet_hi_vec.head);
 478         __this_cpu_write(tasklet_hi_vec.head, t);
 479         __raise_softirq_irqoff(HI_SOFTIRQ);
 480 }
 481
 482 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 483
 484 static void tasklet_action(struct softirq_action *a)
 485 {
 486         struct tasklet_struct *list;
 487
 488         local_irq_disable();
 489         list = __this_cpu_read(tasklet_vec.head);
 490         __this_cpu_write(tasklet_vec.head, NULL);
 491         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 492         local_irq_enable();
 493
 494         while (list) {
 495                 struct tasklet_struct *t = list;
 496
 497                 list = list->next;
 498
 499                 if (tasklet_trylock(t)) {
 500                         if (!atomic_read(&t->count)) {
 501                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 502                                         BUG();
 503                                 t->func(t->data);
 504                                 tasklet_unlock(t);
 505                                 continue;
 506                         }
 507                         tasklet_unlock(t);
 508                 }
 509
 510                 local_irq_disable();
 511                 t->next = NULL;
 512                 *__this_cpu_read(tasklet_vec.tail) = t;
 513                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 514                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 515                 local_irq_enable();
 516         }
 517 }
 518
 519 static void tasklet_hi_action(struct softirq_action *a)
 520 {
 521         struct tasklet_struct *list;
 522
 523         local_irq_disable();
 524         list = __this_cpu_read(tasklet_hi_vec.head);
 525         __this_cpu_write(tasklet_hi_vec.head, NULL);
 526         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 527         local_irq_enable();
 528
 529         while (list) {
 530                 struct tasklet_struct *t = list;
 531
 532                 list = list->next;
 533
 534                 if (tasklet_trylock(t)) {
 535                         if (!atomic_read(&t->count)) {
 536                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 537                                         BUG();
 538                                 t->func(t->data);
 539                                 tasklet_unlock(t);
 540                                 continue;
 541                         }
 542                         tasklet_unlock(t);
 543                 }
 544
 545                 local_irq_disable();
 546                 t->next = NULL;
 547                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 548                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 549                 __raise_softirq_irqoff(HI_SOFTIRQ);
 550                 local_irq_enable();
 551         }
 552 }
 553
 554
 555 void tasklet_init(struct tasklet_struct *t,
 556                   void (*func)(unsigned long), unsigned long data)
 557 {
 558         t->next = NULL;
 559         t->state = 0;
 560         atomic_set(&t->count, 0);
 561         t->func = func;
 562         t->data = data;
 563 }
 564
 565 EXPORT_SYMBOL(tasklet_init);
 566
 567 void tasklet_kill(struct tasklet_struct *t)
 568 {
 569         if (in_interrupt())
 570                 printk("Attempt to kill tasklet from interrupt\n");
 571
 572         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 573                 do {
 574                         yield();
 575                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 576         }
 577         tasklet_unlock_wait(t);
 578         clear_bit(TASKLET_STATE_SCHED, &t->state);
 579 }
 580
 581 EXPORT_SYMBOL(tasklet_kill);
 582
 583 /*
 584  * tasklet_hrtimer
 585  */
 586
 587 /*
 588  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 589  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 590  * hrtimer callback, but from softirq context.
 591  */
 592 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 593 {
 594         struct tasklet_hrtimer *ttimer =
 595                 container_of(timer, struct tasklet_hrtimer, timer);
 596
 597         tasklet_hi_schedule(&ttimer->tasklet);
 598         return HRTIMER_NORESTART;
 599 }
 600
 601 /*
 602  * Helper function which calls the hrtimer callback from
 603  * tasklet/softirq context
 604  */
 605 static void __tasklet_hrtimer_trampoline(unsigned long data)
 606 {
 607         struct tasklet_hrtimer *ttimer = (void *)data;
 608         enum hrtimer_restart restart;
 609
 610         restart = ttimer->function(&ttimer->timer);
 611         if (restart != HRTIMER_NORESTART)
 612                 hrtimer_restart(&ttimer->timer);
 613 }
 614
 615 /**
 616  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 617  * @ttimer:      tasklet_hrtimer which is initialized
 618  * @function:    hrtimer callback function which gets called from softirq context
 619  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 620  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 621  */
 622 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 623                           enum hrtimer_restart (*function)(struct hrtimer *),
 624                           clockid_t which_clock, enum hrtimer_mode mode)
 625 {
 626         hrtimer_init(&ttimer->timer, which_clock, mode);
 627         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 628         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 629                      (unsigned long)ttimer);
 630         ttimer->function = function;
 631 }
 632 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 633
 634 void __init softirq_init(void)
 635 {
 636         int cpu;
 637
 638         for_each_possible_cpu(cpu) {
 639                 per_cpu(tasklet_vec, cpu).tail =
 640                         &per_cpu(tasklet_vec, cpu).head;
 641                 per_cpu(tasklet_hi_vec, cpu).tail =
 642                         &per_cpu(tasklet_hi_vec, cpu).head;
 643         }
 644
 645         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 646         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 647 }
 648
 649 static int ksoftirqd_should_run(unsigned int cpu)
 650 {
 651         return local_softirq_pending();
 652 }
 653
 654 static void run_ksoftirqd(unsigned int cpu)
 655 {
 656         local_irq_disable();
 657         if (local_softirq_pending()) {
 658                 /*
 659                  * We can safely run softirq on inline stack, as we are not deep
 660                  * in the task stack here.
 661                  */
 662                 __do_softirq();
 663                 rcu_note_context_switch(cpu);
 664                 local_irq_enable();
 665                 cond_resched();
 666                 return;
 667         }
 668         local_irq_enable();
 669 }
 670
 671 #ifdef CONFIG_HOTPLUG_CPU
 672 /*
 673  * tasklet_kill_immediate is called to remove a tasklet which can already be
 674  * scheduled for execution on @cpu.
 675  *
 676  * Unlike tasklet_kill, this function removes the tasklet
 677  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 678  *
 679  * When this function is called, @cpu must be in the CPU_DEAD state.
 680  */
 681 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 682 {
 683         struct tasklet_struct **i;
 684
 685         BUG_ON(cpu_online(cpu));
 686         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 687
 688         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 689                 return;
 690
 691         /* CPU is dead, so no lock needed. */
 692         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 693                 if (*i == t) {
 694                         *i = t->next;
 695                         /* If this was the tail element, move the tail ptr */
 696                         if (*i == NULL)
 697                                 per_cpu(tasklet_vec, cpu).tail = i;
 698                         return;
 699                 }
 700         }
 701         BUG();
 702 }
 703
 704 static void takeover_tasklets(unsigned int cpu)
 705 {
 706         /* CPU is dead, so no lock needed. */
 707         local_irq_disable();
 708
 709         /* Find end, append list for that CPU. */
 710         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 711                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 712                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 713                 per_cpu(tasklet_vec, cpu).head = NULL;
 714                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 715         }
 716         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 717
 718         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 719                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 720                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 721                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 722                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 723         }
 724         raise_softirq_irqoff(HI_SOFTIRQ);
 725
 726         local_irq_enable();
 727 }
 728 #endif /* CONFIG_HOTPLUG_CPU */
 729
 730 static int cpu_callback(struct notifier_block *nfb,
 731                                   unsigned long action,
 732                                   void *hcpu)
 733 {
 734         switch (action) {
 735 #ifdef CONFIG_HOTPLUG_CPU
 736         case CPU_DEAD:
 737         case CPU_DEAD_FROZEN:
 738                 takeover_tasklets((unsigned long)hcpu);
 739                 break;
 740 #endif /* CONFIG_HOTPLUG_CPU */
 741         }
 742         return NOTIFY_OK;
 743 }
 744
 745 static struct notifier_block cpu_nfb = {
 746         .notifier_call = cpu_callback
 747 };
 748
 749 static struct smp_hotplug_thread softirq_threads = {
 750         .store                  = &ksoftirqd,
 751         .thread_should_run      = ksoftirqd_should_run,
 752         .thread_fn              = run_ksoftirqd,
 753         .thread_comm            = "ksoftirqd/%u",
 754 };
 755
 756 static __init int spawn_ksoftirqd(void)
 757 {
 758         register_cpu_notifier(&cpu_nfb);
 759
 760         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 761
 762         return 0;
 763 }
 764 early_initcall(spawn_ksoftirqd);
 765
 766 /*
 767  * [ These __weak aliases are kept in a separate compilation unit, so that
 768  *   GCC does not inline them incorrectly. ]
 769  */
 770
 771 int __init __weak early_irq_init(void)
 772 {
 773         return 0;
 774 }
 775
 776 int __init __weak arch_probe_nr_irqs(void)
 777 {
 778         return NR_IRQS_LEGACY;
 779 }
 780
 781 int __init __weak arch_early_irq_init(void)
 782 {
 783         return 0;
 784 }