kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  */
  10
  11 #include <linux/export.h>
  12 #include <linux/kernel_stat.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/init.h>
  15 #include <linux/mm.h>
  16 #include <linux/notifier.h>
  17 #include <linux/percpu.h>
  18 #include <linux/cpu.h>
  19 #include <linux/freezer.h>
  20 #include <linux/kthread.h>
  21 #include <linux/rcupdate.h>
  22 #include <linux/ftrace.h>
  23 #include <linux/smp.h>
  24 #include <linux/smpboot.h>
  25 #include <linux/tick.h>
  26
  27 #define CREATE_TRACE_POINTS
  28 #include <trace/events/irq.h>
  29
  30 /*
  31    - No shared variables, all the data are CPU local.
  32    - If a softirq needs serialization, let it serialize itself
  33      by its own spinlocks.
  34    - Even if softirq is serialized, only local cpu is marked for
  35      execution. Hence, we get something sort of weak cpu binding.
  36      Though it is still not clear, will it result in better locality
  37      or will not.
  38
  39    Examples:
  40    - NET RX softirq. It is multithreaded and does not require
  41      any global serialization.
  42    - NET TX softirq. It kicks software netdevice queues, hence
  43      it is logically serialized per device, but this serialization
  44      is invisible to common code.
  45    - Tasklets: serialized wrt itself.
  46  */
  47
  48 #ifndef __ARCH_IRQ_STAT
  49 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  50 EXPORT_SYMBOL(irq_stat);
  51 #endif
  52
  53 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  54
  55 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  56
  57 char *softirq_to_name[NR_SOFTIRQS] = {
  58         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  59         "TASKLET", "SCHED", "HRTIMER", "RCU"
  60 };
  61
  62 /*
  63  * we cannot loop indefinitely here to avoid userspace starvation,
  64  * but we also don't want to introduce a worst case 1/HZ latency
  65  * to the pending events, so lets the scheduler to balance
  66  * the softirq load for us.
  67  */
  68 static void wakeup_softirqd(void)
  69 {
  70         /* Interrupts are disabled: no need to stop preemption */
  71         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  72
  73         if (tsk && tsk->state != TASK_RUNNING)
  74                 wake_up_process(tsk);
  75 }
  76
  77 /*
  78  * preempt_count and SOFTIRQ_OFFSET usage:
  79  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  80  *   softirq processing.
  81  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  82  *   on local_bh_disable or local_bh_enable.
  83  * This lets us distinguish between whether we are currently processing
  84  * softirq and whether we just have bh disabled.
  85  */
  86
  87 /*
  88  * This one is for softirq.c-internal use,
  89  * where hardirqs are disabled legitimately:
  90  */
  91 #ifdef CONFIG_TRACE_IRQFLAGS
  92 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
  93 {
  94         unsigned long flags;
  95
  96         WARN_ON_ONCE(in_irq());
  97
  98         raw_local_irq_save(flags);
  99         /*
 100          * The preempt tracer hooks into preempt_count_add and will break
 101          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 102          * is set and before current->softirq_enabled is cleared.
 103          * We must manually increment preempt_count here and manually
 104          * call the trace_preempt_off later.
 105          */
 106         __preempt_count_add(cnt);
 107         /*
 108          * Were softirqs turned off above:
 109          */
 110         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 111                 trace_softirqs_off(ip);
 112         raw_local_irq_restore(flags);
 113
 114         if (preempt_count() == cnt)
 115                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 116 }
 117 EXPORT_SYMBOL(__local_bh_disable_ip);
 118 #endif /* CONFIG_TRACE_IRQFLAGS */
 119
 120 static void __local_bh_enable(unsigned int cnt)
 121 {
 122         WARN_ON_ONCE(!irqs_disabled());
 123
 124         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 125                 trace_softirqs_on(_RET_IP_);
 126         preempt_count_sub(cnt);
 127 }
 128
 129 /*
 130  * Special-case - softirqs can safely be enabled in
 131  * cond_resched_softirq(), or by __do_softirq(),
 132  * without processing still-pending softirqs:
 133  */
 134 void _local_bh_enable(void)
 135 {
 136         WARN_ON_ONCE(in_irq());
 137         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 138 }
 139
 140 EXPORT_SYMBOL(_local_bh_enable);
 141
 142 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 143 {
 144         WARN_ON_ONCE(in_irq() || irqs_disabled());
 145 #ifdef CONFIG_TRACE_IRQFLAGS
 146         local_irq_disable();
 147 #endif
 148         /*
 149          * Are softirqs going to be turned on now:
 150          */
 151         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 152                 trace_softirqs_on(ip);
 153         /*
 154          * Keep preemption disabled until we are done with
 155          * softirq processing:
 156          */
 157         preempt_count_sub(cnt - 1);
 158
 159         if (unlikely(!in_interrupt() && local_softirq_pending())) {
 160                 /*
 161                  * Run softirq if any pending. And do it in its own stack
 162                  * as we may be calling this deep in a task call stack already.
 163                  */
 164                 do_softirq();
 165         }
 166
 167         preempt_count_dec();
 168 #ifdef CONFIG_TRACE_IRQFLAGS
 169         local_irq_enable();
 170 #endif
 171         preempt_check_resched();
 172 }
 173 EXPORT_SYMBOL(__local_bh_enable_ip);
 174
 175 /*
 176  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 177  * but break the loop if need_resched() is set or after 2 ms.
 178  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 179  * certain cases, such as stop_machine(), jiffies may cease to
 180  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 181  * well to make sure we eventually return from this method.
 182  *
 183  * These limits have been established via experimentation.
 184  * The two things to balance is latency against fairness -
 185  * we want to handle softirqs as soon as possible, but they
 186  * should not be able to lock up the box.
 187  */
 188 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 189 #define MAX_SOFTIRQ_RESTART 10
 190
 191 #ifdef CONFIG_TRACE_IRQFLAGS
 192 /*
 193  * When we run softirqs from irq_exit() and thus on the hardirq stack we need
 194  * to keep the lockdep irq context tracking as tight as possible in order to
 195  * not miss-qualify lock contexts and miss possible deadlocks.
 196  */
 197
 198 static inline bool lockdep_softirq_start(void)
 199 {
 200         bool in_hardirq = false;
 201
 202         if (trace_hardirq_context(current)) {
 203                 in_hardirq = true;
 204                 trace_hardirq_exit();
 205         }
 206
 207         lockdep_softirq_enter();
 208
 209         return in_hardirq;
 210 }
 211
 212 static inline void lockdep_softirq_end(bool in_hardirq)
 213 {
 214         lockdep_softirq_exit();
 215
 216         if (in_hardirq)
 217                 trace_hardirq_enter();
 218 }
 219 #else
 220 static inline bool lockdep_softirq_start(void) { return false; }
 221 static inline void lockdep_softirq_end(bool in_hardirq) { }
 222 #endif
 223
 224 asmlinkage void __do_softirq(void)
 225 {
 226         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 227         unsigned long old_flags = current->flags;
 228         int max_restart = MAX_SOFTIRQ_RESTART;
 229         struct softirq_action *h;
 230         bool in_hardirq;
 231         __u32 pending;
 232         int cpu;
 233
 234         /*
 235          * Mask out PF_MEMALLOC s current task context is borrowed for the
 236          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 237          * again if the socket is related to swap
 238          */
 239         current->flags &= ~PF_MEMALLOC;
 240
 241         pending = local_softirq_pending();
 242         account_irq_enter_time(current);
 243
 244         __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 245         in_hardirq = lockdep_softirq_start();
 246
 247         cpu = smp_processor_id();
 248 restart:
 249         /* Reset the pending bitmask before enabling irqs */
 250         set_softirq_pending(0);
 251
 252         local_irq_enable();
 253
 254         h = softirq_vec;
 255
 256         do {
 257                 if (pending & 1) {
 258                         unsigned int vec_nr = h - softirq_vec;
 259                         int prev_count = preempt_count();
 260
 261                         kstat_incr_softirqs_this_cpu(vec_nr);
 262
 263                         trace_softirq_entry(vec_nr);
 264                         h->action(h);
 265                         trace_softirq_exit(vec_nr);
 266                         if (unlikely(prev_count != preempt_count())) {
 267                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 268                                        "with preempt_count %08x,"
 269                                        " exited with %08x?\n", vec_nr,
 270                                        softirq_to_name[vec_nr], h->action,
 271                                        prev_count, preempt_count());
 272                                 preempt_count_set(prev_count);
 273                         }
 274
 275                         rcu_bh_qs(cpu);
 276                 }
 277                 h++;
 278                 pending >>= 1;
 279         } while (pending);
 280
 281         local_irq_disable();
 282
 283         pending = local_softirq_pending();
 284         if (pending) {
 285                 if (time_before(jiffies, end) && !need_resched() &&
 286                     --max_restart)
 287                         goto restart;
 288
 289                 wakeup_softirqd();
 290         }
 291
 292         lockdep_softirq_end(in_hardirq);
 293         account_irq_exit_time(current);
 294         __local_bh_enable(SOFTIRQ_OFFSET);
 295         WARN_ON_ONCE(in_interrupt());
 296         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 297 }
 298
 299 asmlinkage void do_softirq(void)
 300 {
 301         __u32 pending;
 302         unsigned long flags;
 303
 304         if (in_interrupt())
 305                 return;
 306
 307         local_irq_save(flags);
 308
 309         pending = local_softirq_pending();
 310
 311         if (pending)
 312                 do_softirq_own_stack();
 313
 314         local_irq_restore(flags);
 315 }
 316
 317 /*
 318  * Enter an interrupt context.
 319  */
 320 void irq_enter(void)
 321 {
 322         int cpu = smp_processor_id();
 323
 324         rcu_irq_enter();
 325         if (is_idle_task(current) && !in_interrupt()) {
 326                 /*
 327                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 328                  * here, as softirq will be serviced on return from interrupt.
 329                  */
 330                 local_bh_disable();
 331                 tick_check_idle(cpu);
 332                 _local_bh_enable();
 333         }
 334
 335         __irq_enter();
 336 }
 337
 338 static inline void invoke_softirq(void)
 339 {
 340         if (!force_irqthreads) {
 341 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 342                 /*
 343                  * We can safely execute softirq on the current stack if
 344                  * it is the irq stack, because it should be near empty
 345                  * at this stage.
 346                  */
 347                 __do_softirq();
 348 #else
 349                 /*
 350                  * Otherwise, irq_exit() is called on the task stack that can
 351                  * be potentially deep already. So call softirq in its own stack
 352                  * to prevent from any overrun.
 353                  */
 354                 do_softirq_own_stack();
 355 #endif
 356         } else {
 357                 wakeup_softirqd();
 358         }
 359 }
 360
 361 static inline void tick_irq_exit(void)
 362 {
 363 #ifdef CONFIG_NO_HZ_COMMON
 364         int cpu = smp_processor_id();
 365
 366         /* Make sure that timer wheel updates are propagated */
 367         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 368                 if (!in_interrupt())
 369                         tick_nohz_irq_exit();
 370         }
 371 #endif
 372 }
 373
 374 /*
 375  * Exit an interrupt context. Process softirqs if needed and possible:
 376  */
 377 void irq_exit(void)
 378 {
 379 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 380         local_irq_disable();
 381 #else
 382         WARN_ON_ONCE(!irqs_disabled());
 383 #endif
 384
 385         account_irq_exit_time(current);
 386         preempt_count_sub(HARDIRQ_OFFSET);
 387         if (!in_interrupt() && local_softirq_pending())
 388                 invoke_softirq();
 389
 390         tick_irq_exit();
 391         rcu_irq_exit();
 392         trace_hardirq_exit(); /* must be last! */
 393 }
 394
 395 /*
 396  * This function must run with irqs disabled!
 397  */
 398 inline void raise_softirq_irqoff(unsigned int nr)
 399 {
 400         __raise_softirq_irqoff(nr);
 401
 402         /*
 403          * If we're in an interrupt or softirq, we're done
 404          * (this also catches softirq-disabled code). We will
 405          * actually run the softirq once we return from
 406          * the irq or softirq.
 407          *
 408          * Otherwise we wake up ksoftirqd to make sure we
 409          * schedule the softirq soon.
 410          */
 411         if (!in_interrupt())
 412                 wakeup_softirqd();
 413 }
 414
 415 void raise_softirq(unsigned int nr)
 416 {
 417         unsigned long flags;
 418
 419         local_irq_save(flags);
 420         raise_softirq_irqoff(nr);
 421         local_irq_restore(flags);
 422 }
 423
 424 void __raise_softirq_irqoff(unsigned int nr)
 425 {
 426         trace_softirq_raise(nr);
 427         or_softirq_pending(1UL << nr);
 428 }
 429
 430 void open_softirq(int nr, void (*action)(struct softirq_action *))
 431 {
 432         softirq_vec[nr].action = action;
 433 }
 434
 435 /*
 436  * Tasklets
 437  */
 438 struct tasklet_head
 439 {
 440         struct tasklet_struct *head;
 441         struct tasklet_struct **tail;
 442 };
 443
 444 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 445 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 446
 447 void __tasklet_schedule(struct tasklet_struct *t)
 448 {
 449         unsigned long flags;
 450
 451         local_irq_save(flags);
 452         t->next = NULL;
 453         *__this_cpu_read(tasklet_vec.tail) = t;
 454         __this_cpu_write(tasklet_vec.tail, &(t->next));
 455         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 456         local_irq_restore(flags);
 457 }
 458
 459 EXPORT_SYMBOL(__tasklet_schedule);
 460
 461 void __tasklet_hi_schedule(struct tasklet_struct *t)
 462 {
 463         unsigned long flags;
 464
 465         local_irq_save(flags);
 466         t->next = NULL;
 467         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 468         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 469         raise_softirq_irqoff(HI_SOFTIRQ);
 470         local_irq_restore(flags);
 471 }
 472
 473 EXPORT_SYMBOL(__tasklet_hi_schedule);
 474
 475 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 476 {
 477         BUG_ON(!irqs_disabled());
 478
 479         t->next = __this_cpu_read(tasklet_hi_vec.head);
 480         __this_cpu_write(tasklet_hi_vec.head, t);
 481         __raise_softirq_irqoff(HI_SOFTIRQ);
 482 }
 483
 484 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 485
 486 static void tasklet_action(struct softirq_action *a)
 487 {
 488         struct tasklet_struct *list;
 489
 490         local_irq_disable();
 491         list = __this_cpu_read(tasklet_vec.head);
 492         __this_cpu_write(tasklet_vec.head, NULL);
 493         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 494         local_irq_enable();
 495
 496         while (list) {
 497                 struct tasklet_struct *t = list;
 498
 499                 list = list->next;
 500
 501                 if (tasklet_trylock(t)) {
 502                         if (!atomic_read(&t->count)) {
 503                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 504                                         BUG();
 505                                 t->func(t->data);
 506                                 tasklet_unlock(t);
 507                                 continue;
 508                         }
 509                         tasklet_unlock(t);
 510                 }
 511
 512                 local_irq_disable();
 513                 t->next = NULL;
 514                 *__this_cpu_read(tasklet_vec.tail) = t;
 515                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 516                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 517                 local_irq_enable();
 518         }
 519 }
 520
 521 static void tasklet_hi_action(struct softirq_action *a)
 522 {
 523         struct tasklet_struct *list;
 524
 525         local_irq_disable();
 526         list = __this_cpu_read(tasklet_hi_vec.head);
 527         __this_cpu_write(tasklet_hi_vec.head, NULL);
 528         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 529         local_irq_enable();
 530
 531         while (list) {
 532                 struct tasklet_struct *t = list;
 533
 534                 list = list->next;
 535
 536                 if (tasklet_trylock(t)) {
 537                         if (!atomic_read(&t->count)) {
 538                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 539                                         BUG();
 540                                 t->func(t->data);
 541                                 tasklet_unlock(t);
 542                                 continue;
 543                         }
 544                         tasklet_unlock(t);
 545                 }
 546
 547                 local_irq_disable();
 548                 t->next = NULL;
 549                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 550                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 551                 __raise_softirq_irqoff(HI_SOFTIRQ);
 552                 local_irq_enable();
 553         }
 554 }
 555
 556
 557 void tasklet_init(struct tasklet_struct *t,
 558                   void (*func)(unsigned long), unsigned long data)
 559 {
 560         t->next = NULL;
 561         t->state = 0;
 562         atomic_set(&t->count, 0);
 563         t->func = func;
 564         t->data = data;
 565 }
 566
 567 EXPORT_SYMBOL(tasklet_init);
 568
 569 void tasklet_kill(struct tasklet_struct *t)
 570 {
 571         if (in_interrupt())
 572                 printk("Attempt to kill tasklet from interrupt\n");
 573
 574         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 575                 do {
 576                         yield();
 577                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 578         }
 579         tasklet_unlock_wait(t);
 580         clear_bit(TASKLET_STATE_SCHED, &t->state);
 581 }
 582
 583 EXPORT_SYMBOL(tasklet_kill);
 584
 585 /*
 586  * tasklet_hrtimer
 587  */
 588
 589 /*
 590  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 591  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 592  * hrtimer callback, but from softirq context.
 593  */
 594 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 595 {
 596         struct tasklet_hrtimer *ttimer =
 597                 container_of(timer, struct tasklet_hrtimer, timer);
 598
 599         tasklet_hi_schedule(&ttimer->tasklet);
 600         return HRTIMER_NORESTART;
 601 }
 602
 603 /*
 604  * Helper function which calls the hrtimer callback from
 605  * tasklet/softirq context
 606  */
 607 static void __tasklet_hrtimer_trampoline(unsigned long data)
 608 {
 609         struct tasklet_hrtimer *ttimer = (void *)data;
 610         enum hrtimer_restart restart;
 611
 612         restart = ttimer->function(&ttimer->timer);
 613         if (restart != HRTIMER_NORESTART)
 614                 hrtimer_restart(&ttimer->timer);
 615 }
 616
 617 /**
 618  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 619  * @ttimer:      tasklet_hrtimer which is initialized
 620  * @function:    hrtimer callback function which gets called from softirq context
 621  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 622  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 623  */
 624 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 625                           enum hrtimer_restart (*function)(struct hrtimer *),
 626                           clockid_t which_clock, enum hrtimer_mode mode)
 627 {
 628         hrtimer_init(&ttimer->timer, which_clock, mode);
 629         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 630         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 631                      (unsigned long)ttimer);
 632         ttimer->function = function;
 633 }
 634 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 635
 636 void __init softirq_init(void)
 637 {
 638         int cpu;
 639
 640         for_each_possible_cpu(cpu) {
 641                 per_cpu(tasklet_vec, cpu).tail =
 642                         &per_cpu(tasklet_vec, cpu).head;
 643                 per_cpu(tasklet_hi_vec, cpu).tail =
 644                         &per_cpu(tasklet_hi_vec, cpu).head;
 645         }
 646
 647         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 648         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 649 }
 650
 651 static int ksoftirqd_should_run(unsigned int cpu)
 652 {
 653         return local_softirq_pending();
 654 }
 655
 656 static void run_ksoftirqd(unsigned int cpu)
 657 {
 658         local_irq_disable();
 659         if (local_softirq_pending()) {
 660                 /*
 661                  * We can safely run softirq on inline stack, as we are not deep
 662                  * in the task stack here.
 663                  */
 664                 __do_softirq();
 665                 rcu_note_context_switch(cpu);
 666                 local_irq_enable();
 667                 cond_resched();
 668                 return;
 669         }
 670         local_irq_enable();
 671 }
 672
 673 #ifdef CONFIG_HOTPLUG_CPU
 674 /*
 675  * tasklet_kill_immediate is called to remove a tasklet which can already be
 676  * scheduled for execution on @cpu.
 677  *
 678  * Unlike tasklet_kill, this function removes the tasklet
 679  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 680  *
 681  * When this function is called, @cpu must be in the CPU_DEAD state.
 682  */
 683 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 684 {
 685         struct tasklet_struct **i;
 686
 687         BUG_ON(cpu_online(cpu));
 688         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 689
 690         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 691                 return;
 692
 693         /* CPU is dead, so no lock needed. */
 694         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 695                 if (*i == t) {
 696                         *i = t->next;
 697                         /* If this was the tail element, move the tail ptr */
 698                         if (*i == NULL)
 699                                 per_cpu(tasklet_vec, cpu).tail = i;
 700                         return;
 701                 }
 702         }
 703         BUG();
 704 }
 705
 706 static void takeover_tasklets(unsigned int cpu)
 707 {
 708         /* CPU is dead, so no lock needed. */
 709         local_irq_disable();
 710
 711         /* Find end, append list for that CPU. */
 712         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 713                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 714                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 715                 per_cpu(tasklet_vec, cpu).head = NULL;
 716                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 717         }
 718         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 719
 720         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 721                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 722                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 723                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 724                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 725         }
 726         raise_softirq_irqoff(HI_SOFTIRQ);
 727
 728         local_irq_enable();
 729 }
 730 #endif /* CONFIG_HOTPLUG_CPU */
 731
 732 static int cpu_callback(struct notifier_block *nfb,
 733                                   unsigned long action,
 734                                   void *hcpu)
 735 {
 736         switch (action) {
 737 #ifdef CONFIG_HOTPLUG_CPU
 738         case CPU_DEAD:
 739         case CPU_DEAD_FROZEN:
 740                 takeover_tasklets((unsigned long)hcpu);
 741                 break;
 742 #endif /* CONFIG_HOTPLUG_CPU */
 743         }
 744         return NOTIFY_OK;
 745 }
 746
 747 static struct notifier_block cpu_nfb = {
 748         .notifier_call = cpu_callback
 749 };
 750
 751 static struct smp_hotplug_thread softirq_threads = {
 752         .store                  = &ksoftirqd,
 753         .thread_should_run      = ksoftirqd_should_run,
 754         .thread_fn              = run_ksoftirqd,
 755         .thread_comm            = "ksoftirqd/%u",
 756 };
 757
 758 static __init int spawn_ksoftirqd(void)
 759 {
 760         register_cpu_notifier(&cpu_nfb);
 761
 762         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 763
 764         return 0;
 765 }
 766 early_initcall(spawn_ksoftirqd);
 767
 768 /*
 769  * [ These __weak aliases are kept in a separate compilation unit, so that
 770  *   GCC does not inline them incorrectly. ]
 771  */
 772
 773 int __init __weak early_irq_init(void)
 774 {
 775         return 0;
 776 }
 777
 778 int __init __weak arch_probe_nr_irqs(void)
 779 {
 780         return NR_IRQS_LEGACY;
 781 }
 782
 783 int __init __weak arch_early_irq_init(void)
 784 {
 785         return 0;
 786 }