kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  */
  10
  11 #include <linux/export.h>
  12 #include <linux/kernel_stat.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/init.h>
  15 #include <linux/mm.h>
  16 #include <linux/notifier.h>
  17 #include <linux/percpu.h>
  18 #include <linux/cpu.h>
  19 #include <linux/freezer.h>
  20 #include <linux/kthread.h>
  21 #include <linux/rcupdate.h>
  22 #include <linux/ftrace.h>
  23 #include <linux/smp.h>
  24 #include <linux/smpboot.h>
  25 #include <linux/tick.h>
  26
  27 #define CREATE_TRACE_POINTS
  28 #include <trace/events/irq.h>
  29
  30 /*
  31    - No shared variables, all the data are CPU local.
  32    - If a softirq needs serialization, let it serialize itself
  33      by its own spinlocks.
  34    - Even if softirq is serialized, only local cpu is marked for
  35      execution. Hence, we get something sort of weak cpu binding.
  36      Though it is still not clear, will it result in better locality
  37      or will not.
  38
  39    Examples:
  40    - NET RX softirq. It is multithreaded and does not require
  41      any global serialization.
  42    - NET TX softirq. It kicks software netdevice queues, hence
  43      it is logically serialized per device, but this serialization
  44      is invisible to common code.
  45    - Tasklets: serialized wrt itself.
  46  */
  47
  48 #ifndef __ARCH_IRQ_STAT
  49 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  50 EXPORT_SYMBOL(irq_stat);
  51 #endif
  52
  53 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  54
  55 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  56
  57 char *softirq_to_name[NR_SOFTIRQS] = {
  58         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  59         "TASKLET", "SCHED", "HRTIMER", "RCU"
  60 };
  61
  62 /*
  63  * we cannot loop indefinitely here to avoid userspace starvation,
  64  * but we also don't want to introduce a worst case 1/HZ latency
  65  * to the pending events, so lets the scheduler to balance
  66  * the softirq load for us.
  67  */
  68 static void wakeup_softirqd(void)
  69 {
  70         /* Interrupts are disabled: no need to stop preemption */
  71         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  72
  73         if (tsk && tsk->state != TASK_RUNNING)
  74                 wake_up_process(tsk);
  75 }
  76
  77 /*
  78  * preempt_count and SOFTIRQ_OFFSET usage:
  79  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  80  *   softirq processing.
  81  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  82  *   on local_bh_disable or local_bh_enable.
  83  * This lets us distinguish between whether we are currently processing
  84  * softirq and whether we just have bh disabled.
  85  */
  86
  87 /*
  88  * This one is for softirq.c-internal use,
  89  * where hardirqs are disabled legitimately:
  90  */
  91 #ifdef CONFIG_TRACE_IRQFLAGS
  92 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  93 {
  94         unsigned long flags;
  95
  96         WARN_ON_ONCE(in_irq());
  97
  98         raw_local_irq_save(flags);
  99         /*
 100          * The preempt tracer hooks into preempt_count_add and will break
 101          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 102          * is set and before current->softirq_enabled is cleared.
 103          * We must manually increment preempt_count here and manually
 104          * call the trace_preempt_off later.
 105          */
 106         __preempt_count_add(cnt);
 107         /*
 108          * Were softirqs turned off above:
 109          */
 110         if (softirq_count() == cnt)
 111                 trace_softirqs_off(ip);
 112         raw_local_irq_restore(flags);
 113
 114         if (preempt_count() == cnt)
 115                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 116 }
 117 #else /* !CONFIG_TRACE_IRQFLAGS */
 118 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 119 {
 120         preempt_count_add(cnt);
 121         barrier();
 122 }
 123 #endif /* CONFIG_TRACE_IRQFLAGS */
 124
 125 void local_bh_disable(void)
 126 {
 127         __local_bh_disable(_RET_IP_, SOFTIRQ_DISABLE_OFFSET);
 128 }
 129
 130 EXPORT_SYMBOL(local_bh_disable);
 131
 132 static void __local_bh_enable(unsigned int cnt)
 133 {
 134         WARN_ON_ONCE(!irqs_disabled());
 135
 136         if (softirq_count() == cnt)
 137                 trace_softirqs_on(_RET_IP_);
 138         preempt_count_sub(cnt);
 139 }
 140
 141 /*
 142  * Special-case - softirqs can safely be enabled in
 143  * cond_resched_softirq(), or by __do_softirq(),
 144  * without processing still-pending softirqs:
 145  */
 146 void _local_bh_enable(void)
 147 {
 148         WARN_ON_ONCE(in_irq());
 149         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 150 }
 151
 152 EXPORT_SYMBOL(_local_bh_enable);
 153
 154 static inline void _local_bh_enable_ip(unsigned long ip)
 155 {
 156         WARN_ON_ONCE(in_irq() || irqs_disabled());
 157 #ifdef CONFIG_TRACE_IRQFLAGS
 158         local_irq_disable();
 159 #endif
 160         /*
 161          * Are softirqs going to be turned on now:
 162          */
 163         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 164                 trace_softirqs_on(ip);
 165         /*
 166          * Keep preemption disabled until we are done with
 167          * softirq processing:
 168          */
 169         preempt_count_sub(SOFTIRQ_DISABLE_OFFSET - 1);
 170
 171         if (unlikely(!in_interrupt() && local_softirq_pending())) {
 172                 /*
 173                  * Run softirq if any pending. And do it in its own stack
 174                  * as we may be calling this deep in a task call stack already.
 175                  */
 176                 do_softirq();
 177         }
 178
 179         preempt_count_dec();
 180 #ifdef CONFIG_TRACE_IRQFLAGS
 181         local_irq_enable();
 182 #endif
 183         preempt_check_resched();
 184 }
 185
 186 void local_bh_enable(void)
 187 {
 188         _local_bh_enable_ip(_RET_IP_);
 189 }
 190 EXPORT_SYMBOL(local_bh_enable);
 191
 192 void local_bh_enable_ip(unsigned long ip)
 193 {
 194         _local_bh_enable_ip(ip);
 195 }
 196 EXPORT_SYMBOL(local_bh_enable_ip);
 197
 198 /*
 199  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 200  * but break the loop if need_resched() is set or after 2 ms.
 201  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 202  * certain cases, such as stop_machine(), jiffies may cease to
 203  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 204  * well to make sure we eventually return from this method.
 205  *
 206  * These limits have been established via experimentation.
 207  * The two things to balance is latency against fairness -
 208  * we want to handle softirqs as soon as possible, but they
 209  * should not be able to lock up the box.
 210  */
 211 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 212 #define MAX_SOFTIRQ_RESTART 10
 213
 214 asmlinkage void __do_softirq(void)
 215 {
 216         struct softirq_action *h;
 217         __u32 pending;
 218         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 219         int cpu;
 220         unsigned long old_flags = current->flags;
 221         int max_restart = MAX_SOFTIRQ_RESTART;
 222
 223         /*
 224          * Mask out PF_MEMALLOC s current task context is borrowed for the
 225          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 226          * again if the socket is related to swap
 227          */
 228         current->flags &= ~PF_MEMALLOC;
 229
 230         pending = local_softirq_pending();
 231         account_irq_enter_time(current);
 232
 233         __local_bh_disable(_RET_IP_, SOFTIRQ_OFFSET);
 234         lockdep_softirq_enter();
 235
 236         cpu = smp_processor_id();
 237 restart:
 238         /* Reset the pending bitmask before enabling irqs */
 239         set_softirq_pending(0);
 240
 241         local_irq_enable();
 242
 243         h = softirq_vec;
 244
 245         do {
 246                 if (pending & 1) {
 247                         unsigned int vec_nr = h - softirq_vec;
 248                         int prev_count = preempt_count();
 249
 250                         kstat_incr_softirqs_this_cpu(vec_nr);
 251
 252                         trace_softirq_entry(vec_nr);
 253                         h->action(h);
 254                         trace_softirq_exit(vec_nr);
 255                         if (unlikely(prev_count != preempt_count())) {
 256                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 257                                        "with preempt_count %08x,"
 258                                        " exited with %08x?\n", vec_nr,
 259                                        softirq_to_name[vec_nr], h->action,
 260                                        prev_count, preempt_count());
 261                                 preempt_count_set(prev_count);
 262                         }
 263
 264                         rcu_bh_qs(cpu);
 265                 }
 266                 h++;
 267                 pending >>= 1;
 268         } while (pending);
 269
 270         local_irq_disable();
 271
 272         pending = local_softirq_pending();
 273         if (pending) {
 274                 if (time_before(jiffies, end) && !need_resched() &&
 275                     --max_restart)
 276                         goto restart;
 277
 278                 wakeup_softirqd();
 279         }
 280
 281         lockdep_softirq_exit();
 282
 283         account_irq_exit_time(current);
 284         __local_bh_enable(SOFTIRQ_OFFSET);
 285         WARN_ON_ONCE(in_interrupt());
 286         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 287 }
 288
 289
 290
 291 asmlinkage void do_softirq(void)
 292 {
 293         __u32 pending;
 294         unsigned long flags;
 295
 296         if (in_interrupt())
 297                 return;
 298
 299         local_irq_save(flags);
 300
 301         pending = local_softirq_pending();
 302
 303         if (pending)
 304                 do_softirq_own_stack();
 305
 306         local_irq_restore(flags);
 307 }
 308
 309 /*
 310  * Enter an interrupt context.
 311  */
 312 void irq_enter(void)
 313 {
 314         int cpu = smp_processor_id();
 315
 316         rcu_irq_enter();
 317         if (is_idle_task(current) && !in_interrupt()) {
 318                 /*
 319                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 320                  * here, as softirq will be serviced on return from interrupt.
 321                  */
 322                 local_bh_disable();
 323                 tick_check_idle(cpu);
 324                 _local_bh_enable();
 325         }
 326
 327         __irq_enter();
 328 }
 329
 330 static inline void invoke_softirq(void)
 331 {
 332         if (!force_irqthreads) {
 333 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 334                 /*
 335                  * We can safely execute softirq on the current stack if
 336                  * it is the irq stack, because it should be near empty
 337                  * at this stage.
 338                  */
 339                 __do_softirq();
 340 #else
 341                 /*
 342                  * Otherwise, irq_exit() is called on the task stack that can
 343                  * be potentially deep already. So call softirq in its own stack
 344                  * to prevent from any overrun.
 345                  */
 346                 do_softirq_own_stack();
 347 #endif
 348         } else {
 349                 wakeup_softirqd();
 350         }
 351 }
 352
 353 static inline void tick_irq_exit(void)
 354 {
 355 #ifdef CONFIG_NO_HZ_COMMON
 356         int cpu = smp_processor_id();
 357
 358         /* Make sure that timer wheel updates are propagated */
 359         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 360                 if (!in_interrupt())
 361                         tick_nohz_irq_exit();
 362         }
 363 #endif
 364 }
 365
 366 /*
 367  * Exit an interrupt context. Process softirqs if needed and possible:
 368  */
 369 void irq_exit(void)
 370 {
 371 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 372         local_irq_disable();
 373 #else
 374         WARN_ON_ONCE(!irqs_disabled());
 375 #endif
 376
 377         account_irq_exit_time(current);
 378         trace_hardirq_exit();
 379         preempt_count_sub(HARDIRQ_OFFSET);
 380         if (!in_interrupt() && local_softirq_pending())
 381                 invoke_softirq();
 382
 383         tick_irq_exit();
 384         rcu_irq_exit();
 385 }
 386
 387 /*
 388  * This function must run with irqs disabled!
 389  */
 390 inline void raise_softirq_irqoff(unsigned int nr)
 391 {
 392         __raise_softirq_irqoff(nr);
 393
 394         /*
 395          * If we're in an interrupt or softirq, we're done
 396          * (this also catches softirq-disabled code). We will
 397          * actually run the softirq once we return from
 398          * the irq or softirq.
 399          *
 400          * Otherwise we wake up ksoftirqd to make sure we
 401          * schedule the softirq soon.
 402          */
 403         if (!in_interrupt())
 404                 wakeup_softirqd();
 405 }
 406
 407 void raise_softirq(unsigned int nr)
 408 {
 409         unsigned long flags;
 410
 411         local_irq_save(flags);
 412         raise_softirq_irqoff(nr);
 413         local_irq_restore(flags);
 414 }
 415
 416 void __raise_softirq_irqoff(unsigned int nr)
 417 {
 418         trace_softirq_raise(nr);
 419         or_softirq_pending(1UL << nr);
 420 }
 421
 422 void open_softirq(int nr, void (*action)(struct softirq_action *))
 423 {
 424         softirq_vec[nr].action = action;
 425 }
 426
 427 /*
 428  * Tasklets
 429  */
 430 struct tasklet_head
 431 {
 432         struct tasklet_struct *head;
 433         struct tasklet_struct **tail;
 434 };
 435
 436 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 437 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 438
 439 void __tasklet_schedule(struct tasklet_struct *t)
 440 {
 441         unsigned long flags;
 442
 443         local_irq_save(flags);
 444         t->next = NULL;
 445         *__this_cpu_read(tasklet_vec.tail) = t;
 446         __this_cpu_write(tasklet_vec.tail, &(t->next));
 447         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 448         local_irq_restore(flags);
 449 }
 450
 451 EXPORT_SYMBOL(__tasklet_schedule);
 452
 453 void __tasklet_hi_schedule(struct tasklet_struct *t)
 454 {
 455         unsigned long flags;
 456
 457         local_irq_save(flags);
 458         t->next = NULL;
 459         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 460         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 461         raise_softirq_irqoff(HI_SOFTIRQ);
 462         local_irq_restore(flags);
 463 }
 464
 465 EXPORT_SYMBOL(__tasklet_hi_schedule);
 466
 467 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 468 {
 469         BUG_ON(!irqs_disabled());
 470
 471         t->next = __this_cpu_read(tasklet_hi_vec.head);
 472         __this_cpu_write(tasklet_hi_vec.head, t);
 473         __raise_softirq_irqoff(HI_SOFTIRQ);
 474 }
 475
 476 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 477
 478 static void tasklet_action(struct softirq_action *a)
 479 {
 480         struct tasklet_struct *list;
 481
 482         local_irq_disable();
 483         list = __this_cpu_read(tasklet_vec.head);
 484         __this_cpu_write(tasklet_vec.head, NULL);
 485         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 486         local_irq_enable();
 487
 488         while (list) {
 489                 struct tasklet_struct *t = list;
 490
 491                 list = list->next;
 492
 493                 if (tasklet_trylock(t)) {
 494                         if (!atomic_read(&t->count)) {
 495                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 496                                         BUG();
 497                                 t->func(t->data);
 498                                 tasklet_unlock(t);
 499                                 continue;
 500                         }
 501                         tasklet_unlock(t);
 502                 }
 503
 504                 local_irq_disable();
 505                 t->next = NULL;
 506                 *__this_cpu_read(tasklet_vec.tail) = t;
 507                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 508                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 509                 local_irq_enable();
 510         }
 511 }
 512
 513 static void tasklet_hi_action(struct softirq_action *a)
 514 {
 515         struct tasklet_struct *list;
 516
 517         local_irq_disable();
 518         list = __this_cpu_read(tasklet_hi_vec.head);
 519         __this_cpu_write(tasklet_hi_vec.head, NULL);
 520         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 521         local_irq_enable();
 522
 523         while (list) {
 524                 struct tasklet_struct *t = list;
 525
 526                 list = list->next;
 527
 528                 if (tasklet_trylock(t)) {
 529                         if (!atomic_read(&t->count)) {
 530                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 531                                         BUG();
 532                                 t->func(t->data);
 533                                 tasklet_unlock(t);
 534                                 continue;
 535                         }
 536                         tasklet_unlock(t);
 537                 }
 538
 539                 local_irq_disable();
 540                 t->next = NULL;
 541                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 542                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 543                 __raise_softirq_irqoff(HI_SOFTIRQ);
 544                 local_irq_enable();
 545         }
 546 }
 547
 548
 549 void tasklet_init(struct tasklet_struct *t,
 550                   void (*func)(unsigned long), unsigned long data)
 551 {
 552         t->next = NULL;
 553         t->state = 0;
 554         atomic_set(&t->count, 0);
 555         t->func = func;
 556         t->data = data;
 557 }
 558
 559 EXPORT_SYMBOL(tasklet_init);
 560
 561 void tasklet_kill(struct tasklet_struct *t)
 562 {
 563         if (in_interrupt())
 564                 printk("Attempt to kill tasklet from interrupt\n");
 565
 566         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 567                 do {
 568                         yield();
 569                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 570         }
 571         tasklet_unlock_wait(t);
 572         clear_bit(TASKLET_STATE_SCHED, &t->state);
 573 }
 574
 575 EXPORT_SYMBOL(tasklet_kill);
 576
 577 /*
 578  * tasklet_hrtimer
 579  */
 580
 581 /*
 582  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 583  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 584  * hrtimer callback, but from softirq context.
 585  */
 586 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 587 {
 588         struct tasklet_hrtimer *ttimer =
 589                 container_of(timer, struct tasklet_hrtimer, timer);
 590
 591         tasklet_hi_schedule(&ttimer->tasklet);
 592         return HRTIMER_NORESTART;
 593 }
 594
 595 /*
 596  * Helper function which calls the hrtimer callback from
 597  * tasklet/softirq context
 598  */
 599 static void __tasklet_hrtimer_trampoline(unsigned long data)
 600 {
 601         struct tasklet_hrtimer *ttimer = (void *)data;
 602         enum hrtimer_restart restart;
 603
 604         restart = ttimer->function(&ttimer->timer);
 605         if (restart != HRTIMER_NORESTART)
 606                 hrtimer_restart(&ttimer->timer);
 607 }
 608
 609 /**
 610  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 611  * @ttimer:      tasklet_hrtimer which is initialized
 612  * @function:    hrtimer callback function which gets called from softirq context
 613  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 614  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 615  */
 616 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 617                           enum hrtimer_restart (*function)(struct hrtimer *),
 618                           clockid_t which_clock, enum hrtimer_mode mode)
 619 {
 620         hrtimer_init(&ttimer->timer, which_clock, mode);
 621         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 622         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 623                      (unsigned long)ttimer);
 624         ttimer->function = function;
 625 }
 626 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 627
 628 void __init softirq_init(void)
 629 {
 630         int cpu;
 631
 632         for_each_possible_cpu(cpu) {
 633                 per_cpu(tasklet_vec, cpu).tail =
 634                         &per_cpu(tasklet_vec, cpu).head;
 635                 per_cpu(tasklet_hi_vec, cpu).tail =
 636                         &per_cpu(tasklet_hi_vec, cpu).head;
 637         }
 638
 639         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 640         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 641 }
 642
 643 static int ksoftirqd_should_run(unsigned int cpu)
 644 {
 645         return local_softirq_pending();
 646 }
 647
 648 static void run_ksoftirqd(unsigned int cpu)
 649 {
 650         local_irq_disable();
 651         if (local_softirq_pending()) {
 652                 /*
 653                  * We can safely run softirq on inline stack, as we are not deep
 654                  * in the task stack here.
 655                  */
 656                 __do_softirq();
 657                 rcu_note_context_switch(cpu);
 658                 local_irq_enable();
 659                 cond_resched();
 660                 return;
 661         }
 662         local_irq_enable();
 663 }
 664
 665 #ifdef CONFIG_HOTPLUG_CPU
 666 /*
 667  * tasklet_kill_immediate is called to remove a tasklet which can already be
 668  * scheduled for execution on @cpu.
 669  *
 670  * Unlike tasklet_kill, this function removes the tasklet
 671  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 672  *
 673  * When this function is called, @cpu must be in the CPU_DEAD state.
 674  */
 675 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 676 {
 677         struct tasklet_struct **i;
 678
 679         BUG_ON(cpu_online(cpu));
 680         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 681
 682         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 683                 return;
 684
 685         /* CPU is dead, so no lock needed. */
 686         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 687                 if (*i == t) {
 688                         *i = t->next;
 689                         /* If this was the tail element, move the tail ptr */
 690                         if (*i == NULL)
 691                                 per_cpu(tasklet_vec, cpu).tail = i;
 692                         return;
 693                 }
 694         }
 695         BUG();
 696 }
 697
 698 static void takeover_tasklets(unsigned int cpu)
 699 {
 700         /* CPU is dead, so no lock needed. */
 701         local_irq_disable();
 702
 703         /* Find end, append list for that CPU. */
 704         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 705                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 706                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 707                 per_cpu(tasklet_vec, cpu).head = NULL;
 708                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 709         }
 710         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 711
 712         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 713                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 714                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 715                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 716                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 717         }
 718         raise_softirq_irqoff(HI_SOFTIRQ);
 719
 720         local_irq_enable();
 721 }
 722 #endif /* CONFIG_HOTPLUG_CPU */
 723
 724 static int cpu_callback(struct notifier_block *nfb,
 725                                   unsigned long action,
 726                                   void *hcpu)
 727 {
 728         switch (action) {
 729 #ifdef CONFIG_HOTPLUG_CPU
 730         case CPU_DEAD:
 731         case CPU_DEAD_FROZEN:
 732                 takeover_tasklets((unsigned long)hcpu);
 733                 break;
 734 #endif /* CONFIG_HOTPLUG_CPU */
 735         }
 736         return NOTIFY_OK;
 737 }
 738
 739 static struct notifier_block cpu_nfb = {
 740         .notifier_call = cpu_callback
 741 };
 742
 743 static struct smp_hotplug_thread softirq_threads = {
 744         .store                  = &ksoftirqd,
 745         .thread_should_run      = ksoftirqd_should_run,
 746         .thread_fn              = run_ksoftirqd,
 747         .thread_comm            = "ksoftirqd/%u",
 748 };
 749
 750 static __init int spawn_ksoftirqd(void)
 751 {
 752         register_cpu_notifier(&cpu_nfb);
 753
 754         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 755
 756         return 0;
 757 }
 758 early_initcall(spawn_ksoftirqd);
 759
 760 /*
 761  * [ These __weak aliases are kept in a separate compilation unit, so that
 762  *   GCC does not inline them incorrectly. ]
 763  */
 764
 765 int __init __weak early_irq_init(void)
 766 {
 767         return 0;
 768 }
 769
 770 int __init __weak arch_probe_nr_irqs(void)
 771 {
 772         return NR_IRQS_LEGACY;
 773 }
 774
 775 int __init __weak arch_early_irq_init(void)
 776 {
 777         return 0;
 778 }