kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  */
  10
  11 #include <linux/export.h>
  12 #include <linux/kernel_stat.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/init.h>
  15 #include <linux/mm.h>
  16 #include <linux/notifier.h>
  17 #include <linux/percpu.h>
  18 #include <linux/cpu.h>
  19 #include <linux/freezer.h>
  20 #include <linux/kthread.h>
  21 #include <linux/rcupdate.h>
  22 #include <linux/ftrace.h>
  23 #include <linux/smp.h>
  24 #include <linux/smpboot.h>
  25 #include <linux/tick.h>
  26
  27 #define CREATE_TRACE_POINTS
  28 #include <trace/events/irq.h>
  29
  30 /*
  31    - No shared variables, all the data are CPU local.
  32    - If a softirq needs serialization, let it serialize itself
  33      by its own spinlocks.
  34    - Even if softirq is serialized, only local cpu is marked for
  35      execution. Hence, we get something sort of weak cpu binding.
  36      Though it is still not clear, will it result in better locality
  37      or will not.
  38
  39    Examples:
  40    - NET RX softirq. It is multithreaded and does not require
  41      any global serialization.
  42    - NET TX softirq. It kicks software netdevice queues, hence
  43      it is logically serialized per device, but this serialization
  44      is invisible to common code.
  45    - Tasklets: serialized wrt itself.
  46  */
  47
  48 #ifndef __ARCH_IRQ_STAT
  49 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  50 EXPORT_SYMBOL(irq_stat);
  51 #endif
  52
  53 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  54
  55 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  56
  57 char *softirq_to_name[NR_SOFTIRQS] = {
  58         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  59         "TASKLET", "SCHED", "HRTIMER", "RCU"
  60 };
  61
  62 /*
  63  * we cannot loop indefinitely here to avoid userspace starvation,
  64  * but we also don't want to introduce a worst case 1/HZ latency
  65  * to the pending events, so lets the scheduler to balance
  66  * the softirq load for us.
  67  */
  68 static void wakeup_softirqd(void)
  69 {
  70         /* Interrupts are disabled: no need to stop preemption */
  71         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  72
  73         if (tsk && tsk->state != TASK_RUNNING)
  74                 wake_up_process(tsk);
  75 }
  76
  77 /*
  78  * preempt_count and SOFTIRQ_OFFSET usage:
  79  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  80  *   softirq processing.
  81  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  82  *   on local_bh_disable or local_bh_enable.
  83  * This lets us distinguish between whether we are currently processing
  84  * softirq and whether we just have bh disabled.
  85  */
  86
  87 /*
  88  * This one is for softirq.c-internal use,
  89  * where hardirqs are disabled legitimately:
  90  */
  91 #ifdef CONFIG_TRACE_IRQFLAGS
  92 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
  93 {
  94         unsigned long flags;
  95
  96         WARN_ON_ONCE(in_irq());
  97
  98         raw_local_irq_save(flags);
  99         /*
 100          * The preempt tracer hooks into preempt_count_add and will break
 101          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 102          * is set and before current->softirq_enabled is cleared.
 103          * We must manually increment preempt_count here and manually
 104          * call the trace_preempt_off later.
 105          */
 106         __preempt_count_add(cnt);
 107         /*
 108          * Were softirqs turned off above:
 109          */
 110         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 111                 trace_softirqs_off(ip);
 112         raw_local_irq_restore(flags);
 113
 114         if (preempt_count() == cnt)
 115                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 116 }
 117 EXPORT_SYMBOL(__local_bh_disable_ip);
 118 #endif /* CONFIG_TRACE_IRQFLAGS */
 119
 120 static void __local_bh_enable(unsigned int cnt)
 121 {
 122         WARN_ON_ONCE(!irqs_disabled());
 123
 124         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 125                 trace_softirqs_on(_RET_IP_);
 126         preempt_count_sub(cnt);
 127 }
 128
 129 /*
 130  * Special-case - softirqs can safely be enabled in
 131  * cond_resched_softirq(), or by __do_softirq(),
 132  * without processing still-pending softirqs:
 133  */
 134 void _local_bh_enable(void)
 135 {
 136         WARN_ON_ONCE(in_irq());
 137         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 138 }
 139
 140 EXPORT_SYMBOL(_local_bh_enable);
 141
 142 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 143 {
 144         WARN_ON_ONCE(in_irq() || irqs_disabled());
 145 #ifdef CONFIG_TRACE_IRQFLAGS
 146         local_irq_disable();
 147 #endif
 148         /*
 149          * Are softirqs going to be turned on now:
 150          */
 151         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 152                 trace_softirqs_on(ip);
 153         /*
 154          * Keep preemption disabled until we are done with
 155          * softirq processing:
 156          */
 157         preempt_count_sub(cnt - 1);
 158
 159         if (unlikely(!in_interrupt() && local_softirq_pending())) {
 160                 /*
 161                  * Run softirq if any pending. And do it in its own stack
 162                  * as we may be calling this deep in a task call stack already.
 163                  */
 164                 do_softirq();
 165         }
 166
 167         preempt_count_dec();
 168 #ifdef CONFIG_TRACE_IRQFLAGS
 169         local_irq_enable();
 170 #endif
 171         preempt_check_resched();
 172 }
 173 EXPORT_SYMBOL(__local_bh_enable_ip);
 174
 175 /*
 176  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 177  * but break the loop if need_resched() is set or after 2 ms.
 178  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 179  * certain cases, such as stop_machine(), jiffies may cease to
 180  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 181  * well to make sure we eventually return from this method.
 182  *
 183  * These limits have been established via experimentation.
 184  * The two things to balance is latency against fairness -
 185  * we want to handle softirqs as soon as possible, but they
 186  * should not be able to lock up the box.
 187  */
 188 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 189 #define MAX_SOFTIRQ_RESTART 10
 190
 191 asmlinkage void __do_softirq(void)
 192 {
 193         struct softirq_action *h;
 194         __u32 pending;
 195         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 196         int cpu;
 197         unsigned long old_flags = current->flags;
 198         int max_restart = MAX_SOFTIRQ_RESTART;
 199
 200         /*
 201          * Mask out PF_MEMALLOC s current task context is borrowed for the
 202          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 203          * again if the socket is related to swap
 204          */
 205         current->flags &= ~PF_MEMALLOC;
 206
 207         pending = local_softirq_pending();
 208         account_irq_enter_time(current);
 209
 210         __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 211         lockdep_softirq_enter();
 212
 213         cpu = smp_processor_id();
 214 restart:
 215         /* Reset the pending bitmask before enabling irqs */
 216         set_softirq_pending(0);
 217
 218         local_irq_enable();
 219
 220         h = softirq_vec;
 221
 222         do {
 223                 if (pending & 1) {
 224                         unsigned int vec_nr = h - softirq_vec;
 225                         int prev_count = preempt_count();
 226
 227                         kstat_incr_softirqs_this_cpu(vec_nr);
 228
 229                         trace_softirq_entry(vec_nr);
 230                         h->action(h);
 231                         trace_softirq_exit(vec_nr);
 232                         if (unlikely(prev_count != preempt_count())) {
 233                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 234                                        "with preempt_count %08x,"
 235                                        " exited with %08x?\n", vec_nr,
 236                                        softirq_to_name[vec_nr], h->action,
 237                                        prev_count, preempt_count());
 238                                 preempt_count_set(prev_count);
 239                         }
 240
 241                         rcu_bh_qs(cpu);
 242                 }
 243                 h++;
 244                 pending >>= 1;
 245         } while (pending);
 246
 247         local_irq_disable();
 248
 249         pending = local_softirq_pending();
 250         if (pending) {
 251                 if (time_before(jiffies, end) && !need_resched() &&
 252                     --max_restart)
 253                         goto restart;
 254
 255                 wakeup_softirqd();
 256         }
 257
 258         lockdep_softirq_exit();
 259
 260         account_irq_exit_time(current);
 261         __local_bh_enable(SOFTIRQ_OFFSET);
 262         WARN_ON_ONCE(in_interrupt());
 263         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 264 }
 265
 266
 267
 268 asmlinkage void do_softirq(void)
 269 {
 270         __u32 pending;
 271         unsigned long flags;
 272
 273         if (in_interrupt())
 274                 return;
 275
 276         local_irq_save(flags);
 277
 278         pending = local_softirq_pending();
 279
 280         if (pending)
 281                 do_softirq_own_stack();
 282
 283         local_irq_restore(flags);
 284 }
 285
 286 /*
 287  * Enter an interrupt context.
 288  */
 289 void irq_enter(void)
 290 {
 291         int cpu = smp_processor_id();
 292
 293         rcu_irq_enter();
 294         if (is_idle_task(current) && !in_interrupt()) {
 295                 /*
 296                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 297                  * here, as softirq will be serviced on return from interrupt.
 298                  */
 299                 local_bh_disable();
 300                 tick_check_idle(cpu);
 301                 _local_bh_enable();
 302         }
 303
 304         __irq_enter();
 305 }
 306
 307 static inline void invoke_softirq(void)
 308 {
 309         if (!force_irqthreads) {
 310 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 311                 /*
 312                  * We can safely execute softirq on the current stack if
 313                  * it is the irq stack, because it should be near empty
 314                  * at this stage.
 315                  */
 316                 __do_softirq();
 317 #else
 318                 /*
 319                  * Otherwise, irq_exit() is called on the task stack that can
 320                  * be potentially deep already. So call softirq in its own stack
 321                  * to prevent from any overrun.
 322                  */
 323                 do_softirq_own_stack();
 324 #endif
 325         } else {
 326                 wakeup_softirqd();
 327         }
 328 }
 329
 330 static inline void tick_irq_exit(void)
 331 {
 332 #ifdef CONFIG_NO_HZ_COMMON
 333         int cpu = smp_processor_id();
 334
 335         /* Make sure that timer wheel updates are propagated */
 336         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 337                 if (!in_interrupt())
 338                         tick_nohz_irq_exit();
 339         }
 340 #endif
 341 }
 342
 343 /*
 344  * Exit an interrupt context. Process softirqs if needed and possible:
 345  */
 346 void irq_exit(void)
 347 {
 348 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 349         local_irq_disable();
 350 #else
 351         WARN_ON_ONCE(!irqs_disabled());
 352 #endif
 353
 354         account_irq_exit_time(current);
 355         trace_hardirq_exit();
 356         preempt_count_sub(HARDIRQ_OFFSET);
 357         if (!in_interrupt() && local_softirq_pending())
 358                 invoke_softirq();
 359
 360         tick_irq_exit();
 361         rcu_irq_exit();
 362 }
 363
 364 /*
 365  * This function must run with irqs disabled!
 366  */
 367 inline void raise_softirq_irqoff(unsigned int nr)
 368 {
 369         __raise_softirq_irqoff(nr);
 370
 371         /*
 372          * If we're in an interrupt or softirq, we're done
 373          * (this also catches softirq-disabled code). We will
 374          * actually run the softirq once we return from
 375          * the irq or softirq.
 376          *
 377          * Otherwise we wake up ksoftirqd to make sure we
 378          * schedule the softirq soon.
 379          */
 380         if (!in_interrupt())
 381                 wakeup_softirqd();
 382 }
 383
 384 void raise_softirq(unsigned int nr)
 385 {
 386         unsigned long flags;
 387
 388         local_irq_save(flags);
 389         raise_softirq_irqoff(nr);
 390         local_irq_restore(flags);
 391 }
 392
 393 void __raise_softirq_irqoff(unsigned int nr)
 394 {
 395         trace_softirq_raise(nr);
 396         or_softirq_pending(1UL << nr);
 397 }
 398
 399 void open_softirq(int nr, void (*action)(struct softirq_action *))
 400 {
 401         softirq_vec[nr].action = action;
 402 }
 403
 404 /*
 405  * Tasklets
 406  */
 407 struct tasklet_head
 408 {
 409         struct tasklet_struct *head;
 410         struct tasklet_struct **tail;
 411 };
 412
 413 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 414 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 415
 416 void __tasklet_schedule(struct tasklet_struct *t)
 417 {
 418         unsigned long flags;
 419
 420         local_irq_save(flags);
 421         t->next = NULL;
 422         *__this_cpu_read(tasklet_vec.tail) = t;
 423         __this_cpu_write(tasklet_vec.tail, &(t->next));
 424         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 425         local_irq_restore(flags);
 426 }
 427
 428 EXPORT_SYMBOL(__tasklet_schedule);
 429
 430 void __tasklet_hi_schedule(struct tasklet_struct *t)
 431 {
 432         unsigned long flags;
 433
 434         local_irq_save(flags);
 435         t->next = NULL;
 436         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 437         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 438         raise_softirq_irqoff(HI_SOFTIRQ);
 439         local_irq_restore(flags);
 440 }
 441
 442 EXPORT_SYMBOL(__tasklet_hi_schedule);
 443
 444 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 445 {
 446         BUG_ON(!irqs_disabled());
 447
 448         t->next = __this_cpu_read(tasklet_hi_vec.head);
 449         __this_cpu_write(tasklet_hi_vec.head, t);
 450         __raise_softirq_irqoff(HI_SOFTIRQ);
 451 }
 452
 453 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 454
 455 static void tasklet_action(struct softirq_action *a)
 456 {
 457         struct tasklet_struct *list;
 458
 459         local_irq_disable();
 460         list = __this_cpu_read(tasklet_vec.head);
 461         __this_cpu_write(tasklet_vec.head, NULL);
 462         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 463         local_irq_enable();
 464
 465         while (list) {
 466                 struct tasklet_struct *t = list;
 467
 468                 list = list->next;
 469
 470                 if (tasklet_trylock(t)) {
 471                         if (!atomic_read(&t->count)) {
 472                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 473                                         BUG();
 474                                 t->func(t->data);
 475                                 tasklet_unlock(t);
 476                                 continue;
 477                         }
 478                         tasklet_unlock(t);
 479                 }
 480
 481                 local_irq_disable();
 482                 t->next = NULL;
 483                 *__this_cpu_read(tasklet_vec.tail) = t;
 484                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 485                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 486                 local_irq_enable();
 487         }
 488 }
 489
 490 static void tasklet_hi_action(struct softirq_action *a)
 491 {
 492         struct tasklet_struct *list;
 493
 494         local_irq_disable();
 495         list = __this_cpu_read(tasklet_hi_vec.head);
 496         __this_cpu_write(tasklet_hi_vec.head, NULL);
 497         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 498         local_irq_enable();
 499
 500         while (list) {
 501                 struct tasklet_struct *t = list;
 502
 503                 list = list->next;
 504
 505                 if (tasklet_trylock(t)) {
 506                         if (!atomic_read(&t->count)) {
 507                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 508                                         BUG();
 509                                 t->func(t->data);
 510                                 tasklet_unlock(t);
 511                                 continue;
 512                         }
 513                         tasklet_unlock(t);
 514                 }
 515
 516                 local_irq_disable();
 517                 t->next = NULL;
 518                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 519                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 520                 __raise_softirq_irqoff(HI_SOFTIRQ);
 521                 local_irq_enable();
 522         }
 523 }
 524
 525
 526 void tasklet_init(struct tasklet_struct *t,
 527                   void (*func)(unsigned long), unsigned long data)
 528 {
 529         t->next = NULL;
 530         t->state = 0;
 531         atomic_set(&t->count, 0);
 532         t->func = func;
 533         t->data = data;
 534 }
 535
 536 EXPORT_SYMBOL(tasklet_init);
 537
 538 void tasklet_kill(struct tasklet_struct *t)
 539 {
 540         if (in_interrupt())
 541                 printk("Attempt to kill tasklet from interrupt\n");
 542
 543         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 544                 do {
 545                         yield();
 546                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 547         }
 548         tasklet_unlock_wait(t);
 549         clear_bit(TASKLET_STATE_SCHED, &t->state);
 550 }
 551
 552 EXPORT_SYMBOL(tasklet_kill);
 553
 554 /*
 555  * tasklet_hrtimer
 556  */
 557
 558 /*
 559  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 560  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 561  * hrtimer callback, but from softirq context.
 562  */
 563 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 564 {
 565         struct tasklet_hrtimer *ttimer =
 566                 container_of(timer, struct tasklet_hrtimer, timer);
 567
 568         tasklet_hi_schedule(&ttimer->tasklet);
 569         return HRTIMER_NORESTART;
 570 }
 571
 572 /*
 573  * Helper function which calls the hrtimer callback from
 574  * tasklet/softirq context
 575  */
 576 static void __tasklet_hrtimer_trampoline(unsigned long data)
 577 {
 578         struct tasklet_hrtimer *ttimer = (void *)data;
 579         enum hrtimer_restart restart;
 580
 581         restart = ttimer->function(&ttimer->timer);
 582         if (restart != HRTIMER_NORESTART)
 583                 hrtimer_restart(&ttimer->timer);
 584 }
 585
 586 /**
 587  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 588  * @ttimer:      tasklet_hrtimer which is initialized
 589  * @function:    hrtimer callback function which gets called from softirq context
 590  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 591  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 592  */
 593 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 594                           enum hrtimer_restart (*function)(struct hrtimer *),
 595                           clockid_t which_clock, enum hrtimer_mode mode)
 596 {
 597         hrtimer_init(&ttimer->timer, which_clock, mode);
 598         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 599         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 600                      (unsigned long)ttimer);
 601         ttimer->function = function;
 602 }
 603 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 604
 605 void __init softirq_init(void)
 606 {
 607         int cpu;
 608
 609         for_each_possible_cpu(cpu) {
 610                 per_cpu(tasklet_vec, cpu).tail =
 611                         &per_cpu(tasklet_vec, cpu).head;
 612                 per_cpu(tasklet_hi_vec, cpu).tail =
 613                         &per_cpu(tasklet_hi_vec, cpu).head;
 614         }
 615
 616         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 617         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 618 }
 619
 620 static int ksoftirqd_should_run(unsigned int cpu)
 621 {
 622         return local_softirq_pending();
 623 }
 624
 625 static void run_ksoftirqd(unsigned int cpu)
 626 {
 627         local_irq_disable();
 628         if (local_softirq_pending()) {
 629                 /*
 630                  * We can safely run softirq on inline stack, as we are not deep
 631                  * in the task stack here.
 632                  */
 633                 __do_softirq();
 634                 rcu_note_context_switch(cpu);
 635                 local_irq_enable();
 636                 cond_resched();
 637                 return;
 638         }
 639         local_irq_enable();
 640 }
 641
 642 #ifdef CONFIG_HOTPLUG_CPU
 643 /*
 644  * tasklet_kill_immediate is called to remove a tasklet which can already be
 645  * scheduled for execution on @cpu.
 646  *
 647  * Unlike tasklet_kill, this function removes the tasklet
 648  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 649  *
 650  * When this function is called, @cpu must be in the CPU_DEAD state.
 651  */
 652 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 653 {
 654         struct tasklet_struct **i;
 655
 656         BUG_ON(cpu_online(cpu));
 657         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 658
 659         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 660                 return;
 661
 662         /* CPU is dead, so no lock needed. */
 663         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 664                 if (*i == t) {
 665                         *i = t->next;
 666                         /* If this was the tail element, move the tail ptr */
 667                         if (*i == NULL)
 668                                 per_cpu(tasklet_vec, cpu).tail = i;
 669                         return;
 670                 }
 671         }
 672         BUG();
 673 }
 674
 675 static void takeover_tasklets(unsigned int cpu)
 676 {
 677         /* CPU is dead, so no lock needed. */
 678         local_irq_disable();
 679
 680         /* Find end, append list for that CPU. */
 681         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 682                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 683                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 684                 per_cpu(tasklet_vec, cpu).head = NULL;
 685                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 686         }
 687         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 688
 689         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 690                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 691                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 692                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 693                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 694         }
 695         raise_softirq_irqoff(HI_SOFTIRQ);
 696
 697         local_irq_enable();
 698 }
 699 #endif /* CONFIG_HOTPLUG_CPU */
 700
 701 static int cpu_callback(struct notifier_block *nfb,
 702                                   unsigned long action,
 703                                   void *hcpu)
 704 {
 705         switch (action) {
 706 #ifdef CONFIG_HOTPLUG_CPU
 707         case CPU_DEAD:
 708         case CPU_DEAD_FROZEN:
 709                 takeover_tasklets((unsigned long)hcpu);
 710                 break;
 711 #endif /* CONFIG_HOTPLUG_CPU */
 712         }
 713         return NOTIFY_OK;
 714 }
 715
 716 static struct notifier_block cpu_nfb = {
 717         .notifier_call = cpu_callback
 718 };
 719
 720 static struct smp_hotplug_thread softirq_threads = {
 721         .store                  = &ksoftirqd,
 722         .thread_should_run      = ksoftirqd_should_run,
 723         .thread_fn              = run_ksoftirqd,
 724         .thread_comm            = "ksoftirqd/%u",
 725 };
 726
 727 static __init int spawn_ksoftirqd(void)
 728 {
 729         register_cpu_notifier(&cpu_nfb);
 730
 731         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 732
 733         return 0;
 734 }
 735 early_initcall(spawn_ksoftirqd);
 736
 737 /*
 738  * [ These __weak aliases are kept in a separate compilation unit, so that
 739  *   GCC does not inline them incorrectly. ]
 740  */
 741
 742 int __init __weak early_irq_init(void)
 743 {
 744         return 0;
 745 }
 746
 747 int __init __weak arch_probe_nr_irqs(void)
 748 {
 749         return NR_IRQS_LEGACY;
 750 }
 751
 752 int __init __weak arch_early_irq_init(void)
 753 {
 754         return 0;
 755 }