kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  */
  10
  11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
  13 #include <linux/export.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/smpboot.h>
  27 #include <linux/tick.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include <trace/events/irq.h>
  31
  32 /*
  33    - No shared variables, all the data are CPU local.
  34    - If a softirq needs serialization, let it serialize itself
  35      by its own spinlocks.
  36    - Even if softirq is serialized, only local cpu is marked for
  37      execution. Hence, we get something sort of weak cpu binding.
  38      Though it is still not clear, will it result in better locality
  39      or will not.
  40
  41    Examples:
  42    - NET RX softirq. It is multithreaded and does not require
  43      any global serialization.
  44    - NET TX softirq. It kicks software netdevice queues, hence
  45      it is logically serialized per device, but this serialization
  46      is invisible to common code.
  47    - Tasklets: serialized wrt itself.
  48  */
  49
  50 #ifndef __ARCH_IRQ_STAT
  51 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  52 EXPORT_SYMBOL(irq_stat);
  53 #endif
  54
  55 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  56
  57 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  58
  59 const char * const softirq_to_name[NR_SOFTIRQS] = {
  60         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  61         "TASKLET", "SCHED", "HRTIMER", "RCU"
  62 };
  63
  64 /*
  65  * we cannot loop indefinitely here to avoid userspace starvation,
  66  * but we also don't want to introduce a worst case 1/HZ latency
  67  * to the pending events, so lets the scheduler to balance
  68  * the softirq load for us.
  69  */
  70 static void wakeup_softirqd(void)
  71 {
  72         /* Interrupts are disabled: no need to stop preemption */
  73         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  74
  75         if (tsk && tsk->state != TASK_RUNNING)
  76                 wake_up_process(tsk);
  77 }
  78
  79 /*
  80  * preempt_count and SOFTIRQ_OFFSET usage:
  81  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  82  *   softirq processing.
  83  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  84  *   on local_bh_disable or local_bh_enable.
  85  * This lets us distinguish between whether we are currently processing
  86  * softirq and whether we just have bh disabled.
  87  */
  88
  89 /*
  90  * This one is for softirq.c-internal use,
  91  * where hardirqs are disabled legitimately:
  92  */
  93 #ifdef CONFIG_TRACE_IRQFLAGS
  94 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
  95 {
  96         unsigned long flags;
  97
  98         WARN_ON_ONCE(in_irq());
  99
 100         raw_local_irq_save(flags);
 101         /*
 102          * The preempt tracer hooks into preempt_count_add and will break
 103          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 104          * is set and before current->softirq_enabled is cleared.
 105          * We must manually increment preempt_count here and manually
 106          * call the trace_preempt_off later.
 107          */
 108         __preempt_count_add(cnt);
 109         /*
 110          * Were softirqs turned off above:
 111          */
 112         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 113                 trace_softirqs_off(ip);
 114         raw_local_irq_restore(flags);
 115
 116         if (preempt_count() == cnt)
 117                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 118 }
 119 EXPORT_SYMBOL(__local_bh_disable_ip);
 120 #endif /* CONFIG_TRACE_IRQFLAGS */
 121
 122 static void __local_bh_enable(unsigned int cnt)
 123 {
 124         WARN_ON_ONCE(!irqs_disabled());
 125
 126         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 127                 trace_softirqs_on(_RET_IP_);
 128         preempt_count_sub(cnt);
 129 }
 130
 131 /*
 132  * Special-case - softirqs can safely be enabled in
 133  * cond_resched_softirq(), or by __do_softirq(),
 134  * without processing still-pending softirqs:
 135  */
 136 void _local_bh_enable(void)
 137 {
 138         WARN_ON_ONCE(in_irq());
 139         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 140 }
 141 EXPORT_SYMBOL(_local_bh_enable);
 142
 143 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 144 {
 145         WARN_ON_ONCE(in_irq() || irqs_disabled());
 146 #ifdef CONFIG_TRACE_IRQFLAGS
 147         local_irq_disable();
 148 #endif
 149         /*
 150          * Are softirqs going to be turned on now:
 151          */
 152         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 153                 trace_softirqs_on(ip);
 154         /*
 155          * Keep preemption disabled until we are done with
 156          * softirq processing:
 157          */
 158         preempt_count_sub(cnt - 1);
 159
 160         if (unlikely(!in_interrupt() && local_softirq_pending())) {
 161                 /*
 162                  * Run softirq if any pending. And do it in its own stack
 163                  * as we may be calling this deep in a task call stack already.
 164                  */
 165                 do_softirq();
 166         }
 167
 168         preempt_count_dec();
 169 #ifdef CONFIG_TRACE_IRQFLAGS
 170         local_irq_enable();
 171 #endif
 172         preempt_check_resched();
 173 }
 174 EXPORT_SYMBOL(__local_bh_enable_ip);
 175
 176 /*
 177  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 178  * but break the loop if need_resched() is set or after 2 ms.
 179  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 180  * certain cases, such as stop_machine(), jiffies may cease to
 181  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 182  * well to make sure we eventually return from this method.
 183  *
 184  * These limits have been established via experimentation.
 185  * The two things to balance is latency against fairness -
 186  * we want to handle softirqs as soon as possible, but they
 187  * should not be able to lock up the box.
 188  */
 189 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 190 #define MAX_SOFTIRQ_RESTART 10
 191
 192 #ifdef CONFIG_TRACE_IRQFLAGS
 193 /*
 194  * When we run softirqs from irq_exit() and thus on the hardirq stack we need
 195  * to keep the lockdep irq context tracking as tight as possible in order to
 196  * not miss-qualify lock contexts and miss possible deadlocks.
 197  */
 198
 199 static inline bool lockdep_softirq_start(void)
 200 {
 201         bool in_hardirq = false;
 202
 203         if (trace_hardirq_context(current)) {
 204                 in_hardirq = true;
 205                 trace_hardirq_exit();
 206         }
 207
 208         lockdep_softirq_enter();
 209
 210         return in_hardirq;
 211 }
 212
 213 static inline void lockdep_softirq_end(bool in_hardirq)
 214 {
 215         lockdep_softirq_exit();
 216
 217         if (in_hardirq)
 218                 trace_hardirq_enter();
 219 }
 220 #else
 221 static inline bool lockdep_softirq_start(void) { return false; }
 222 static inline void lockdep_softirq_end(bool in_hardirq) { }
 223 #endif
 224
 225 asmlinkage void __do_softirq(void)
 226 {
 227         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 228         unsigned long old_flags = current->flags;
 229         int max_restart = MAX_SOFTIRQ_RESTART;
 230         struct softirq_action *h;
 231         bool in_hardirq;
 232         __u32 pending;
 233         int softirq_bit;
 234         int cpu;
 235
 236         /*
 237          * Mask out PF_MEMALLOC s current task context is borrowed for the
 238          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 239          * again if the socket is related to swap
 240          */
 241         current->flags &= ~PF_MEMALLOC;
 242
 243         pending = local_softirq_pending();
 244         account_irq_enter_time(current);
 245
 246         __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 247         in_hardirq = lockdep_softirq_start();
 248
 249         cpu = smp_processor_id();
 250 restart:
 251         /* Reset the pending bitmask before enabling irqs */
 252         set_softirq_pending(0);
 253
 254         local_irq_enable();
 255
 256         h = softirq_vec;
 257
 258         while ((softirq_bit = ffs(pending))) {
 259                 unsigned int vec_nr;
 260                 int prev_count;
 261
 262                 h += softirq_bit - 1;
 263
 264                 vec_nr = h - softirq_vec;
 265                 prev_count = preempt_count();
 266
 267                 kstat_incr_softirqs_this_cpu(vec_nr);
 268
 269                 trace_softirq_entry(vec_nr);
 270                 h->action(h);
 271                 trace_softirq_exit(vec_nr);
 272                 if (unlikely(prev_count != preempt_count())) {
 273                         pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
 274                                vec_nr, softirq_to_name[vec_nr], h->action,
 275                                prev_count, preempt_count());
 276                         preempt_count_set(prev_count);
 277                 }
 278                 rcu_bh_qs(cpu);
 279                 h++;
 280                 pending >>= softirq_bit;
 281         }
 282
 283         local_irq_disable();
 284
 285         pending = local_softirq_pending();
 286         if (pending) {
 287                 if (time_before(jiffies, end) && !need_resched() &&
 288                     --max_restart)
 289                         goto restart;
 290
 291                 wakeup_softirqd();
 292         }
 293
 294         lockdep_softirq_end(in_hardirq);
 295         account_irq_exit_time(current);
 296         __local_bh_enable(SOFTIRQ_OFFSET);
 297         WARN_ON_ONCE(in_interrupt());
 298         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 299 }
 300
 301 asmlinkage void do_softirq(void)
 302 {
 303         __u32 pending;
 304         unsigned long flags;
 305
 306         if (in_interrupt())
 307                 return;
 308
 309         local_irq_save(flags);
 310
 311         pending = local_softirq_pending();
 312
 313         if (pending)
 314                 do_softirq_own_stack();
 315
 316         local_irq_restore(flags);
 317 }
 318
 319 /*
 320  * Enter an interrupt context.
 321  */
 322 void irq_enter(void)
 323 {
 324         rcu_irq_enter();
 325         if (is_idle_task(current) && !in_interrupt()) {
 326                 /*
 327                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 328                  * here, as softirq will be serviced on return from interrupt.
 329                  */
 330                 local_bh_disable();
 331                 tick_check_idle();
 332                 _local_bh_enable();
 333         }
 334
 335         __irq_enter();
 336 }
 337
 338 static inline void invoke_softirq(void)
 339 {
 340         if (!force_irqthreads) {
 341 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 342                 /*
 343                  * We can safely execute softirq on the current stack if
 344                  * it is the irq stack, because it should be near empty
 345                  * at this stage.
 346                  */
 347                 __do_softirq();
 348 #else
 349                 /*
 350                  * Otherwise, irq_exit() is called on the task stack that can
 351                  * be potentially deep already. So call softirq in its own stack
 352                  * to prevent from any overrun.
 353                  */
 354                 do_softirq_own_stack();
 355 #endif
 356         } else {
 357                 wakeup_softirqd();
 358         }
 359 }
 360
 361 static inline void tick_irq_exit(void)
 362 {
 363 #ifdef CONFIG_NO_HZ_COMMON
 364         int cpu = smp_processor_id();
 365
 366         /* Make sure that timer wheel updates are propagated */
 367         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 368                 if (!in_interrupt())
 369                         tick_nohz_irq_exit();
 370         }
 371 #endif
 372 }
 373
 374 /*
 375  * Exit an interrupt context. Process softirqs if needed and possible:
 376  */
 377 void irq_exit(void)
 378 {
 379 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 380         local_irq_disable();
 381 #else
 382         WARN_ON_ONCE(!irqs_disabled());
 383 #endif
 384
 385         account_irq_exit_time(current);
 386         preempt_count_sub(HARDIRQ_OFFSET);
 387         if (!in_interrupt() && local_softirq_pending())
 388                 invoke_softirq();
 389
 390         tick_irq_exit();
 391         rcu_irq_exit();
 392         trace_hardirq_exit(); /* must be last! */
 393 }
 394
 395 /*
 396  * This function must run with irqs disabled!
 397  */
 398 inline void raise_softirq_irqoff(unsigned int nr)
 399 {
 400         __raise_softirq_irqoff(nr);
 401
 402         /*
 403          * If we're in an interrupt or softirq, we're done
 404          * (this also catches softirq-disabled code). We will
 405          * actually run the softirq once we return from
 406          * the irq or softirq.
 407          *
 408          * Otherwise we wake up ksoftirqd to make sure we
 409          * schedule the softirq soon.
 410          */
 411         if (!in_interrupt())
 412                 wakeup_softirqd();
 413 }
 414
 415 void raise_softirq(unsigned int nr)
 416 {
 417         unsigned long flags;
 418
 419         local_irq_save(flags);
 420         raise_softirq_irqoff(nr);
 421         local_irq_restore(flags);
 422 }
 423
 424 void __raise_softirq_irqoff(unsigned int nr)
 425 {
 426         trace_softirq_raise(nr);
 427         or_softirq_pending(1UL << nr);
 428 }
 429
 430 void open_softirq(int nr, void (*action)(struct softirq_action *))
 431 {
 432         softirq_vec[nr].action = action;
 433 }
 434
 435 /*
 436  * Tasklets
 437  */
 438 struct tasklet_head {
 439         struct tasklet_struct *head;
 440         struct tasklet_struct **tail;
 441 };
 442
 443 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 444 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 445
 446 void __tasklet_schedule(struct tasklet_struct *t)
 447 {
 448         unsigned long flags;
 449
 450         local_irq_save(flags);
 451         t->next = NULL;
 452         *__this_cpu_read(tasklet_vec.tail) = t;
 453         __this_cpu_write(tasklet_vec.tail, &(t->next));
 454         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 455         local_irq_restore(flags);
 456 }
 457 EXPORT_SYMBOL(__tasklet_schedule);
 458
 459 void __tasklet_hi_schedule(struct tasklet_struct *t)
 460 {
 461         unsigned long flags;
 462
 463         local_irq_save(flags);
 464         t->next = NULL;
 465         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 466         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 467         raise_softirq_irqoff(HI_SOFTIRQ);
 468         local_irq_restore(flags);
 469 }
 470 EXPORT_SYMBOL(__tasklet_hi_schedule);
 471
 472 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 473 {
 474         BUG_ON(!irqs_disabled());
 475
 476         t->next = __this_cpu_read(tasklet_hi_vec.head);
 477         __this_cpu_write(tasklet_hi_vec.head, t);
 478         __raise_softirq_irqoff(HI_SOFTIRQ);
 479 }
 480 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 481
 482 static void tasklet_action(struct softirq_action *a)
 483 {
 484         struct tasklet_struct *list;
 485
 486         local_irq_disable();
 487         list = __this_cpu_read(tasklet_vec.head);
 488         __this_cpu_write(tasklet_vec.head, NULL);
 489         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 490         local_irq_enable();
 491
 492         while (list) {
 493                 struct tasklet_struct *t = list;
 494
 495                 list = list->next;
 496
 497                 if (tasklet_trylock(t)) {
 498                         if (!atomic_read(&t->count)) {
 499                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
 500                                                         &t->state))
 501                                         BUG();
 502                                 t->func(t->data);
 503                                 tasklet_unlock(t);
 504                                 continue;
 505                         }
 506                         tasklet_unlock(t);
 507                 }
 508
 509                 local_irq_disable();
 510                 t->next = NULL;
 511                 *__this_cpu_read(tasklet_vec.tail) = t;
 512                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 513                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 514                 local_irq_enable();
 515         }
 516 }
 517
 518 static void tasklet_hi_action(struct softirq_action *a)
 519 {
 520         struct tasklet_struct *list;
 521
 522         local_irq_disable();
 523         list = __this_cpu_read(tasklet_hi_vec.head);
 524         __this_cpu_write(tasklet_hi_vec.head, NULL);
 525         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 526         local_irq_enable();
 527
 528         while (list) {
 529                 struct tasklet_struct *t = list;
 530
 531                 list = list->next;
 532
 533                 if (tasklet_trylock(t)) {
 534                         if (!atomic_read(&t->count)) {
 535                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
 536                                                         &t->state))
 537                                         BUG();
 538                                 t->func(t->data);
 539                                 tasklet_unlock(t);
 540                                 continue;
 541                         }
 542                         tasklet_unlock(t);
 543                 }
 544
 545                 local_irq_disable();
 546                 t->next = NULL;
 547                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 548                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 549                 __raise_softirq_irqoff(HI_SOFTIRQ);
 550                 local_irq_enable();
 551         }
 552 }
 553
 554 void tasklet_init(struct tasklet_struct *t,
 555                   void (*func)(unsigned long), unsigned long data)
 556 {
 557         t->next = NULL;
 558         t->state = 0;
 559         atomic_set(&t->count, 0);
 560         t->func = func;
 561         t->data = data;
 562 }
 563 EXPORT_SYMBOL(tasklet_init);
 564
 565 void tasklet_kill(struct tasklet_struct *t)
 566 {
 567         if (in_interrupt())
 568                 pr_notice("Attempt to kill tasklet from interrupt\n");
 569
 570         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 571                 do {
 572                         yield();
 573                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 574         }
 575         tasklet_unlock_wait(t);
 576         clear_bit(TASKLET_STATE_SCHED, &t->state);
 577 }
 578 EXPORT_SYMBOL(tasklet_kill);
 579
 580 /*
 581  * tasklet_hrtimer
 582  */
 583
 584 /*
 585  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 586  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 587  * hrtimer callback, but from softirq context.
 588  */
 589 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 590 {
 591         struct tasklet_hrtimer *ttimer =
 592                 container_of(timer, struct tasklet_hrtimer, timer);
 593
 594         tasklet_hi_schedule(&ttimer->tasklet);
 595         return HRTIMER_NORESTART;
 596 }
 597
 598 /*
 599  * Helper function which calls the hrtimer callback from
 600  * tasklet/softirq context
 601  */
 602 static void __tasklet_hrtimer_trampoline(unsigned long data)
 603 {
 604         struct tasklet_hrtimer *ttimer = (void *)data;
 605         enum hrtimer_restart restart;
 606
 607         restart = ttimer->function(&ttimer->timer);
 608         if (restart != HRTIMER_NORESTART)
 609                 hrtimer_restart(&ttimer->timer);
 610 }
 611
 612 /**
 613  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 614  * @ttimer:      tasklet_hrtimer which is initialized
 615  * @function:    hrtimer callback function which gets called from softirq context
 616  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 617  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 618  */
 619 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 620                           enum hrtimer_restart (*function)(struct hrtimer *),
 621                           clockid_t which_clock, enum hrtimer_mode mode)
 622 {
 623         hrtimer_init(&ttimer->timer, which_clock, mode);
 624         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 625         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 626                      (unsigned long)ttimer);
 627         ttimer->function = function;
 628 }
 629 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 630
 631 void __init softirq_init(void)
 632 {
 633         int cpu;
 634
 635         for_each_possible_cpu(cpu) {
 636                 per_cpu(tasklet_vec, cpu).tail =
 637                         &per_cpu(tasklet_vec, cpu).head;
 638                 per_cpu(tasklet_hi_vec, cpu).tail =
 639                         &per_cpu(tasklet_hi_vec, cpu).head;
 640         }
 641
 642         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 643         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 644 }
 645
 646 static int ksoftirqd_should_run(unsigned int cpu)
 647 {
 648         return local_softirq_pending();
 649 }
 650
 651 static void run_ksoftirqd(unsigned int cpu)
 652 {
 653         local_irq_disable();
 654         if (local_softirq_pending()) {
 655                 /*
 656                  * We can safely run softirq on inline stack, as we are not deep
 657                  * in the task stack here.
 658                  */
 659                 __do_softirq();
 660                 rcu_note_context_switch(cpu);
 661                 local_irq_enable();
 662                 cond_resched();
 663                 return;
 664         }
 665         local_irq_enable();
 666 }
 667
 668 #ifdef CONFIG_HOTPLUG_CPU
 669 /*
 670  * tasklet_kill_immediate is called to remove a tasklet which can already be
 671  * scheduled for execution on @cpu.
 672  *
 673  * Unlike tasklet_kill, this function removes the tasklet
 674  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 675  *
 676  * When this function is called, @cpu must be in the CPU_DEAD state.
 677  */
 678 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 679 {
 680         struct tasklet_struct **i;
 681
 682         BUG_ON(cpu_online(cpu));
 683         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 684
 685         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 686                 return;
 687
 688         /* CPU is dead, so no lock needed. */
 689         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 690                 if (*i == t) {
 691                         *i = t->next;
 692                         /* If this was the tail element, move the tail ptr */
 693                         if (*i == NULL)
 694                                 per_cpu(tasklet_vec, cpu).tail = i;
 695                         return;
 696                 }
 697         }
 698         BUG();
 699 }
 700
 701 static void takeover_tasklets(unsigned int cpu)
 702 {
 703         /* CPU is dead, so no lock needed. */
 704         local_irq_disable();
 705
 706         /* Find end, append list for that CPU. */
 707         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 708                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 709                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 710                 per_cpu(tasklet_vec, cpu).head = NULL;
 711                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 712         }
 713         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 714
 715         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 716                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 717                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 718                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 719                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 720         }
 721         raise_softirq_irqoff(HI_SOFTIRQ);
 722
 723         local_irq_enable();
 724 }
 725 #endif /* CONFIG_HOTPLUG_CPU */
 726
 727 static int cpu_callback(struct notifier_block *nfb, unsigned long action,
 728                         void *hcpu)
 729 {
 730         switch (action) {
 731 #ifdef CONFIG_HOTPLUG_CPU
 732         case CPU_DEAD:
 733         case CPU_DEAD_FROZEN:
 734                 takeover_tasklets((unsigned long)hcpu);
 735                 break;
 736 #endif /* CONFIG_HOTPLUG_CPU */
 737         }
 738         return NOTIFY_OK;
 739 }
 740
 741 static struct notifier_block cpu_nfb = {
 742         .notifier_call = cpu_callback
 743 };
 744
 745 static struct smp_hotplug_thread softirq_threads = {
 746         .store                  = &ksoftirqd,
 747         .thread_should_run      = ksoftirqd_should_run,
 748         .thread_fn              = run_ksoftirqd,
 749         .thread_comm            = "ksoftirqd/%u",
 750 };
 751
 752 static __init int spawn_ksoftirqd(void)
 753 {
 754         register_cpu_notifier(&cpu_nfb);
 755
 756         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 757
 758         return 0;
 759 }
 760 early_initcall(spawn_ksoftirqd);
 761
 762 /*
 763  * [ These __weak aliases are kept in a separate compilation unit, so that
 764  *   GCC does not inline them incorrectly. ]
 765  */
 766
 767 int __init __weak early_irq_init(void)
 768 {
 769         return 0;
 770 }
 771
 772 int __init __weak arch_probe_nr_irqs(void)
 773 {
 774         return NR_IRQS_LEGACY;
 775 }
 776
 777 int __init __weak arch_early_irq_init(void)
 778 {
 779         return 0;
 780 }