kernel/softirq.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *      linux/kernel/softirq.c
   4  *
   5  *      Copyright (C) 1992 Linus Torvalds
   6  *
   7  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   8  */
   9
  10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  11
  12 #include <linux/export.h>
  13 #include <linux/kernel_stat.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/init.h>
  16 #include <linux/local_lock.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/smpboot.h>
  27 #include <linux/tick.h>
  28 #include <linux/irq.h>
  29 #include <linux/wait_bit.h>
  30
  31 #define CREATE_TRACE_POINTS
  32 #include <trace/events/irq.h>
  33
  34 /*
  35    - No shared variables, all the data are CPU local.
  36    - If a softirq needs serialization, let it serialize itself
  37      by its own spinlocks.
  38    - Even if softirq is serialized, only local cpu is marked for
  39      execution. Hence, we get something sort of weak cpu binding.
  40      Though it is still not clear, will it result in better locality
  41      or will not.
  42
  43    Examples:
  44    - NET RX softirq. It is multithreaded and does not require
  45      any global serialization.
  46    - NET TX softirq. It kicks software netdevice queues, hence
  47      it is logically serialized per device, but this serialization
  48      is invisible to common code.
  49    - Tasklets: serialized wrt itself.
  50  */
  51
  52 #ifndef __ARCH_IRQ_STAT
  53 DEFINE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
  54 EXPORT_PER_CPU_SYMBOL(irq_stat);
  55 #endif
  56
  57 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  58
  59 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  60
  61 const char * const softirq_to_name[NR_SOFTIRQS] = {
  62         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
  63         "TASKLET", "SCHED", "HRTIMER", "RCU"
  64 };
  65
  66 /*
  67  * we cannot loop indefinitely here to avoid userspace starvation,
  68  * but we also don't want to introduce a worst case 1/HZ latency
  69  * to the pending events, so lets the scheduler to balance
  70  * the softirq load for us.
  71  */
  72 static void wakeup_softirqd(void)
  73 {
  74         /* Interrupts are disabled: no need to stop preemption */
  75         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  76
  77         if (tsk && tsk->state != TASK_RUNNING)
  78                 wake_up_process(tsk);
  79 }
  80
  81 /*
  82  * If ksoftirqd is scheduled, we do not want to process pending softirqs
  83  * right now. Let ksoftirqd handle this at its own rate, to get fairness,
  84  * unless we're doing some of the synchronous softirqs.
  85  */
  86 #define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ))
  87 static bool ksoftirqd_running(unsigned long pending)
  88 {
  89         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  90
  91         if (pending & SOFTIRQ_NOW_MASK)
  92                 return false;
  93         return tsk && (tsk->state == TASK_RUNNING) &&
  94                 !__kthread_should_park(tsk);
  95 }
  96
  97 #ifdef CONFIG_TRACE_IRQFLAGS
  98 DEFINE_PER_CPU(int, hardirqs_enabled);
  99 DEFINE_PER_CPU(int, hardirq_context);
 100 EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
 101 EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
 102 #endif
 103
 104 /*
 105  * SOFTIRQ_OFFSET usage:
 106  *
 107  * On !RT kernels 'count' is the preempt counter, on RT kernels this applies
 108  * to a per CPU counter and to task::softirqs_disabled_cnt.
 109  *
 110  * - count is changed by SOFTIRQ_OFFSET on entering or leaving softirq
 111  *   processing.
 112  *
 113  * - count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
 114  *   on local_bh_disable or local_bh_enable.
 115  *
 116  * This lets us distinguish between whether we are currently processing
 117  * softirq and whether we just have bh disabled.
 118  */
 119 #ifdef CONFIG_PREEMPT_RT
 120
 121 /*
 122  * RT accounts for BH disabled sections in task::softirqs_disabled_cnt and
 123  * also in per CPU softirq_ctrl::cnt. This is necessary to allow tasks in a
 124  * softirq disabled section to be preempted.
 125  *
 126  * The per task counter is used for softirq_count(), in_softirq() and
 127  * in_serving_softirqs() because these counts are only valid when the task
 128  * holding softirq_ctrl::lock is running.
 129  *
 130  * The per CPU counter prevents pointless wakeups of ksoftirqd in case that
 131  * the task which is in a softirq disabled section is preempted or blocks.
 132  */
 133 struct softirq_ctrl {
 134         local_lock_t    lock;
 135         int             cnt;
 136 };
 137
 138 static DEFINE_PER_CPU(struct softirq_ctrl, softirq_ctrl) = {
 139         .lock   = INIT_LOCAL_LOCK(softirq_ctrl.lock),
 140 };
 141
 142 /**
 143  * local_bh_blocked() - Check for idle whether BH processing is blocked
 144  *
 145  * Returns false if the per CPU softirq::cnt is 0 otherwise true.
 146  *
 147  * This is invoked from the idle task to guard against false positive
 148  * softirq pending warnings, which would happen when the task which holds
 149  * softirq_ctrl::lock was the only running task on the CPU and blocks on
 150  * some other lock.
 151  */
 152 bool local_bh_blocked(void)
 153 {
 154         return __this_cpu_read(softirq_ctrl.cnt) != 0;
 155 }
 156
 157 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
 158 {
 159         unsigned long flags;
 160         int newcnt;
 161
 162         WARN_ON_ONCE(in_hardirq());
 163
 164         /* First entry of a task into a BH disabled section? */
 165         if (!current->softirq_disable_cnt) {
 166                 if (preemptible()) {
 167                         local_lock(&softirq_ctrl.lock);
 168                         /* Required to meet the RCU bottomhalf requirements. */
 169                         rcu_read_lock();
 170                 } else {
 171                         DEBUG_LOCKS_WARN_ON(this_cpu_read(softirq_ctrl.cnt));
 172                 }
 173         }
 174
 175         /*
 176          * Track the per CPU softirq disabled state. On RT this is per CPU
 177          * state to allow preemption of bottom half disabled sections.
 178          */
 179         newcnt = __this_cpu_add_return(softirq_ctrl.cnt, cnt);
 180         /*
 181          * Reflect the result in the task state to prevent recursion on the
 182          * local lock and to make softirq_count() & al work.
 183          */
 184         current->softirq_disable_cnt = newcnt;
 185
 186         if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && newcnt == cnt) {
 187                 raw_local_irq_save(flags);
 188                 lockdep_softirqs_off(ip);
 189                 raw_local_irq_restore(flags);
 190         }
 191 }
 192 EXPORT_SYMBOL(__local_bh_disable_ip);
 193
 194 static void __local_bh_enable(unsigned int cnt, bool unlock)
 195 {
 196         unsigned long flags;
 197         int newcnt;
 198
 199         DEBUG_LOCKS_WARN_ON(current->softirq_disable_cnt !=
 200                             this_cpu_read(softirq_ctrl.cnt));
 201
 202         if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && softirq_count() == cnt) {
 203                 raw_local_irq_save(flags);
 204                 lockdep_softirqs_on(_RET_IP_);
 205                 raw_local_irq_restore(flags);
 206         }
 207
 208         newcnt = __this_cpu_sub_return(softirq_ctrl.cnt, cnt);
 209         current->softirq_disable_cnt = newcnt;
 210
 211         if (!newcnt && unlock) {
 212                 rcu_read_unlock();
 213                 local_unlock(&softirq_ctrl.lock);
 214         }
 215 }
 216
 217 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 218 {
 219         bool preempt_on = preemptible();
 220         unsigned long flags;
 221         u32 pending;
 222         int curcnt;
 223
 224         WARN_ON_ONCE(in_irq());
 225         lockdep_assert_irqs_enabled();
 226
 227         local_irq_save(flags);
 228         curcnt = __this_cpu_read(softirq_ctrl.cnt);
 229
 230         /*
 231          * If this is not reenabling soft interrupts, no point in trying to
 232          * run pending ones.
 233          */
 234         if (curcnt != cnt)
 235                 goto out;
 236
 237         pending = local_softirq_pending();
 238         if (!pending || ksoftirqd_running(pending))
 239                 goto out;
 240
 241         /*
 242          * If this was called from non preemptible context, wake up the
 243          * softirq daemon.
 244          */
 245         if (!preempt_on) {
 246                 wakeup_softirqd();
 247                 goto out;
 248         }
 249
 250         /*
 251          * Adjust softirq count to SOFTIRQ_OFFSET which makes
 252          * in_serving_softirq() become true.
 253          */
 254         cnt = SOFTIRQ_OFFSET;
 255         __local_bh_enable(cnt, false);
 256         __do_softirq();
 257
 258 out:
 259         __local_bh_enable(cnt, preempt_on);
 260         local_irq_restore(flags);
 261 }
 262 EXPORT_SYMBOL(__local_bh_enable_ip);
 263
 264 /*
 265  * Invoked from ksoftirqd_run() outside of the interrupt disabled section
 266  * to acquire the per CPU local lock for reentrancy protection.
 267  */
 268 static inline void ksoftirqd_run_begin(void)
 269 {
 270         __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 271         local_irq_disable();
 272 }
 273
 274 /* Counterpart to ksoftirqd_run_begin() */
 275 static inline void ksoftirqd_run_end(void)
 276 {
 277         __local_bh_enable(SOFTIRQ_OFFSET, true);
 278         WARN_ON_ONCE(in_interrupt());
 279         local_irq_enable();
 280 }
 281
 282 static inline void softirq_handle_begin(void) { }
 283 static inline void softirq_handle_end(void) { }
 284
 285 static inline bool should_wake_ksoftirqd(void)
 286 {
 287         return !this_cpu_read(softirq_ctrl.cnt);
 288 }
 289
 290 static inline void invoke_softirq(void)
 291 {
 292         if (should_wake_ksoftirqd())
 293                 wakeup_softirqd();
 294 }
 295
 296 #else /* CONFIG_PREEMPT_RT */
 297
 298 /*
 299  * This one is for softirq.c-internal use, where hardirqs are disabled
 300  * legitimately:
 301  */
 302 #ifdef CONFIG_TRACE_IRQFLAGS
 303 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
 304 {
 305         unsigned long flags;
 306
 307         WARN_ON_ONCE(in_irq());
 308
 309         raw_local_irq_save(flags);
 310         /*
 311          * The preempt tracer hooks into preempt_count_add and will break
 312          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 313          * is set and before current->softirq_enabled is cleared.
 314          * We must manually increment preempt_count here and manually
 315          * call the trace_preempt_off later.
 316          */
 317         __preempt_count_add(cnt);
 318         /*
 319          * Were softirqs turned off above:
 320          */
 321         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 322                 lockdep_softirqs_off(ip);
 323         raw_local_irq_restore(flags);
 324
 325         if (preempt_count() == cnt) {
 326 #ifdef CONFIG_DEBUG_PREEMPT
 327                 current->preempt_disable_ip = get_lock_parent_ip();
 328 #endif
 329                 trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip());
 330         }
 331 }
 332 EXPORT_SYMBOL(__local_bh_disable_ip);
 333 #endif /* CONFIG_TRACE_IRQFLAGS */
 334
 335 static void __local_bh_enable(unsigned int cnt)
 336 {
 337         lockdep_assert_irqs_disabled();
 338
 339         if (preempt_count() == cnt)
 340                 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
 341
 342         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 343                 lockdep_softirqs_on(_RET_IP_);
 344
 345         __preempt_count_sub(cnt);
 346 }
 347
 348 /*
 349  * Special-case - softirqs can safely be enabled by __do_softirq(),
 350  * without processing still-pending softirqs:
 351  */
 352 void _local_bh_enable(void)
 353 {
 354         WARN_ON_ONCE(in_irq());
 355         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 356 }
 357 EXPORT_SYMBOL(_local_bh_enable);
 358
 359 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 360 {
 361         WARN_ON_ONCE(in_irq());
 362         lockdep_assert_irqs_enabled();
 363 #ifdef CONFIG_TRACE_IRQFLAGS
 364         local_irq_disable();
 365 #endif
 366         /*
 367          * Are softirqs going to be turned on now:
 368          */
 369         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 370                 lockdep_softirqs_on(ip);
 371         /*
 372          * Keep preemption disabled until we are done with
 373          * softirq processing:
 374          */
 375         preempt_count_sub(cnt - 1);
 376
 377         if (unlikely(!in_interrupt() && local_softirq_pending())) {
 378                 /*
 379                  * Run softirq if any pending. And do it in its own stack
 380                  * as we may be calling this deep in a task call stack already.
 381                  */
 382                 do_softirq();
 383         }
 384
 385         preempt_count_dec();
 386 #ifdef CONFIG_TRACE_IRQFLAGS
 387         local_irq_enable();
 388 #endif
 389         preempt_check_resched();
 390 }
 391 EXPORT_SYMBOL(__local_bh_enable_ip);
 392
 393 static inline void softirq_handle_begin(void)
 394 {
 395         __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 396 }
 397
 398 static inline void softirq_handle_end(void)
 399 {
 400         __local_bh_enable(SOFTIRQ_OFFSET);
 401         WARN_ON_ONCE(in_interrupt());
 402 }
 403
 404 static inline void ksoftirqd_run_begin(void)
 405 {
 406         local_irq_disable();
 407 }
 408
 409 static inline void ksoftirqd_run_end(void)
 410 {
 411         local_irq_enable();
 412 }
 413
 414 static inline bool should_wake_ksoftirqd(void)
 415 {
 416         return true;
 417 }
 418
 419 static inline void invoke_softirq(void)
 420 {
 421         if (ksoftirqd_running(local_softirq_pending()))
 422                 return;
 423
 424         if (!force_irqthreads) {
 425 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 426                 /*
 427                  * We can safely execute softirq on the current stack if
 428                  * it is the irq stack, because it should be near empty
 429                  * at this stage.
 430                  */
 431                 __do_softirq();
 432 #else
 433                 /*
 434                  * Otherwise, irq_exit() is called on the task stack that can
 435                  * be potentially deep already. So call softirq in its own stack
 436                  * to prevent from any overrun.
 437                  */
 438                 do_softirq_own_stack();
 439 #endif
 440         } else {
 441                 wakeup_softirqd();
 442         }
 443 }
 444
 445 asmlinkage __visible void do_softirq(void)
 446 {
 447         __u32 pending;
 448         unsigned long flags;
 449
 450         if (in_interrupt())
 451                 return;
 452
 453         local_irq_save(flags);
 454
 455         pending = local_softirq_pending();
 456
 457         if (pending && !ksoftirqd_running(pending))
 458                 do_softirq_own_stack();
 459
 460         local_irq_restore(flags);
 461 }
 462
 463 #endif /* !CONFIG_PREEMPT_RT */
 464
 465 /*
 466  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 467  * but break the loop if need_resched() is set or after 2 ms.
 468  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 469  * certain cases, such as stop_machine(), jiffies may cease to
 470  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 471  * well to make sure we eventually return from this method.
 472  *
 473  * These limits have been established via experimentation.
 474  * The two things to balance is latency against fairness -
 475  * we want to handle softirqs as soon as possible, but they
 476  * should not be able to lock up the box.
 477  */
 478 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 479 #define MAX_SOFTIRQ_RESTART 10
 480
 481 #ifdef CONFIG_TRACE_IRQFLAGS
 482 /*
 483  * When we run softirqs from irq_exit() and thus on the hardirq stack we need
 484  * to keep the lockdep irq context tracking as tight as possible in order to
 485  * not miss-qualify lock contexts and miss possible deadlocks.
 486  */
 487
 488 static inline bool lockdep_softirq_start(void)
 489 {
 490         bool in_hardirq = false;
 491
 492         if (lockdep_hardirq_context()) {
 493                 in_hardirq = true;
 494                 lockdep_hardirq_exit();
 495         }
 496
 497         lockdep_softirq_enter();
 498
 499         return in_hardirq;
 500 }
 501
 502 static inline void lockdep_softirq_end(bool in_hardirq)
 503 {
 504         lockdep_softirq_exit();
 505
 506         if (in_hardirq)
 507                 lockdep_hardirq_enter();
 508 }
 509 #else
 510 static inline bool lockdep_softirq_start(void) { return false; }
 511 static inline void lockdep_softirq_end(bool in_hardirq) { }
 512 #endif
 513
 514 asmlinkage __visible void __softirq_entry __do_softirq(void)
 515 {
 516         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 517         unsigned long old_flags = current->flags;
 518         int max_restart = MAX_SOFTIRQ_RESTART;
 519         struct softirq_action *h;
 520         bool in_hardirq;
 521         __u32 pending;
 522         int softirq_bit;
 523
 524         /*
 525          * Mask out PF_MEMALLOC as the current task context is borrowed for the
 526          * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
 527          * again if the socket is related to swapping.
 528          */
 529         current->flags &= ~PF_MEMALLOC;
 530
 531         pending = local_softirq_pending();
 532
 533         softirq_handle_begin();
 534         in_hardirq = lockdep_softirq_start();
 535         account_softirq_enter(current);
 536
 537 restart:
 538         /* Reset the pending bitmask before enabling irqs */
 539         set_softirq_pending(0);
 540
 541         local_irq_enable();
 542
 543         h = softirq_vec;
 544
 545         while ((softirq_bit = ffs(pending))) {
 546                 unsigned int vec_nr;
 547                 int prev_count;
 548
 549                 h += softirq_bit - 1;
 550
 551                 vec_nr = h - softirq_vec;
 552                 prev_count = preempt_count();
 553
 554                 kstat_incr_softirqs_this_cpu(vec_nr);
 555
 556                 trace_softirq_entry(vec_nr);
 557                 h->action(h);
 558                 trace_softirq_exit(vec_nr);
 559                 if (unlikely(prev_count != preempt_count())) {
 560                         pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
 561                                vec_nr, softirq_to_name[vec_nr], h->action,
 562                                prev_count, preempt_count());
 563                         preempt_count_set(prev_count);
 564                 }
 565                 h++;
 566                 pending >>= softirq_bit;
 567         }
 568
 569         if (!IS_ENABLED(CONFIG_PREEMPT_RT) &&
 570             __this_cpu_read(ksoftirqd) == current)
 571                 rcu_softirq_qs();
 572
 573         local_irq_disable();
 574
 575         pending = local_softirq_pending();
 576         if (pending) {
 577                 if (time_before(jiffies, end) && !need_resched() &&
 578                     --max_restart)
 579                         goto restart;
 580
 581                 wakeup_softirqd();
 582         }
 583
 584         account_softirq_exit(current);
 585         lockdep_softirq_end(in_hardirq);
 586         softirq_handle_end();
 587         current_restore_flags(old_flags, PF_MEMALLOC);
 588 }
 589
 590 /**
 591  * irq_enter_rcu - Enter an interrupt context with RCU watching
 592  */
 593 void irq_enter_rcu(void)
 594 {
 595         __irq_enter_raw();
 596
 597         if (is_idle_task(current) && (irq_count() == HARDIRQ_OFFSET))
 598                 tick_irq_enter();
 599
 600         account_hardirq_enter(current);
 601 }
 602
 603 /**
 604  * irq_enter - Enter an interrupt context including RCU update
 605  */
 606 void irq_enter(void)
 607 {
 608         rcu_irq_enter();
 609         irq_enter_rcu();
 610 }
 611
 612 static inline void tick_irq_exit(void)
 613 {
 614 #ifdef CONFIG_NO_HZ_COMMON
 615         int cpu = smp_processor_id();
 616
 617         /* Make sure that timer wheel updates are propagated */
 618         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 619                 if (!in_irq())
 620                         tick_nohz_irq_exit();
 621         }
 622 #endif
 623 }
 624
 625 static inline void __irq_exit_rcu(void)
 626 {
 627 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 628         local_irq_disable();
 629 #else
 630         lockdep_assert_irqs_disabled();
 631 #endif
 632         account_hardirq_exit(current);
 633         preempt_count_sub(HARDIRQ_OFFSET);
 634         if (!in_interrupt() && local_softirq_pending())
 635                 invoke_softirq();
 636
 637         tick_irq_exit();
 638 }
 639
 640 /**
 641  * irq_exit_rcu() - Exit an interrupt context without updating RCU
 642  *
 643  * Also processes softirqs if needed and possible.
 644  */
 645 void irq_exit_rcu(void)
 646 {
 647         __irq_exit_rcu();
 648          /* must be last! */
 649         lockdep_hardirq_exit();
 650 }
 651
 652 /**
 653  * irq_exit - Exit an interrupt context, update RCU and lockdep
 654  *
 655  * Also processes softirqs if needed and possible.
 656  */
 657 void irq_exit(void)
 658 {
 659         __irq_exit_rcu();
 660         rcu_irq_exit();
 661          /* must be last! */
 662         lockdep_hardirq_exit();
 663 }
 664
 665 /*
 666  * This function must run with irqs disabled!
 667  */
 668 inline void raise_softirq_irqoff(unsigned int nr)
 669 {
 670         __raise_softirq_irqoff(nr);
 671
 672         /*
 673          * If we're in an interrupt or softirq, we're done
 674          * (this also catches softirq-disabled code). We will
 675          * actually run the softirq once we return from
 676          * the irq or softirq.
 677          *
 678          * Otherwise we wake up ksoftirqd to make sure we
 679          * schedule the softirq soon.
 680          */
 681         if (!in_interrupt() && should_wake_ksoftirqd())
 682                 wakeup_softirqd();
 683 }
 684
 685 void raise_softirq(unsigned int nr)
 686 {
 687         unsigned long flags;
 688
 689         local_irq_save(flags);
 690         raise_softirq_irqoff(nr);
 691         local_irq_restore(flags);
 692 }
 693
 694 void __raise_softirq_irqoff(unsigned int nr)
 695 {
 696         lockdep_assert_irqs_disabled();
 697         trace_softirq_raise(nr);
 698         or_softirq_pending(1UL << nr);
 699 }
 700
 701 void open_softirq(int nr, void (*action)(struct softirq_action *))
 702 {
 703         softirq_vec[nr].action = action;
 704 }
 705
 706 /*
 707  * Tasklets
 708  */
 709 struct tasklet_head {
 710         struct tasklet_struct *head;
 711         struct tasklet_struct **tail;
 712 };
 713
 714 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 715 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 716
 717 static void __tasklet_schedule_common(struct tasklet_struct *t,
 718                                       struct tasklet_head __percpu *headp,
 719                                       unsigned int softirq_nr)
 720 {
 721         struct tasklet_head *head;
 722         unsigned long flags;
 723
 724         local_irq_save(flags);
 725         head = this_cpu_ptr(headp);
 726         t->next = NULL;
 727         *head->tail = t;
 728         head->tail = &(t->next);
 729         raise_softirq_irqoff(softirq_nr);
 730         local_irq_restore(flags);
 731 }
 732
 733 void __tasklet_schedule(struct tasklet_struct *t)
 734 {
 735         __tasklet_schedule_common(t, &tasklet_vec,
 736                                   TASKLET_SOFTIRQ);
 737 }
 738 EXPORT_SYMBOL(__tasklet_schedule);
 739
 740 void __tasklet_hi_schedule(struct tasklet_struct *t)
 741 {
 742         __tasklet_schedule_common(t, &tasklet_hi_vec,
 743                                   HI_SOFTIRQ);
 744 }
 745 EXPORT_SYMBOL(__tasklet_hi_schedule);
 746
 747 static inline bool tasklet_clear_sched(struct tasklet_struct *t)
 748 {
 749         if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) {
 750                 wake_up_var(&t->state);
 751                 return true;
 752         }
 753
 754         return false;
 755 }
 756
 757 static void tasklet_action_common(struct softirq_action *a,
 758                                   struct tasklet_head *tl_head,
 759                                   unsigned int softirq_nr)
 760 {
 761         struct tasklet_struct *list;
 762
 763         local_irq_disable();
 764         list = tl_head->head;
 765         tl_head->head = NULL;
 766         tl_head->tail = &tl_head->head;
 767         local_irq_enable();
 768
 769         while (list) {
 770                 struct tasklet_struct *t = list;
 771
 772                 list = list->next;
 773
 774                 if (tasklet_trylock(t)) {
 775                         if (!atomic_read(&t->count)) {
 776                                 if (!tasklet_clear_sched(t))
 777                                         BUG();
 778                                 if (t->use_callback)
 779                                         t->callback(t);
 780                                 else
 781                                         t->func(t->data);
 782                                 tasklet_unlock(t);
 783                                 continue;
 784                         }
 785                         tasklet_unlock(t);
 786                 }
 787
 788                 local_irq_disable();
 789                 t->next = NULL;
 790                 *tl_head->tail = t;
 791                 tl_head->tail = &t->next;
 792                 __raise_softirq_irqoff(softirq_nr);
 793                 local_irq_enable();
 794         }
 795 }
 796
 797 static __latent_entropy void tasklet_action(struct softirq_action *a)
 798 {
 799         tasklet_action_common(a, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
 800 }
 801
 802 static __latent_entropy void tasklet_hi_action(struct softirq_action *a)
 803 {
 804         tasklet_action_common(a, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
 805 }
 806
 807 void tasklet_setup(struct tasklet_struct *t,
 808                    void (*callback)(struct tasklet_struct *))
 809 {
 810         t->next = NULL;
 811         t->state = 0;
 812         atomic_set(&t->count, 0);
 813         t->callback = callback;
 814         t->use_callback = true;
 815         t->data = 0;
 816 }
 817 EXPORT_SYMBOL(tasklet_setup);
 818
 819 void tasklet_init(struct tasklet_struct *t,
 820                   void (*func)(unsigned long), unsigned long data)
 821 {
 822         t->next = NULL;
 823         t->state = 0;
 824         atomic_set(&t->count, 0);
 825         t->func = func;
 826         t->use_callback = false;
 827         t->data = data;
 828 }
 829 EXPORT_SYMBOL(tasklet_init);
 830
 831 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
 832 /*
 833  * Do not use in new code. Waiting for tasklets from atomic contexts is
 834  * error prone and should be avoided.
 835  */
 836 void tasklet_unlock_spin_wait(struct tasklet_struct *t)
 837 {
 838         while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
 839                 if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
 840                         /*
 841                          * Prevent a live lock when current preempted soft
 842                          * interrupt processing or prevents ksoftirqd from
 843                          * running. If the tasklet runs on a different CPU
 844                          * then this has no effect other than doing the BH
 845                          * disable/enable dance for nothing.
 846                          */
 847                         local_bh_disable();
 848                         local_bh_enable();
 849                 } else {
 850                         cpu_relax();
 851                 }
 852         }
 853 }
 854 EXPORT_SYMBOL(tasklet_unlock_spin_wait);
 855 #endif
 856
 857 void tasklet_kill(struct tasklet_struct *t)
 858 {
 859         if (in_interrupt())
 860                 pr_notice("Attempt to kill tasklet from interrupt\n");
 861
 862         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
 863                 wait_var_event(&t->state, !test_bit(TASKLET_STATE_SCHED, &t->state));
 864
 865         tasklet_unlock_wait(t);
 866         tasklet_clear_sched(t);
 867 }
 868 EXPORT_SYMBOL(tasklet_kill);
 869
 870 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
 871 void tasklet_unlock(struct tasklet_struct *t)
 872 {
 873         smp_mb__before_atomic();
 874         clear_bit(TASKLET_STATE_RUN, &t->state);
 875         smp_mb__after_atomic();
 876         wake_up_var(&t->state);
 877 }
 878 EXPORT_SYMBOL_GPL(tasklet_unlock);
 879
 880 void tasklet_unlock_wait(struct tasklet_struct *t)
 881 {
 882         wait_var_event(&t->state, !test_bit(TASKLET_STATE_RUN, &t->state));
 883 }
 884 EXPORT_SYMBOL_GPL(tasklet_unlock_wait);
 885 #endif
 886
 887 void __init softirq_init(void)
 888 {
 889         int cpu;
 890
 891         for_each_possible_cpu(cpu) {
 892                 per_cpu(tasklet_vec, cpu).tail =
 893                         &per_cpu(tasklet_vec, cpu).head;
 894                 per_cpu(tasklet_hi_vec, cpu).tail =
 895                         &per_cpu(tasklet_hi_vec, cpu).head;
 896         }
 897
 898         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 899         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 900 }
 901
 902 static int ksoftirqd_should_run(unsigned int cpu)
 903 {
 904         return local_softirq_pending();
 905 }
 906
 907 static void run_ksoftirqd(unsigned int cpu)
 908 {
 909         ksoftirqd_run_begin();
 910         if (local_softirq_pending()) {
 911                 /*
 912                  * We can safely run softirq on inline stack, as we are not deep
 913                  * in the task stack here.
 914                  */
 915                 __do_softirq();
 916                 ksoftirqd_run_end();
 917                 cond_resched();
 918                 return;
 919         }
 920         ksoftirqd_run_end();
 921 }
 922
 923 #ifdef CONFIG_HOTPLUG_CPU
 924 /*
 925  * tasklet_kill_immediate is called to remove a tasklet which can already be
 926  * scheduled for execution on @cpu.
 927  *
 928  * Unlike tasklet_kill, this function removes the tasklet
 929  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 930  *
 931  * When this function is called, @cpu must be in the CPU_DEAD state.
 932  */
 933 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 934 {
 935         struct tasklet_struct **i;
 936
 937         BUG_ON(cpu_online(cpu));
 938         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 939
 940         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 941                 return;
 942
 943         /* CPU is dead, so no lock needed. */
 944         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 945                 if (*i == t) {
 946                         *i = t->next;
 947                         /* If this was the tail element, move the tail ptr */
 948                         if (*i == NULL)
 949                                 per_cpu(tasklet_vec, cpu).tail = i;
 950                         return;
 951                 }
 952         }
 953         BUG();
 954 }
 955
 956 static int takeover_tasklets(unsigned int cpu)
 957 {
 958         /* CPU is dead, so no lock needed. */
 959         local_irq_disable();
 960
 961         /* Find end, append list for that CPU. */
 962         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 963                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 964                 __this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 965                 per_cpu(tasklet_vec, cpu).head = NULL;
 966                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 967         }
 968         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 969
 970         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 971                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 972                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 973                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 974                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 975         }
 976         raise_softirq_irqoff(HI_SOFTIRQ);
 977
 978         local_irq_enable();
 979         return 0;
 980 }
 981 #else
 982 #define takeover_tasklets       NULL
 983 #endif /* CONFIG_HOTPLUG_CPU */
 984
 985 static struct smp_hotplug_thread softirq_threads = {
 986         .store                  = &ksoftirqd,
 987         .thread_should_run      = ksoftirqd_should_run,
 988         .thread_fn              = run_ksoftirqd,
 989         .thread_comm            = "ksoftirqd/%u",
 990 };
 991
 992 static __init int spawn_ksoftirqd(void)
 993 {
 994         cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
 995                                   takeover_tasklets);
 996         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 997
 998         return 0;
 999 }
1000 early_initcall(spawn_ksoftirqd);
1001
1002 /*
1003  * [ These __weak aliases are kept in a separate compilation unit, so that
1004  *   GCC does not inline them incorrectly. ]
1005  */
1006
1007 int __init __weak early_irq_init(void)
1008 {
1009         return 0;
1010 }
1011
1012 int __init __weak arch_probe_nr_irqs(void)
1013 {
1014         return NR_IRQS_LEGACY;
1015 }
1016
1017 int __init __weak arch_early_irq_init(void)
1018 {
1019         return 0;
1020 }
1021
1022 unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
1023 {
1024         return from;
1025 }