kernel/cpu.c

   1 /* CPU control.
   2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
   3  *
   4  * This code is licenced under the GPL.
   5  */
   6 #include <linux/sched/mm.h>
   7 #include <linux/proc_fs.h>
   8 #include <linux/smp.h>
   9 #include <linux/init.h>
  10 #include <linux/notifier.h>
  11 #include <linux/sched/signal.h>
  12 #include <linux/sched/hotplug.h>
  13 #include <linux/sched/isolation.h>
  14 #include <linux/sched/task.h>
  15 #include <linux/sched/smt.h>
  16 #include <linux/unistd.h>
  17 #include <linux/cpu.h>
  18 #include <linux/oom.h>
  19 #include <linux/rcupdate.h>
  20 #include <linux/export.h>
  21 #include <linux/bug.h>
  22 #include <linux/kthread.h>
  23 #include <linux/stop_machine.h>
  24 #include <linux/mutex.h>
  25 #include <linux/gfp.h>
  26 #include <linux/suspend.h>
  27 #include <linux/lockdep.h>
  28 #include <linux/tick.h>
  29 #include <linux/irq.h>
  30 #include <linux/nmi.h>
  31 #include <linux/smpboot.h>
  32 #include <linux/relay.h>
  33 #include <linux/slab.h>
  34 #include <linux/scs.h>
  35 #include <linux/percpu-rwsem.h>
  36 #include <linux/cpuset.h>
  37
  38 #include <trace/events/power.h>
  39 #define CREATE_TRACE_POINTS
  40 #include <trace/events/cpuhp.h>
  41
  42 #include "smpboot.h"
  43
  44 /**
  45  * cpuhp_cpu_state - Per cpu hotplug state storage
  46  * @state:      The current cpu state
  47  * @target:     The target state
  48  * @thread:     Pointer to the hotplug thread
  49  * @should_run: Thread should execute
  50  * @rollback:   Perform a rollback
  51  * @single:     Single callback invocation
  52  * @bringup:    Single callback bringup or teardown selector
  53  * @cb_state:   The state for a single callback (install/uninstall)
  54  * @result:     Result of the operation
  55  * @done_up:    Signal completion to the issuer of the task for cpu-up
  56  * @done_down:  Signal completion to the issuer of the task for cpu-down
  57  */
  58 struct cpuhp_cpu_state {
  59         enum cpuhp_state        state;
  60         enum cpuhp_state        target;
  61         enum cpuhp_state        fail;
  62 #ifdef CONFIG_SMP
  63         struct task_struct      *thread;
  64         bool                    should_run;
  65         bool                    rollback;
  66         bool                    single;
  67         bool                    bringup;
  68         struct hlist_node       *node;
  69         struct hlist_node       *last;
  70         enum cpuhp_state        cb_state;
  71         int                     result;
  72         struct completion       done_up;
  73         struct completion       done_down;
  74 #endif
  75 };
  76
  77 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
  78         .fail = CPUHP_INVALID,
  79 };
  80
  81 #ifdef CONFIG_SMP
  82 cpumask_t cpus_booted_once_mask;
  83 #endif
  84
  85 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
  86 static struct lockdep_map cpuhp_state_up_map =
  87         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
  88 static struct lockdep_map cpuhp_state_down_map =
  89         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
  90
  91
  92 static inline void cpuhp_lock_acquire(bool bringup)
  93 {
  94         lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  95 }
  96
  97 static inline void cpuhp_lock_release(bool bringup)
  98 {
  99         lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
 100 }
 101 #else
 102
 103 static inline void cpuhp_lock_acquire(bool bringup) { }
 104 static inline void cpuhp_lock_release(bool bringup) { }
 105
 106 #endif
 107
 108 /**
 109  * cpuhp_step - Hotplug state machine step
 110  * @name:       Name of the step
 111  * @startup:    Startup function of the step
 112  * @teardown:   Teardown function of the step
 113  * @cant_stop:  Bringup/teardown can't be stopped at this step
 114  */
 115 struct cpuhp_step {
 116         const char              *name;
 117         union {
 118                 int             (*single)(unsigned int cpu);
 119                 int             (*multi)(unsigned int cpu,
 120                                          struct hlist_node *node);
 121         } startup;
 122         union {
 123                 int             (*single)(unsigned int cpu);
 124                 int             (*multi)(unsigned int cpu,
 125                                          struct hlist_node *node);
 126         } teardown;
 127         struct hlist_head       list;
 128         bool                    cant_stop;
 129         bool                    multi_instance;
 130 };
 131
 132 static DEFINE_MUTEX(cpuhp_state_mutex);
 133 static struct cpuhp_step cpuhp_hp_states[];
 134
 135 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
 136 {
 137         return cpuhp_hp_states + state;
 138 }
 139
 140 /**
 141  * cpuhp_invoke_callback _ Invoke the callbacks for a given state
 142  * @cpu:        The cpu for which the callback should be invoked
 143  * @state:      The state to do callbacks for
 144  * @bringup:    True if the bringup callback should be invoked
 145  * @node:       For multi-instance, do a single entry callback for install/remove
 146  * @lastp:      For multi-instance rollback, remember how far we got
 147  *
 148  * Called from cpu hotplug and from the state register machinery.
 149  */
 150 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
 151                                  bool bringup, struct hlist_node *node,
 152                                  struct hlist_node **lastp)
 153 {
 154         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 155         struct cpuhp_step *step = cpuhp_get_step(state);
 156         int (*cbm)(unsigned int cpu, struct hlist_node *node);
 157         int (*cb)(unsigned int cpu);
 158         int ret, cnt;
 159
 160         if (st->fail == state) {
 161                 st->fail = CPUHP_INVALID;
 162
 163                 if (!(bringup ? step->startup.single : step->teardown.single))
 164                         return 0;
 165
 166                 return -EAGAIN;
 167         }
 168
 169         if (!step->multi_instance) {
 170                 WARN_ON_ONCE(lastp && *lastp);
 171                 cb = bringup ? step->startup.single : step->teardown.single;
 172                 if (!cb)
 173                         return 0;
 174                 trace_cpuhp_enter(cpu, st->target, state, cb);
 175                 ret = cb(cpu);
 176                 trace_cpuhp_exit(cpu, st->state, state, ret);
 177                 return ret;
 178         }
 179         cbm = bringup ? step->startup.multi : step->teardown.multi;
 180         if (!cbm)
 181                 return 0;
 182
 183         /* Single invocation for instance add/remove */
 184         if (node) {
 185                 WARN_ON_ONCE(lastp && *lastp);
 186                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 187                 ret = cbm(cpu, node);
 188                 trace_cpuhp_exit(cpu, st->state, state, ret);
 189                 return ret;
 190         }
 191
 192         /* State transition. Invoke on all instances */
 193         cnt = 0;
 194         hlist_for_each(node, &step->list) {
 195                 if (lastp && node == *lastp)
 196                         break;
 197
 198                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 199                 ret = cbm(cpu, node);
 200                 trace_cpuhp_exit(cpu, st->state, state, ret);
 201                 if (ret) {
 202                         if (!lastp)
 203                                 goto err;
 204
 205                         *lastp = node;
 206                         return ret;
 207                 }
 208                 cnt++;
 209         }
 210         if (lastp)
 211                 *lastp = NULL;
 212         return 0;
 213 err:
 214         /* Rollback the instances if one failed */
 215         cbm = !bringup ? step->startup.multi : step->teardown.multi;
 216         if (!cbm)
 217                 return ret;
 218
 219         hlist_for_each(node, &step->list) {
 220                 if (!cnt--)
 221                         break;
 222
 223                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 224                 ret = cbm(cpu, node);
 225                 trace_cpuhp_exit(cpu, st->state, state, ret);
 226                 /*
 227                  * Rollback must not fail,
 228                  */
 229                 WARN_ON_ONCE(ret);
 230         }
 231         return ret;
 232 }
 233
 234 #ifdef CONFIG_SMP
 235 static bool cpuhp_is_ap_state(enum cpuhp_state state)
 236 {
 237         /*
 238          * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
 239          * purposes as that state is handled explicitly in cpu_down.
 240          */
 241         return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
 242 }
 243
 244 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 245 {
 246         struct completion *done = bringup ? &st->done_up : &st->done_down;
 247         wait_for_completion(done);
 248 }
 249
 250 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 251 {
 252         struct completion *done = bringup ? &st->done_up : &st->done_down;
 253         complete(done);
 254 }
 255
 256 /*
 257  * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
 258  */
 259 static bool cpuhp_is_atomic_state(enum cpuhp_state state)
 260 {
 261         return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
 262 }
 263
 264 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
 265 static DEFINE_MUTEX(cpu_add_remove_lock);
 266 bool cpuhp_tasks_frozen;
 267 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
 268
 269 /*
 270  * The following two APIs (cpu_maps_update_begin/done) must be used when
 271  * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
 272  */
 273 void cpu_maps_update_begin(void)
 274 {
 275         mutex_lock(&cpu_add_remove_lock);
 276 }
 277
 278 void cpu_maps_update_done(void)
 279 {
 280         mutex_unlock(&cpu_add_remove_lock);
 281 }
 282
 283 /*
 284  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
 285  * Should always be manipulated under cpu_add_remove_lock
 286  */
 287 static int cpu_hotplug_disabled;
 288
 289 #ifdef CONFIG_HOTPLUG_CPU
 290
 291 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 292
 293 void cpus_read_lock(void)
 294 {
 295         percpu_down_read(&cpu_hotplug_lock);
 296 }
 297 EXPORT_SYMBOL_GPL(cpus_read_lock);
 298
 299 int cpus_read_trylock(void)
 300 {
 301         return percpu_down_read_trylock(&cpu_hotplug_lock);
 302 }
 303 EXPORT_SYMBOL_GPL(cpus_read_trylock);
 304
 305 void cpus_read_unlock(void)
 306 {
 307         percpu_up_read(&cpu_hotplug_lock);
 308 }
 309 EXPORT_SYMBOL_GPL(cpus_read_unlock);
 310
 311 void cpus_write_lock(void)
 312 {
 313         percpu_down_write(&cpu_hotplug_lock);
 314 }
 315
 316 void cpus_write_unlock(void)
 317 {
 318         percpu_up_write(&cpu_hotplug_lock);
 319 }
 320
 321 void lockdep_assert_cpus_held(void)
 322 {
 323         /*
 324          * We can't have hotplug operations before userspace starts running,
 325          * and some init codepaths will knowingly not take the hotplug lock.
 326          * This is all valid, so mute lockdep until it makes sense to report
 327          * unheld locks.
 328          */
 329         if (system_state < SYSTEM_RUNNING)
 330                 return;
 331
 332         percpu_rwsem_assert_held(&cpu_hotplug_lock);
 333 }
 334
 335 static void lockdep_acquire_cpus_lock(void)
 336 {
 337         rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
 338 }
 339
 340 static void lockdep_release_cpus_lock(void)
 341 {
 342         rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
 343 }
 344
 345 /*
 346  * Wait for currently running CPU hotplug operations to complete (if any) and
 347  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 348  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 349  * hotplug path before performing hotplug operations. So acquiring that lock
 350  * guarantees mutual exclusion from any currently running hotplug operations.
 351  */
 352 void cpu_hotplug_disable(void)
 353 {
 354         cpu_maps_update_begin();
 355         cpu_hotplug_disabled++;
 356         cpu_maps_update_done();
 357 }
 358 EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
 359
 360 static void __cpu_hotplug_enable(void)
 361 {
 362         if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
 363                 return;
 364         cpu_hotplug_disabled--;
 365 }
 366
 367 void cpu_hotplug_enable(void)
 368 {
 369         cpu_maps_update_begin();
 370         __cpu_hotplug_enable();
 371         cpu_maps_update_done();
 372 }
 373 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 374
 375 #else
 376
 377 static void lockdep_acquire_cpus_lock(void)
 378 {
 379 }
 380
 381 static void lockdep_release_cpus_lock(void)
 382 {
 383 }
 384
 385 #endif  /* CONFIG_HOTPLUG_CPU */
 386
 387 /*
 388  * Architectures that need SMT-specific errata handling during SMT hotplug
 389  * should override this.
 390  */
 391 void __weak arch_smt_update(void) { }
 392
 393 #ifdef CONFIG_HOTPLUG_SMT
 394 enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
 395
 396 void __init cpu_smt_disable(bool force)
 397 {
 398         if (!cpu_smt_possible())
 399                 return;
 400
 401         if (force) {
 402                 pr_info("SMT: Force disabled\n");
 403                 cpu_smt_control = CPU_SMT_FORCE_DISABLED;
 404         } else {
 405                 pr_info("SMT: disabled\n");
 406                 cpu_smt_control = CPU_SMT_DISABLED;
 407         }
 408 }
 409
 410 /*
 411  * The decision whether SMT is supported can only be done after the full
 412  * CPU identification. Called from architecture code.
 413  */
 414 void __init cpu_smt_check_topology(void)
 415 {
 416         if (!topology_smt_supported())
 417                 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
 418 }
 419
 420 static int __init smt_cmdline_disable(char *str)
 421 {
 422         cpu_smt_disable(str && !strcmp(str, "force"));
 423         return 0;
 424 }
 425 early_param("nosmt", smt_cmdline_disable);
 426
 427 static inline bool cpu_smt_allowed(unsigned int cpu)
 428 {
 429         if (cpu_smt_control == CPU_SMT_ENABLED)
 430                 return true;
 431
 432         if (topology_is_primary_thread(cpu))
 433                 return true;
 434
 435         /*
 436          * On x86 it's required to boot all logical CPUs at least once so
 437          * that the init code can get a chance to set CR4.MCE on each
 438          * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
 439          * core will shutdown the machine.
 440          */
 441         return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
 442 }
 443
 444 /* Returns true if SMT is not supported of forcefully (irreversibly) disabled */
 445 bool cpu_smt_possible(void)
 446 {
 447         return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
 448                 cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
 449 }
 450 EXPORT_SYMBOL_GPL(cpu_smt_possible);
 451 #else
 452 static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
 453 #endif
 454
 455 static inline enum cpuhp_state
 456 cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 457 {
 458         enum cpuhp_state prev_state = st->state;
 459
 460         st->rollback = false;
 461         st->last = NULL;
 462
 463         st->target = target;
 464         st->single = false;
 465         st->bringup = st->state < target;
 466
 467         return prev_state;
 468 }
 469
 470 static inline void
 471 cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
 472 {
 473         st->rollback = true;
 474
 475         /*
 476          * If we have st->last we need to undo partial multi_instance of this
 477          * state first. Otherwise start undo at the previous state.
 478          */
 479         if (!st->last) {
 480                 if (st->bringup)
 481                         st->state--;
 482                 else
 483                         st->state++;
 484         }
 485
 486         st->target = prev_state;
 487         st->bringup = !st->bringup;
 488 }
 489
 490 /* Regular hotplug invocation of the AP hotplug thread */
 491 static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
 492 {
 493         if (!st->single && st->state == st->target)
 494                 return;
 495
 496         st->result = 0;
 497         /*
 498          * Make sure the above stores are visible before should_run becomes
 499          * true. Paired with the mb() above in cpuhp_thread_fun()
 500          */
 501         smp_mb();
 502         st->should_run = true;
 503         wake_up_process(st->thread);
 504         wait_for_ap_thread(st, st->bringup);
 505 }
 506
 507 static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 508 {
 509         enum cpuhp_state prev_state;
 510         int ret;
 511
 512         prev_state = cpuhp_set_state(st, target);
 513         __cpuhp_kick_ap(st);
 514         if ((ret = st->result)) {
 515                 cpuhp_reset_state(st, prev_state);
 516                 __cpuhp_kick_ap(st);
 517         }
 518
 519         return ret;
 520 }
 521
 522 static int bringup_wait_for_ap(unsigned int cpu)
 523 {
 524         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 525
 526         /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
 527         wait_for_ap_thread(st, true);
 528         if (WARN_ON_ONCE((!cpu_online(cpu))))
 529                 return -ECANCELED;
 530
 531         /* Unpark the hotplug thread of the target cpu */
 532         kthread_unpark(st->thread);
 533
 534         /*
 535          * SMT soft disabling on X86 requires to bring the CPU out of the
 536          * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
 537          * CPU marked itself as booted_once in notify_cpu_starting() so the
 538          * cpu_smt_allowed() check will now return false if this is not the
 539          * primary sibling.
 540          */
 541         if (!cpu_smt_allowed(cpu))
 542                 return -ECANCELED;
 543
 544         if (st->target <= CPUHP_AP_ONLINE_IDLE)
 545                 return 0;
 546
 547         return cpuhp_kick_ap(st, st->target);
 548 }
 549
 550 static int bringup_cpu(unsigned int cpu)
 551 {
 552         struct task_struct *idle = idle_thread_get(cpu);
 553         int ret;
 554
 555         /*
 556          * Reset stale stack state from the last time this CPU was online.
 557          */
 558         scs_task_reset(idle);
 559         kasan_unpoison_task_stack(idle);
 560
 561         /*
 562          * Some architectures have to walk the irq descriptors to
 563          * setup the vector space for the cpu which comes online.
 564          * Prevent irq alloc/free across the bringup.
 565          */
 566         irq_lock_sparse();
 567
 568         /* Arch-specific enabling code. */
 569         ret = __cpu_up(cpu, idle);
 570         irq_unlock_sparse();
 571         if (ret)
 572                 return ret;
 573         return bringup_wait_for_ap(cpu);
 574 }
 575
 576 static int finish_cpu(unsigned int cpu)
 577 {
 578         struct task_struct *idle = idle_thread_get(cpu);
 579         struct mm_struct *mm = idle->active_mm;
 580
 581         /*
 582          * idle_task_exit() will have switched to &init_mm, now
 583          * clean up any remaining active_mm state.
 584          */
 585         if (mm != &init_mm)
 586                 idle->active_mm = &init_mm;
 587         mmdrop(mm);
 588         return 0;
 589 }
 590
 591 /*
 592  * Hotplug state machine related functions
 593  */
 594
 595 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
 596 {
 597         for (st->state--; st->state > st->target; st->state--)
 598                 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 599 }
 600
 601 static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
 602 {
 603         if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
 604                 return true;
 605         /*
 606          * When CPU hotplug is disabled, then taking the CPU down is not
 607          * possible because takedown_cpu() and the architecture and
 608          * subsystem specific mechanisms are not available. So the CPU
 609          * which would be completely unplugged again needs to stay around
 610          * in the current state.
 611          */
 612         return st->state <= CPUHP_BRINGUP_CPU;
 613 }
 614
 615 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 616                               enum cpuhp_state target)
 617 {
 618         enum cpuhp_state prev_state = st->state;
 619         int ret = 0;
 620
 621         while (st->state < target) {
 622                 st->state++;
 623                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 624                 if (ret) {
 625                         if (can_rollback_cpu(st)) {
 626                                 st->target = prev_state;
 627                                 undo_cpu_up(cpu, st);
 628                         }
 629                         break;
 630                 }
 631         }
 632         return ret;
 633 }
 634
 635 /*
 636  * The cpu hotplug threads manage the bringup and teardown of the cpus
 637  */
 638 static void cpuhp_create(unsigned int cpu)
 639 {
 640         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 641
 642         init_completion(&st->done_up);
 643         init_completion(&st->done_down);
 644 }
 645
 646 static int cpuhp_should_run(unsigned int cpu)
 647 {
 648         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 649
 650         return st->should_run;
 651 }
 652
 653 /*
 654  * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
 655  * callbacks when a state gets [un]installed at runtime.
 656  *
 657  * Each invocation of this function by the smpboot thread does a single AP
 658  * state callback.
 659  *
 660  * It has 3 modes of operation:
 661  *  - single: runs st->cb_state
 662  *  - up:     runs ++st->state, while st->state < st->target
 663  *  - down:   runs st->state--, while st->state > st->target
 664  *
 665  * When complete or on error, should_run is cleared and the completion is fired.
 666  */
 667 static void cpuhp_thread_fun(unsigned int cpu)
 668 {
 669         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 670         bool bringup = st->bringup;
 671         enum cpuhp_state state;
 672
 673         if (WARN_ON_ONCE(!st->should_run))
 674                 return;
 675
 676         /*
 677          * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
 678          * that if we see ->should_run we also see the rest of the state.
 679          */
 680         smp_mb();
 681
 682         /*
 683          * The BP holds the hotplug lock, but we're now running on the AP,
 684          * ensure that anybody asserting the lock is held, will actually find
 685          * it so.
 686          */
 687         lockdep_acquire_cpus_lock();
 688         cpuhp_lock_acquire(bringup);
 689
 690         if (st->single) {
 691                 state = st->cb_state;
 692                 st->should_run = false;
 693         } else {
 694                 if (bringup) {
 695                         st->state++;
 696                         state = st->state;
 697                         st->should_run = (st->state < st->target);
 698                         WARN_ON_ONCE(st->state > st->target);
 699                 } else {
 700                         state = st->state;
 701                         st->state--;
 702                         st->should_run = (st->state > st->target);
 703                         WARN_ON_ONCE(st->state < st->target);
 704                 }
 705         }
 706
 707         WARN_ON_ONCE(!cpuhp_is_ap_state(state));
 708
 709         if (cpuhp_is_atomic_state(state)) {
 710                 local_irq_disable();
 711                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 712                 local_irq_enable();
 713
 714                 /*
 715                  * STARTING/DYING must not fail!
 716                  */
 717                 WARN_ON_ONCE(st->result);
 718         } else {
 719                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 720         }
 721
 722         if (st->result) {
 723                 /*
 724                  * If we fail on a rollback, we're up a creek without no
 725                  * paddle, no way forward, no way back. We loose, thanks for
 726                  * playing.
 727                  */
 728                 WARN_ON_ONCE(st->rollback);
 729                 st->should_run = false;
 730         }
 731
 732         cpuhp_lock_release(bringup);
 733         lockdep_release_cpus_lock();
 734
 735         if (!st->should_run)
 736                 complete_ap_thread(st, bringup);
 737 }
 738
 739 /* Invoke a single callback on a remote cpu */
 740 static int
 741 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
 742                          struct hlist_node *node)
 743 {
 744         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 745         int ret;
 746
 747         if (!cpu_online(cpu))
 748                 return 0;
 749
 750         cpuhp_lock_acquire(false);
 751         cpuhp_lock_release(false);
 752
 753         cpuhp_lock_acquire(true);
 754         cpuhp_lock_release(true);
 755
 756         /*
 757          * If we are up and running, use the hotplug thread. For early calls
 758          * we invoke the thread function directly.
 759          */
 760         if (!st->thread)
 761                 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
 762
 763         st->rollback = false;
 764         st->last = NULL;
 765
 766         st->node = node;
 767         st->bringup = bringup;
 768         st->cb_state = state;
 769         st->single = true;
 770
 771         __cpuhp_kick_ap(st);
 772
 773         /*
 774          * If we failed and did a partial, do a rollback.
 775          */
 776         if ((ret = st->result) && st->last) {
 777                 st->rollback = true;
 778                 st->bringup = !bringup;
 779
 780                 __cpuhp_kick_ap(st);
 781         }
 782
 783         /*
 784          * Clean up the leftovers so the next hotplug operation wont use stale
 785          * data.
 786          */
 787         st->node = st->last = NULL;
 788         return ret;
 789 }
 790
 791 static int cpuhp_kick_ap_work(unsigned int cpu)
 792 {
 793         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 794         enum cpuhp_state prev_state = st->state;
 795         int ret;
 796
 797         cpuhp_lock_acquire(false);
 798         cpuhp_lock_release(false);
 799
 800         cpuhp_lock_acquire(true);
 801         cpuhp_lock_release(true);
 802
 803         trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
 804         ret = cpuhp_kick_ap(st, st->target);
 805         trace_cpuhp_exit(cpu, st->state, prev_state, ret);
 806
 807         return ret;
 808 }
 809
 810 static struct smp_hotplug_thread cpuhp_threads = {
 811         .store                  = &cpuhp_state.thread,
 812         .create                 = &cpuhp_create,
 813         .thread_should_run      = cpuhp_should_run,
 814         .thread_fn              = cpuhp_thread_fun,
 815         .thread_comm            = "cpuhp/%u",
 816         .selfparking            = true,
 817 };
 818
 819 void __init cpuhp_threads_init(void)
 820 {
 821         BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
 822         kthread_unpark(this_cpu_read(cpuhp_state.thread));
 823 }
 824
 825 /*
 826  *
 827  * Serialize hotplug trainwrecks outside of the cpu_hotplug_lock
 828  * protected region.
 829  *
 830  * The operation is still serialized against concurrent CPU hotplug via
 831  * cpu_add_remove_lock, i.e. CPU map protection.  But it is _not_
 832  * serialized against other hotplug related activity like adding or
 833  * removing of state callbacks and state instances, which invoke either the
 834  * startup or the teardown callback of the affected state.
 835  *
 836  * This is required for subsystems which are unfixable vs. CPU hotplug and
 837  * evade lock inversion problems by scheduling work which has to be
 838  * completed _before_ cpu_up()/_cpu_down() returns.
 839  *
 840  * Don't even think about adding anything to this for any new code or even
 841  * drivers. It's only purpose is to keep existing lock order trainwrecks
 842  * working.
 843  *
 844  * For cpu_down() there might be valid reasons to finish cleanups which are
 845  * not required to be done under cpu_hotplug_lock, but that's a different
 846  * story and would be not invoked via this.
 847  */
 848 static void cpu_up_down_serialize_trainwrecks(bool tasks_frozen)
 849 {
 850         /*
 851          * cpusets delegate hotplug operations to a worker to "solve" the
 852          * lock order problems. Wait for the worker, but only if tasks are
 853          * _not_ frozen (suspend, hibernate) as that would wait forever.
 854          *
 855          * The wait is required because otherwise the hotplug operation
 856          * returns with inconsistent state, which could even be observed in
 857          * user space when a new CPU is brought up. The CPU plug uevent
 858          * would be delivered and user space reacting on it would fail to
 859          * move tasks to the newly plugged CPU up to the point where the
 860          * work has finished because up to that point the newly plugged CPU
 861          * is not assignable in cpusets/cgroups. On unplug that's not
 862          * necessarily a visible issue, but it is still inconsistent state,
 863          * which is the real problem which needs to be "fixed". This can't
 864          * prevent the transient state between scheduling the work and
 865          * returning from waiting for it.
 866          */
 867         if (!tasks_frozen)
 868                 cpuset_wait_for_hotplug();
 869 }
 870
 871 #ifdef CONFIG_HOTPLUG_CPU
 872 #ifndef arch_clear_mm_cpumask_cpu
 873 #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
 874 #endif
 875
 876 /**
 877  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
 878  * @cpu: a CPU id
 879  *
 880  * This function walks all processes, finds a valid mm struct for each one and
 881  * then clears a corresponding bit in mm's cpumask.  While this all sounds
 882  * trivial, there are various non-obvious corner cases, which this function
 883  * tries to solve in a safe manner.
 884  *
 885  * Also note that the function uses a somewhat relaxed locking scheme, so it may
 886  * be called only for an already offlined CPU.
 887  */
 888 void clear_tasks_mm_cpumask(int cpu)
 889 {
 890         struct task_struct *p;
 891
 892         /*
 893          * This function is called after the cpu is taken down and marked
 894          * offline, so its not like new tasks will ever get this cpu set in
 895          * their mm mask. -- Peter Zijlstra
 896          * Thus, we may use rcu_read_lock() here, instead of grabbing
 897          * full-fledged tasklist_lock.
 898          */
 899         WARN_ON(cpu_online(cpu));
 900         rcu_read_lock();
 901         for_each_process(p) {
 902                 struct task_struct *t;
 903
 904                 /*
 905                  * Main thread might exit, but other threads may still have
 906                  * a valid mm. Find one.
 907                  */
 908                 t = find_lock_task_mm(p);
 909                 if (!t)
 910                         continue;
 911                 arch_clear_mm_cpumask_cpu(cpu, t->mm);
 912                 task_unlock(t);
 913         }
 914         rcu_read_unlock();
 915 }
 916
 917 /* Take this CPU down. */
 918 static int take_cpu_down(void *_param)
 919 {
 920         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 921         enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
 922         int err, cpu = smp_processor_id();
 923         int ret;
 924
 925         /* Ensure this CPU doesn't handle any more interrupts. */
 926         err = __cpu_disable();
 927         if (err < 0)
 928                 return err;
 929
 930         /*
 931          * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
 932          * do this step again.
 933          */
 934         WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
 935         st->state--;
 936         /* Invoke the former CPU_DYING callbacks */
 937         for (; st->state > target; st->state--) {
 938                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 939                 /*
 940                  * DYING must not fail!
 941                  */
 942                 WARN_ON_ONCE(ret);
 943         }
 944
 945         /* Give up timekeeping duties */
 946         tick_handover_do_timer();
 947         /* Remove CPU from timer broadcasting */
 948         tick_offline_cpu(cpu);
 949         /* Park the stopper thread */
 950         stop_machine_park(cpu);
 951         return 0;
 952 }
 953
 954 static int takedown_cpu(unsigned int cpu)
 955 {
 956         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 957         int err;
 958
 959         /* Park the smpboot threads */
 960         kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 961
 962         /*
 963          * Prevent irq alloc/free while the dying cpu reorganizes the
 964          * interrupt affinities.
 965          */
 966         irq_lock_sparse();
 967
 968         /*
 969          * So now all preempt/rcu users must observe !cpu_active().
 970          */
 971         err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
 972         if (err) {
 973                 /* CPU refused to die */
 974                 irq_unlock_sparse();
 975                 /* Unpark the hotplug thread so we can rollback there */
 976                 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 977                 return err;
 978         }
 979         BUG_ON(cpu_online(cpu));
 980
 981         /*
 982          * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
 983          * all runnable tasks from the CPU, there's only the idle task left now
 984          * that the migration thread is done doing the stop_machine thing.
 985          *
 986          * Wait for the stop thread to go away.
 987          */
 988         wait_for_ap_thread(st, false);
 989         BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
 990
 991         /* Interrupts are moved away from the dying cpu, reenable alloc/free */
 992         irq_unlock_sparse();
 993
 994         hotplug_cpu__broadcast_tick_pull(cpu);
 995         /* This actually kills the CPU. */
 996         __cpu_die(cpu);
 997
 998         tick_cleanup_dead_cpu(cpu);
 999         rcutree_migrate_callbacks(cpu);
1000         return 0;
1001 }
1002
1003 static void cpuhp_complete_idle_dead(void *arg)
1004 {
1005         struct cpuhp_cpu_state *st = arg;
1006
1007         complete_ap_thread(st, false);
1008 }
1009
1010 void cpuhp_report_idle_dead(void)
1011 {
1012         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1013
1014         BUG_ON(st->state != CPUHP_AP_OFFLINE);
1015         rcu_report_dead(smp_processor_id());
1016         st->state = CPUHP_AP_IDLE_DEAD;
1017         /*
1018          * We cannot call complete after rcu_report_dead() so we delegate it
1019          * to an online cpu.
1020          */
1021         smp_call_function_single(cpumask_first(cpu_online_mask),
1022                                  cpuhp_complete_idle_dead, st, 0);
1023 }
1024
1025 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
1026 {
1027         for (st->state++; st->state < st->target; st->state++)
1028                 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1029 }
1030
1031 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
1032                                 enum cpuhp_state target)
1033 {
1034         enum cpuhp_state prev_state = st->state;
1035         int ret = 0;
1036
1037         for (; st->state > target; st->state--) {
1038                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
1039                 if (ret) {
1040                         st->target = prev_state;
1041                         if (st->state < prev_state)
1042                                 undo_cpu_down(cpu, st);
1043                         break;
1044                 }
1045         }
1046         return ret;
1047 }
1048
1049 /* Requires cpu_add_remove_lock to be held */
1050 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
1051                            enum cpuhp_state target)
1052 {
1053         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1054         int prev_state, ret = 0;
1055
1056         if (num_online_cpus() == 1)
1057                 return -EBUSY;
1058
1059         if (!cpu_present(cpu))
1060                 return -EINVAL;
1061
1062         cpus_write_lock();
1063
1064         cpuhp_tasks_frozen = tasks_frozen;
1065
1066         prev_state = cpuhp_set_state(st, target);
1067         /*
1068          * If the current CPU state is in the range of the AP hotplug thread,
1069          * then we need to kick the thread.
1070          */
1071         if (st->state > CPUHP_TEARDOWN_CPU) {
1072                 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
1073                 ret = cpuhp_kick_ap_work(cpu);
1074                 /*
1075                  * The AP side has done the error rollback already. Just
1076                  * return the error code..
1077                  */
1078                 if (ret)
1079                         goto out;
1080
1081                 /*
1082                  * We might have stopped still in the range of the AP hotplug
1083                  * thread. Nothing to do anymore.
1084                  */
1085                 if (st->state > CPUHP_TEARDOWN_CPU)
1086                         goto out;
1087
1088                 st->target = target;
1089         }
1090         /*
1091          * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1092          * to do the further cleanups.
1093          */
1094         ret = cpuhp_down_callbacks(cpu, st, target);
1095         if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
1096                 cpuhp_reset_state(st, prev_state);
1097                 __cpuhp_kick_ap(st);
1098         }
1099
1100 out:
1101         cpus_write_unlock();
1102         /*
1103          * Do post unplug cleanup. This is still protected against
1104          * concurrent CPU hotplug via cpu_add_remove_lock.
1105          */
1106         lockup_detector_cleanup();
1107         arch_smt_update();
1108         cpu_up_down_serialize_trainwrecks(tasks_frozen);
1109         return ret;
1110 }
1111
1112 static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1113 {
1114         if (cpu_hotplug_disabled)
1115                 return -EBUSY;
1116         return _cpu_down(cpu, 0, target);
1117 }
1118
1119 static int cpu_down(unsigned int cpu, enum cpuhp_state target)
1120 {
1121         int err;
1122
1123         cpu_maps_update_begin();
1124         err = cpu_down_maps_locked(cpu, target);
1125         cpu_maps_update_done();
1126         return err;
1127 }
1128
1129 /**
1130  * cpu_device_down - Bring down a cpu device
1131  * @dev: Pointer to the cpu device to offline
1132  *
1133  * This function is meant to be used by device core cpu subsystem only.
1134  *
1135  * Other subsystems should use remove_cpu() instead.
1136  */
1137 int cpu_device_down(struct device *dev)
1138 {
1139         return cpu_down(dev->id, CPUHP_OFFLINE);
1140 }
1141
1142 int remove_cpu(unsigned int cpu)
1143 {
1144         int ret;
1145
1146         lock_device_hotplug();
1147         ret = device_offline(get_cpu_device(cpu));
1148         unlock_device_hotplug();
1149
1150         return ret;
1151 }
1152 EXPORT_SYMBOL_GPL(remove_cpu);
1153
1154 void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
1155 {
1156         unsigned int cpu;
1157         int error;
1158
1159         cpu_maps_update_begin();
1160
1161         /*
1162          * Make certain the cpu I'm about to reboot on is online.
1163          *
1164          * This is inline to what migrate_to_reboot_cpu() already do.
1165          */
1166         if (!cpu_online(primary_cpu))
1167                 primary_cpu = cpumask_first(cpu_online_mask);
1168
1169         for_each_online_cpu(cpu) {
1170                 if (cpu == primary_cpu)
1171                         continue;
1172
1173                 error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
1174                 if (error) {
1175                         pr_err("Failed to offline CPU%d - error=%d",
1176                                 cpu, error);
1177                         break;
1178                 }
1179         }
1180
1181         /*
1182          * Ensure all but the reboot CPU are offline.
1183          */
1184         BUG_ON(num_online_cpus() > 1);
1185
1186         /*
1187          * Make sure the CPUs won't be enabled by someone else after this
1188          * point. Kexec will reboot to a new kernel shortly resetting
1189          * everything along the way.
1190          */
1191         cpu_hotplug_disabled++;
1192
1193         cpu_maps_update_done();
1194 }
1195
1196 #else
1197 #define takedown_cpu            NULL
1198 #endif /*CONFIG_HOTPLUG_CPU*/
1199
1200 /**
1201  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1202  * @cpu: cpu that just started
1203  *
1204  * It must be called by the arch code on the new cpu, before the new cpu
1205  * enables interrupts and before the "boot" cpu returns from __cpu_up().
1206  */
1207 void notify_cpu_starting(unsigned int cpu)
1208 {
1209         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1210         enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1211         int ret;
1212
1213         rcu_cpu_starting(cpu);  /* Enables RCU usage on this CPU. */
1214         cpumask_set_cpu(cpu, &cpus_booted_once_mask);
1215         while (st->state < target) {
1216                 st->state++;
1217                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1218                 /*
1219                  * STARTING must not fail!
1220                  */
1221                 WARN_ON_ONCE(ret);
1222         }
1223 }
1224
1225 /*
1226  * Called from the idle task. Wake up the controlling task which brings the
1227  * hotplug thread of the upcoming CPU up and then delegates the rest of the
1228  * online bringup to the hotplug thread.
1229  */
1230 void cpuhp_online_idle(enum cpuhp_state state)
1231 {
1232         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1233
1234         /* Happens for the boot cpu */
1235         if (state != CPUHP_AP_ONLINE_IDLE)
1236                 return;
1237
1238         /*
1239          * Unpart the stopper thread before we start the idle loop (and start
1240          * scheduling); this ensures the stopper task is always available.
1241          */
1242         stop_machine_unpark(smp_processor_id());
1243
1244         st->state = CPUHP_AP_ONLINE_IDLE;
1245         complete_ap_thread(st, true);
1246 }
1247
1248 /* Requires cpu_add_remove_lock to be held */
1249 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1250 {
1251         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1252         struct task_struct *idle;
1253         int ret = 0;
1254
1255         cpus_write_lock();
1256
1257         if (!cpu_present(cpu)) {
1258                 ret = -EINVAL;
1259                 goto out;
1260         }
1261
1262         /*
1263          * The caller of cpu_up() might have raced with another
1264          * caller. Nothing to do.
1265          */
1266         if (st->state >= target)
1267                 goto out;
1268
1269         if (st->state == CPUHP_OFFLINE) {
1270                 /* Let it fail before we try to bring the cpu up */
1271                 idle = idle_thread_get(cpu);
1272                 if (IS_ERR(idle)) {
1273                         ret = PTR_ERR(idle);
1274                         goto out;
1275                 }
1276         }
1277
1278         cpuhp_tasks_frozen = tasks_frozen;
1279
1280         cpuhp_set_state(st, target);
1281         /*
1282          * If the current CPU state is in the range of the AP hotplug thread,
1283          * then we need to kick the thread once more.
1284          */
1285         if (st->state > CPUHP_BRINGUP_CPU) {
1286                 ret = cpuhp_kick_ap_work(cpu);
1287                 /*
1288                  * The AP side has done the error rollback already. Just
1289                  * return the error code..
1290                  */
1291                 if (ret)
1292                         goto out;
1293         }
1294
1295         /*
1296          * Try to reach the target state. We max out on the BP at
1297          * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1298          * responsible for bringing it up to the target state.
1299          */
1300         target = min((int)target, CPUHP_BRINGUP_CPU);
1301         ret = cpuhp_up_callbacks(cpu, st, target);
1302 out:
1303         cpus_write_unlock();
1304         arch_smt_update();
1305         cpu_up_down_serialize_trainwrecks(tasks_frozen);
1306         return ret;
1307 }
1308
1309 static int cpu_up(unsigned int cpu, enum cpuhp_state target)
1310 {
1311         int err = 0;
1312
1313         if (!cpu_possible(cpu)) {
1314                 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1315                        cpu);
1316 #if defined(CONFIG_IA64)
1317                 pr_err("please check additional_cpus= boot parameter\n");
1318 #endif
1319                 return -EINVAL;
1320         }
1321
1322         err = try_online_node(cpu_to_node(cpu));
1323         if (err)
1324                 return err;
1325
1326         cpu_maps_update_begin();
1327
1328         if (cpu_hotplug_disabled) {
1329                 err = -EBUSY;
1330                 goto out;
1331         }
1332         if (!cpu_smt_allowed(cpu)) {
1333                 err = -EPERM;
1334                 goto out;
1335         }
1336
1337         err = _cpu_up(cpu, 0, target);
1338 out:
1339         cpu_maps_update_done();
1340         return err;
1341 }
1342
1343 /**
1344  * cpu_device_up - Bring up a cpu device
1345  * @dev: Pointer to the cpu device to online
1346  *
1347  * This function is meant to be used by device core cpu subsystem only.
1348  *
1349  * Other subsystems should use add_cpu() instead.
1350  */
1351 int cpu_device_up(struct device *dev)
1352 {
1353         return cpu_up(dev->id, CPUHP_ONLINE);
1354 }
1355
1356 int add_cpu(unsigned int cpu)
1357 {
1358         int ret;
1359
1360         lock_device_hotplug();
1361         ret = device_online(get_cpu_device(cpu));
1362         unlock_device_hotplug();
1363
1364         return ret;
1365 }
1366 EXPORT_SYMBOL_GPL(add_cpu);
1367
1368 /**
1369  * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1370  * @sleep_cpu: The cpu we hibernated on and should be brought up.
1371  *
1372  * On some architectures like arm64, we can hibernate on any CPU, but on
1373  * wake up the CPU we hibernated on might be offline as a side effect of
1374  * using maxcpus= for example.
1375  */
1376 int bringup_hibernate_cpu(unsigned int sleep_cpu)
1377 {
1378         int ret;
1379
1380         if (!cpu_online(sleep_cpu)) {
1381                 pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
1382                 ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
1383                 if (ret) {
1384                         pr_err("Failed to bring hibernate-CPU up!\n");
1385                         return ret;
1386                 }
1387         }
1388         return 0;
1389 }
1390
1391 void bringup_nonboot_cpus(unsigned int setup_max_cpus)
1392 {
1393         unsigned int cpu;
1394
1395         for_each_present_cpu(cpu) {
1396                 if (num_online_cpus() >= setup_max_cpus)
1397                         break;
1398                 if (!cpu_online(cpu))
1399                         cpu_up(cpu, CPUHP_ONLINE);
1400         }
1401 }
1402
1403 #ifdef CONFIG_PM_SLEEP_SMP
1404 static cpumask_var_t frozen_cpus;
1405
1406 int freeze_secondary_cpus(int primary)
1407 {
1408         int cpu, error = 0;
1409
1410         cpu_maps_update_begin();
1411         if (primary == -1) {
1412                 primary = cpumask_first(cpu_online_mask);
1413                 if (!housekeeping_cpu(primary, HK_FLAG_TIMER))
1414                         primary = housekeeping_any_cpu(HK_FLAG_TIMER);
1415         } else {
1416                 if (!cpu_online(primary))
1417                         primary = cpumask_first(cpu_online_mask);
1418         }
1419
1420         /*
1421          * We take down all of the non-boot CPUs in one shot to avoid races
1422          * with the userspace trying to use the CPU hotplug at the same time
1423          */
1424         cpumask_clear(frozen_cpus);
1425
1426         pr_info("Disabling non-boot CPUs ...\n");
1427         for_each_online_cpu(cpu) {
1428                 if (cpu == primary)
1429                         continue;
1430
1431                 if (pm_wakeup_pending()) {
1432                         pr_info("Wakeup pending. Abort CPU freeze\n");
1433                         error = -EBUSY;
1434                         break;
1435                 }
1436
1437                 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1438                 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1439                 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1440                 if (!error)
1441                         cpumask_set_cpu(cpu, frozen_cpus);
1442                 else {
1443                         pr_err("Error taking CPU%d down: %d\n", cpu, error);
1444                         break;
1445                 }
1446         }
1447
1448         if (!error)
1449                 BUG_ON(num_online_cpus() > 1);
1450         else
1451                 pr_err("Non-boot CPUs are not disabled\n");
1452
1453         /*
1454          * Make sure the CPUs won't be enabled by someone else. We need to do
1455          * this even in case of failure as all freeze_secondary_cpus() users are
1456          * supposed to do thaw_secondary_cpus() on the failure path.
1457          */
1458         cpu_hotplug_disabled++;
1459
1460         cpu_maps_update_done();
1461         return error;
1462 }
1463
1464 void __weak arch_thaw_secondary_cpus_begin(void)
1465 {
1466 }
1467
1468 void __weak arch_thaw_secondary_cpus_end(void)
1469 {
1470 }
1471
1472 void thaw_secondary_cpus(void)
1473 {
1474         int cpu, error;
1475
1476         /* Allow everyone to use the CPU hotplug again */
1477         cpu_maps_update_begin();
1478         __cpu_hotplug_enable();
1479         if (cpumask_empty(frozen_cpus))
1480                 goto out;
1481
1482         pr_info("Enabling non-boot CPUs ...\n");
1483
1484         arch_thaw_secondary_cpus_begin();
1485
1486         for_each_cpu(cpu, frozen_cpus) {
1487                 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1488                 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1489                 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1490                 if (!error) {
1491                         pr_info("CPU%d is up\n", cpu);
1492                         continue;
1493                 }
1494                 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1495         }
1496
1497         arch_thaw_secondary_cpus_end();
1498
1499         cpumask_clear(frozen_cpus);
1500 out:
1501         cpu_maps_update_done();
1502 }
1503
1504 static int __init alloc_frozen_cpus(void)
1505 {
1506         if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1507                 return -ENOMEM;
1508         return 0;
1509 }
1510 core_initcall(alloc_frozen_cpus);
1511
1512 /*
1513  * When callbacks for CPU hotplug notifications are being executed, we must
1514  * ensure that the state of the system with respect to the tasks being frozen
1515  * or not, as reported by the notification, remains unchanged *throughout the
1516  * duration* of the execution of the callbacks.
1517  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1518  *
1519  * This synchronization is implemented by mutually excluding regular CPU
1520  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1521  * Hibernate notifications.
1522  */
1523 static int
1524 cpu_hotplug_pm_callback(struct notifier_block *nb,
1525                         unsigned long action, void *ptr)
1526 {
1527         switch (action) {
1528
1529         case PM_SUSPEND_PREPARE:
1530         case PM_HIBERNATION_PREPARE:
1531                 cpu_hotplug_disable();
1532                 break;
1533
1534         case PM_POST_SUSPEND:
1535         case PM_POST_HIBERNATION:
1536                 cpu_hotplug_enable();
1537                 break;
1538
1539         default:
1540                 return NOTIFY_DONE;
1541         }
1542
1543         return NOTIFY_OK;
1544 }
1545
1546
1547 static int __init cpu_hotplug_pm_sync_init(void)
1548 {
1549         /*
1550          * cpu_hotplug_pm_callback has higher priority than x86
1551          * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1552          * to disable cpu hotplug to avoid cpu hotplug race.
1553          */
1554         pm_notifier(cpu_hotplug_pm_callback, 0);
1555         return 0;
1556 }
1557 core_initcall(cpu_hotplug_pm_sync_init);
1558
1559 #endif /* CONFIG_PM_SLEEP_SMP */
1560
1561 int __boot_cpu_id;
1562
1563 #endif /* CONFIG_SMP */
1564
1565 /* Boot processor state steps */
1566 static struct cpuhp_step cpuhp_hp_states[] = {
1567         [CPUHP_OFFLINE] = {
1568                 .name                   = "offline",
1569                 .startup.single         = NULL,
1570                 .teardown.single        = NULL,
1571         },
1572 #ifdef CONFIG_SMP
1573         [CPUHP_CREATE_THREADS]= {
1574                 .name                   = "threads:prepare",
1575                 .startup.single         = smpboot_create_threads,
1576                 .teardown.single        = NULL,
1577                 .cant_stop              = true,
1578         },
1579         [CPUHP_PERF_PREPARE] = {
1580                 .name                   = "perf:prepare",
1581                 .startup.single         = perf_event_init_cpu,
1582                 .teardown.single        = perf_event_exit_cpu,
1583         },
1584         [CPUHP_WORKQUEUE_PREP] = {
1585                 .name                   = "workqueue:prepare",
1586                 .startup.single         = workqueue_prepare_cpu,
1587                 .teardown.single        = NULL,
1588         },
1589         [CPUHP_HRTIMERS_PREPARE] = {
1590                 .name                   = "hrtimers:prepare",
1591                 .startup.single         = hrtimers_prepare_cpu,
1592                 .teardown.single        = hrtimers_dead_cpu,
1593         },
1594         [CPUHP_SMPCFD_PREPARE] = {
1595                 .name                   = "smpcfd:prepare",
1596                 .startup.single         = smpcfd_prepare_cpu,
1597                 .teardown.single        = smpcfd_dead_cpu,
1598         },
1599         [CPUHP_RELAY_PREPARE] = {
1600                 .name                   = "relay:prepare",
1601                 .startup.single         = relay_prepare_cpu,
1602                 .teardown.single        = NULL,
1603         },
1604         [CPUHP_SLAB_PREPARE] = {
1605                 .name                   = "slab:prepare",
1606                 .startup.single         = slab_prepare_cpu,
1607                 .teardown.single        = slab_dead_cpu,
1608         },
1609         [CPUHP_RCUTREE_PREP] = {
1610                 .name                   = "RCU/tree:prepare",
1611                 .startup.single         = rcutree_prepare_cpu,
1612                 .teardown.single        = rcutree_dead_cpu,
1613         },
1614         /*
1615          * On the tear-down path, timers_dead_cpu() must be invoked
1616          * before blk_mq_queue_reinit_notify() from notify_dead(),
1617          * otherwise a RCU stall occurs.
1618          */
1619         [CPUHP_TIMERS_PREPARE] = {
1620                 .name                   = "timers:prepare",
1621                 .startup.single         = timers_prepare_cpu,
1622                 .teardown.single        = timers_dead_cpu,
1623         },
1624         /* Kicks the plugged cpu into life */
1625         [CPUHP_BRINGUP_CPU] = {
1626                 .name                   = "cpu:bringup",
1627                 .startup.single         = bringup_cpu,
1628                 .teardown.single        = finish_cpu,
1629                 .cant_stop              = true,
1630         },
1631         /* Final state before CPU kills itself */
1632         [CPUHP_AP_IDLE_DEAD] = {
1633                 .name                   = "idle:dead",
1634         },
1635         /*
1636          * Last state before CPU enters the idle loop to die. Transient state
1637          * for synchronization.
1638          */
1639         [CPUHP_AP_OFFLINE] = {
1640                 .name                   = "ap:offline",
1641                 .cant_stop              = true,
1642         },
1643         /* First state is scheduler control. Interrupts are disabled */
1644         [CPUHP_AP_SCHED_STARTING] = {
1645                 .name                   = "sched:starting",
1646                 .startup.single         = sched_cpu_starting,
1647                 .teardown.single        = sched_cpu_dying,
1648         },
1649         [CPUHP_AP_RCUTREE_DYING] = {
1650                 .name                   = "RCU/tree:dying",
1651                 .startup.single         = NULL,
1652                 .teardown.single        = rcutree_dying_cpu,
1653         },
1654         [CPUHP_AP_SMPCFD_DYING] = {
1655                 .name                   = "smpcfd:dying",
1656                 .startup.single         = NULL,
1657                 .teardown.single        = smpcfd_dying_cpu,
1658         },
1659         /* Entry state on starting. Interrupts enabled from here on. Transient
1660          * state for synchronsization */
1661         [CPUHP_AP_ONLINE] = {
1662                 .name                   = "ap:online",
1663         },
1664         /*
1665          * Handled on controll processor until the plugged processor manages
1666          * this itself.
1667          */
1668         [CPUHP_TEARDOWN_CPU] = {
1669                 .name                   = "cpu:teardown",
1670                 .startup.single         = NULL,
1671                 .teardown.single        = takedown_cpu,
1672                 .cant_stop              = true,
1673         },
1674         /* Handle smpboot threads park/unpark */
1675         [CPUHP_AP_SMPBOOT_THREADS] = {
1676                 .name                   = "smpboot/threads:online",
1677                 .startup.single         = smpboot_unpark_threads,
1678                 .teardown.single        = smpboot_park_threads,
1679         },
1680         [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1681                 .name                   = "irq/affinity:online",
1682                 .startup.single         = irq_affinity_online_cpu,
1683                 .teardown.single        = NULL,
1684         },
1685         [CPUHP_AP_PERF_ONLINE] = {
1686                 .name                   = "perf:online",
1687                 .startup.single         = perf_event_init_cpu,
1688                 .teardown.single        = perf_event_exit_cpu,
1689         },
1690         [CPUHP_AP_WATCHDOG_ONLINE] = {
1691                 .name                   = "lockup_detector:online",
1692                 .startup.single         = lockup_detector_online_cpu,
1693                 .teardown.single        = lockup_detector_offline_cpu,
1694         },
1695         [CPUHP_AP_WORKQUEUE_ONLINE] = {
1696                 .name                   = "workqueue:online",
1697                 .startup.single         = workqueue_online_cpu,
1698                 .teardown.single        = workqueue_offline_cpu,
1699         },
1700         [CPUHP_AP_RCUTREE_ONLINE] = {
1701                 .name                   = "RCU/tree:online",
1702                 .startup.single         = rcutree_online_cpu,
1703                 .teardown.single        = rcutree_offline_cpu,
1704         },
1705 #endif
1706         /*
1707          * The dynamically registered state space is here
1708          */
1709
1710 #ifdef CONFIG_SMP
1711         /* Last state is scheduler control setting the cpu active */
1712         [CPUHP_AP_ACTIVE] = {
1713                 .name                   = "sched:active",
1714                 .startup.single         = sched_cpu_activate,
1715                 .teardown.single        = sched_cpu_deactivate,
1716         },
1717 #endif
1718
1719         /* CPU is fully up and running. */
1720         [CPUHP_ONLINE] = {
1721                 .name                   = "online",
1722                 .startup.single         = NULL,
1723                 .teardown.single        = NULL,
1724         },
1725 };
1726
1727 /* Sanity check for callbacks */
1728 static int cpuhp_cb_check(enum cpuhp_state state)
1729 {
1730         if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1731                 return -EINVAL;
1732         return 0;
1733 }
1734
1735 /*
1736  * Returns a free for dynamic slot assignment of the Online state. The states
1737  * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1738  * by having no name assigned.
1739  */
1740 static int cpuhp_reserve_state(enum cpuhp_state state)
1741 {
1742         enum cpuhp_state i, end;
1743         struct cpuhp_step *step;
1744
1745         switch (state) {
1746         case CPUHP_AP_ONLINE_DYN:
1747                 step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1748                 end = CPUHP_AP_ONLINE_DYN_END;
1749                 break;
1750         case CPUHP_BP_PREPARE_DYN:
1751                 step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1752                 end = CPUHP_BP_PREPARE_DYN_END;
1753                 break;
1754         default:
1755                 return -EINVAL;
1756         }
1757
1758         for (i = state; i <= end; i++, step++) {
1759                 if (!step->name)
1760                         return i;
1761         }
1762         WARN(1, "No more dynamic states available for CPU hotplug\n");
1763         return -ENOSPC;
1764 }
1765
1766 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1767                                  int (*startup)(unsigned int cpu),
1768                                  int (*teardown)(unsigned int cpu),
1769                                  bool multi_instance)
1770 {
1771         /* (Un)Install the callbacks for further cpu hotplug operations */
1772         struct cpuhp_step *sp;
1773         int ret = 0;
1774
1775         /*
1776          * If name is NULL, then the state gets removed.
1777          *
1778          * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1779          * the first allocation from these dynamic ranges, so the removal
1780          * would trigger a new allocation and clear the wrong (already
1781          * empty) state, leaving the callbacks of the to be cleared state
1782          * dangling, which causes wreckage on the next hotplug operation.
1783          */
1784         if (name && (state == CPUHP_AP_ONLINE_DYN ||
1785                      state == CPUHP_BP_PREPARE_DYN)) {
1786                 ret = cpuhp_reserve_state(state);
1787                 if (ret < 0)
1788                         return ret;
1789                 state = ret;
1790         }
1791         sp = cpuhp_get_step(state);
1792         if (name && sp->name)
1793                 return -EBUSY;
1794
1795         sp->startup.single = startup;
1796         sp->teardown.single = teardown;
1797         sp->name = name;
1798         sp->multi_instance = multi_instance;
1799         INIT_HLIST_HEAD(&sp->list);
1800         return ret;
1801 }
1802
1803 static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1804 {
1805         return cpuhp_get_step(state)->teardown.single;
1806 }
1807
1808 /*
1809  * Call the startup/teardown function for a step either on the AP or
1810  * on the current CPU.
1811  */
1812 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1813                             struct hlist_node *node)
1814 {
1815         struct cpuhp_step *sp = cpuhp_get_step(state);
1816         int ret;
1817
1818         /*
1819          * If there's nothing to do, we done.
1820          * Relies on the union for multi_instance.
1821          */
1822         if ((bringup && !sp->startup.single) ||
1823             (!bringup && !sp->teardown.single))
1824                 return 0;
1825         /*
1826          * The non AP bound callbacks can fail on bringup. On teardown
1827          * e.g. module removal we crash for now.
1828          */
1829 #ifdef CONFIG_SMP
1830         if (cpuhp_is_ap_state(state))
1831                 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1832         else
1833                 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1834 #else
1835         ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1836 #endif
1837         BUG_ON(ret && !bringup);
1838         return ret;
1839 }
1840
1841 /*
1842  * Called from __cpuhp_setup_state on a recoverable failure.
1843  *
1844  * Note: The teardown callbacks for rollback are not allowed to fail!
1845  */
1846 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1847                                    struct hlist_node *node)
1848 {
1849         int cpu;
1850
1851         /* Roll back the already executed steps on the other cpus */
1852         for_each_present_cpu(cpu) {
1853                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1854                 int cpustate = st->state;
1855
1856                 if (cpu >= failedcpu)
1857                         break;
1858
1859                 /* Did we invoke the startup call on that cpu ? */
1860                 if (cpustate >= state)
1861                         cpuhp_issue_call(cpu, state, false, node);
1862         }
1863 }
1864
1865 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1866                                           struct hlist_node *node,
1867                                           bool invoke)
1868 {
1869         struct cpuhp_step *sp;
1870         int cpu;
1871         int ret;
1872
1873         lockdep_assert_cpus_held();
1874
1875         sp = cpuhp_get_step(state);
1876         if (sp->multi_instance == false)
1877                 return -EINVAL;
1878
1879         mutex_lock(&cpuhp_state_mutex);
1880
1881         if (!invoke || !sp->startup.multi)
1882                 goto add_node;
1883
1884         /*
1885          * Try to call the startup callback for each present cpu
1886          * depending on the hotplug state of the cpu.
1887          */
1888         for_each_present_cpu(cpu) {
1889                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1890                 int cpustate = st->state;
1891
1892                 if (cpustate < state)
1893                         continue;
1894
1895                 ret = cpuhp_issue_call(cpu, state, true, node);
1896                 if (ret) {
1897                         if (sp->teardown.multi)
1898                                 cpuhp_rollback_install(cpu, state, node);
1899                         goto unlock;
1900                 }
1901         }
1902 add_node:
1903         ret = 0;
1904         hlist_add_head(node, &sp->list);
1905 unlock:
1906         mutex_unlock(&cpuhp_state_mutex);
1907         return ret;
1908 }
1909
1910 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1911                                bool invoke)
1912 {
1913         int ret;
1914
1915         cpus_read_lock();
1916         ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1917         cpus_read_unlock();
1918         return ret;
1919 }
1920 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1921
1922 /**
1923  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1924  * @state:              The state to setup
1925  * @invoke:             If true, the startup function is invoked for cpus where
1926  *                      cpu state >= @state
1927  * @startup:            startup callback function
1928  * @teardown:           teardown callback function
1929  * @multi_instance:     State is set up for multiple instances which get
1930  *                      added afterwards.
1931  *
1932  * The caller needs to hold cpus read locked while calling this function.
1933  * Returns:
1934  *   On success:
1935  *      Positive state number if @state is CPUHP_AP_ONLINE_DYN
1936  *      0 for all other states
1937  *   On failure: proper (negative) error code
1938  */
1939 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1940                                    const char *name, bool invoke,
1941                                    int (*startup)(unsigned int cpu),
1942                                    int (*teardown)(unsigned int cpu),
1943                                    bool multi_instance)
1944 {
1945         int cpu, ret = 0;
1946         bool dynstate;
1947
1948         lockdep_assert_cpus_held();
1949
1950         if (cpuhp_cb_check(state) || !name)
1951                 return -EINVAL;
1952
1953         mutex_lock(&cpuhp_state_mutex);
1954
1955         ret = cpuhp_store_callbacks(state, name, startup, teardown,
1956                                     multi_instance);
1957
1958         dynstate = state == CPUHP_AP_ONLINE_DYN;
1959         if (ret > 0 && dynstate) {
1960                 state = ret;
1961                 ret = 0;
1962         }
1963
1964         if (ret || !invoke || !startup)
1965                 goto out;
1966
1967         /*
1968          * Try to call the startup callback for each present cpu
1969          * depending on the hotplug state of the cpu.
1970          */
1971         for_each_present_cpu(cpu) {
1972                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1973                 int cpustate = st->state;
1974
1975                 if (cpustate < state)
1976                         continue;
1977
1978                 ret = cpuhp_issue_call(cpu, state, true, NULL);
1979                 if (ret) {
1980                         if (teardown)
1981                                 cpuhp_rollback_install(cpu, state, NULL);
1982                         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1983                         goto out;
1984                 }
1985         }
1986 out:
1987         mutex_unlock(&cpuhp_state_mutex);
1988         /*
1989          * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1990          * dynamically allocated state in case of success.
1991          */
1992         if (!ret && dynstate)
1993                 return state;
1994         return ret;
1995 }
1996 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1997
1998 int __cpuhp_setup_state(enum cpuhp_state state,
1999                         const char *name, bool invoke,
2000                         int (*startup)(unsigned int cpu),
2001                         int (*teardown)(unsigned int cpu),
2002                         bool multi_instance)
2003 {
2004         int ret;
2005
2006         cpus_read_lock();
2007         ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
2008                                              teardown, multi_instance);
2009         cpus_read_unlock();
2010         return ret;
2011 }
2012 EXPORT_SYMBOL(__cpuhp_setup_state);
2013
2014 int __cpuhp_state_remove_instance(enum cpuhp_state state,
2015                                   struct hlist_node *node, bool invoke)
2016 {
2017         struct cpuhp_step *sp = cpuhp_get_step(state);
2018         int cpu;
2019
2020         BUG_ON(cpuhp_cb_check(state));
2021
2022         if (!sp->multi_instance)
2023                 return -EINVAL;
2024
2025         cpus_read_lock();
2026         mutex_lock(&cpuhp_state_mutex);
2027
2028         if (!invoke || !cpuhp_get_teardown_cb(state))
2029                 goto remove;
2030         /*
2031          * Call the teardown callback for each present cpu depending
2032          * on the hotplug state of the cpu. This function is not
2033          * allowed to fail currently!
2034          */
2035         for_each_present_cpu(cpu) {
2036                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2037                 int cpustate = st->state;
2038
2039                 if (cpustate >= state)
2040                         cpuhp_issue_call(cpu, state, false, node);
2041         }
2042
2043 remove:
2044         hlist_del(node);
2045         mutex_unlock(&cpuhp_state_mutex);
2046         cpus_read_unlock();
2047
2048         return 0;
2049 }
2050 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
2051
2052 /**
2053  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2054  * @state:      The state to remove
2055  * @invoke:     If true, the teardown function is invoked for cpus where
2056  *              cpu state >= @state
2057  *
2058  * The caller needs to hold cpus read locked while calling this function.
2059  * The teardown callback is currently not allowed to fail. Think
2060  * about module removal!
2061  */
2062 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
2063 {
2064         struct cpuhp_step *sp = cpuhp_get_step(state);
2065         int cpu;
2066
2067         BUG_ON(cpuhp_cb_check(state));
2068
2069         lockdep_assert_cpus_held();
2070
2071         mutex_lock(&cpuhp_state_mutex);
2072         if (sp->multi_instance) {
2073                 WARN(!hlist_empty(&sp->list),
2074                      "Error: Removing state %d which has instances left.\n",
2075                      state);
2076                 goto remove;
2077         }
2078
2079         if (!invoke || !cpuhp_get_teardown_cb(state))
2080                 goto remove;
2081
2082         /*
2083          * Call the teardown callback for each present cpu depending
2084          * on the hotplug state of the cpu. This function is not
2085          * allowed to fail currently!
2086          */
2087         for_each_present_cpu(cpu) {
2088                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2089                 int cpustate = st->state;
2090
2091                 if (cpustate >= state)
2092                         cpuhp_issue_call(cpu, state, false, NULL);
2093         }
2094 remove:
2095         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2096         mutex_unlock(&cpuhp_state_mutex);
2097 }
2098 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
2099
2100 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
2101 {
2102         cpus_read_lock();
2103         __cpuhp_remove_state_cpuslocked(state, invoke);
2104         cpus_read_unlock();
2105 }
2106 EXPORT_SYMBOL(__cpuhp_remove_state);
2107
2108 #ifdef CONFIG_HOTPLUG_SMT
2109 static void cpuhp_offline_cpu_device(unsigned int cpu)
2110 {
2111         struct device *dev = get_cpu_device(cpu);
2112
2113         dev->offline = true;
2114         /* Tell user space about the state change */
2115         kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2116 }
2117
2118 static void cpuhp_online_cpu_device(unsigned int cpu)
2119 {
2120         struct device *dev = get_cpu_device(cpu);
2121
2122         dev->offline = false;
2123         /* Tell user space about the state change */
2124         kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2125 }
2126
2127 int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2128 {
2129         int cpu, ret = 0;
2130
2131         cpu_maps_update_begin();
2132         for_each_online_cpu(cpu) {
2133                 if (topology_is_primary_thread(cpu))
2134                         continue;
2135                 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2136                 if (ret)
2137                         break;
2138                 /*
2139                  * As this needs to hold the cpu maps lock it's impossible
2140                  * to call device_offline() because that ends up calling
2141                  * cpu_down() which takes cpu maps lock. cpu maps lock
2142                  * needs to be held as this might race against in kernel
2143                  * abusers of the hotplug machinery (thermal management).
2144                  *
2145                  * So nothing would update device:offline state. That would
2146                  * leave the sysfs entry stale and prevent onlining after
2147                  * smt control has been changed to 'off' again. This is
2148                  * called under the sysfs hotplug lock, so it is properly
2149                  * serialized against the regular offline usage.
2150                  */
2151                 cpuhp_offline_cpu_device(cpu);
2152         }
2153         if (!ret)
2154                 cpu_smt_control = ctrlval;
2155         cpu_maps_update_done();
2156         return ret;
2157 }
2158
2159 int cpuhp_smt_enable(void)
2160 {
2161         int cpu, ret = 0;
2162
2163         cpu_maps_update_begin();
2164         cpu_smt_control = CPU_SMT_ENABLED;
2165         for_each_present_cpu(cpu) {
2166                 /* Skip online CPUs and CPUs on offline nodes */
2167                 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2168                         continue;
2169                 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2170                 if (ret)
2171                         break;
2172                 /* See comment in cpuhp_smt_disable() */
2173                 cpuhp_online_cpu_device(cpu);
2174         }
2175         cpu_maps_update_done();
2176         return ret;
2177 }
2178 #endif
2179
2180 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
2181 static ssize_t show_cpuhp_state(struct device *dev,
2182                                 struct device_attribute *attr, char *buf)
2183 {
2184         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2185
2186         return sprintf(buf, "%d\n", st->state);
2187 }
2188 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
2189
2190 static ssize_t write_cpuhp_target(struct device *dev,
2191                                   struct device_attribute *attr,
2192                                   const char *buf, size_t count)
2193 {
2194         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2195         struct cpuhp_step *sp;
2196         int target, ret;
2197
2198         ret = kstrtoint(buf, 10, &target);
2199         if (ret)
2200                 return ret;
2201
2202 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
2203         if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
2204                 return -EINVAL;
2205 #else
2206         if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
2207                 return -EINVAL;
2208 #endif
2209
2210         ret = lock_device_hotplug_sysfs();
2211         if (ret)
2212                 return ret;
2213
2214         mutex_lock(&cpuhp_state_mutex);
2215         sp = cpuhp_get_step(target);
2216         ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
2217         mutex_unlock(&cpuhp_state_mutex);
2218         if (ret)
2219                 goto out;
2220
2221         if (st->state < target)
2222                 ret = cpu_up(dev->id, target);
2223         else
2224                 ret = cpu_down(dev->id, target);
2225 out:
2226         unlock_device_hotplug();
2227         return ret ? ret : count;
2228 }
2229
2230 static ssize_t show_cpuhp_target(struct device *dev,
2231                                  struct device_attribute *attr, char *buf)
2232 {
2233         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2234
2235         return sprintf(buf, "%d\n", st->target);
2236 }
2237 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
2238
2239
2240 static ssize_t write_cpuhp_fail(struct device *dev,
2241                                 struct device_attribute *attr,
2242                                 const char *buf, size_t count)
2243 {
2244         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2245         struct cpuhp_step *sp;
2246         int fail, ret;
2247
2248         ret = kstrtoint(buf, 10, &fail);
2249         if (ret)
2250                 return ret;
2251
2252         if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
2253                 return -EINVAL;
2254
2255         /*
2256          * Cannot fail STARTING/DYING callbacks.
2257          */
2258         if (cpuhp_is_atomic_state(fail))
2259                 return -EINVAL;
2260
2261         /*
2262          * Cannot fail anything that doesn't have callbacks.
2263          */
2264         mutex_lock(&cpuhp_state_mutex);
2265         sp = cpuhp_get_step(fail);
2266         if (!sp->startup.single && !sp->teardown.single)
2267                 ret = -EINVAL;
2268         mutex_unlock(&cpuhp_state_mutex);
2269         if (ret)
2270                 return ret;
2271
2272         st->fail = fail;
2273
2274         return count;
2275 }
2276
2277 static ssize_t show_cpuhp_fail(struct device *dev,
2278                                struct device_attribute *attr, char *buf)
2279 {
2280         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2281
2282         return sprintf(buf, "%d\n", st->fail);
2283 }
2284
2285 static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
2286
2287 static struct attribute *cpuhp_cpu_attrs[] = {
2288         &dev_attr_state.attr,
2289         &dev_attr_target.attr,
2290         &dev_attr_fail.attr,
2291         NULL
2292 };
2293
2294 static const struct attribute_group cpuhp_cpu_attr_group = {
2295         .attrs = cpuhp_cpu_attrs,
2296         .name = "hotplug",
2297         NULL
2298 };
2299
2300 static ssize_t show_cpuhp_states(struct device *dev,
2301                                  struct device_attribute *attr, char *buf)
2302 {
2303         ssize_t cur, res = 0;
2304         int i;
2305
2306         mutex_lock(&cpuhp_state_mutex);
2307         for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2308                 struct cpuhp_step *sp = cpuhp_get_step(i);
2309
2310                 if (sp->name) {
2311                         cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2312                         buf += cur;
2313                         res += cur;
2314                 }
2315         }
2316         mutex_unlock(&cpuhp_state_mutex);
2317         return res;
2318 }
2319 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
2320
2321 static struct attribute *cpuhp_cpu_root_attrs[] = {
2322         &dev_attr_states.attr,
2323         NULL
2324 };
2325
2326 static const struct attribute_group cpuhp_cpu_root_attr_group = {
2327         .attrs = cpuhp_cpu_root_attrs,
2328         .name = "hotplug",
2329         NULL
2330 };
2331
2332 #ifdef CONFIG_HOTPLUG_SMT
2333
2334 static ssize_t
2335 __store_smt_control(struct device *dev, struct device_attribute *attr,
2336                     const char *buf, size_t count)
2337 {
2338         int ctrlval, ret;
2339
2340         if (sysfs_streq(buf, "on"))
2341                 ctrlval = CPU_SMT_ENABLED;
2342         else if (sysfs_streq(buf, "off"))
2343                 ctrlval = CPU_SMT_DISABLED;
2344         else if (sysfs_streq(buf, "forceoff"))
2345                 ctrlval = CPU_SMT_FORCE_DISABLED;
2346         else
2347                 return -EINVAL;
2348
2349         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2350                 return -EPERM;
2351
2352         if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2353                 return -ENODEV;
2354
2355         ret = lock_device_hotplug_sysfs();
2356         if (ret)
2357                 return ret;
2358
2359         if (ctrlval != cpu_smt_control) {
2360                 switch (ctrlval) {
2361                 case CPU_SMT_ENABLED:
2362                         ret = cpuhp_smt_enable();
2363                         break;
2364                 case CPU_SMT_DISABLED:
2365                 case CPU_SMT_FORCE_DISABLED:
2366                         ret = cpuhp_smt_disable(ctrlval);
2367                         break;
2368                 }
2369         }
2370
2371         unlock_device_hotplug();
2372         return ret ? ret : count;
2373 }
2374
2375 #else /* !CONFIG_HOTPLUG_SMT */
2376 static ssize_t
2377 __store_smt_control(struct device *dev, struct device_attribute *attr,
2378                     const char *buf, size_t count)
2379 {
2380         return -ENODEV;
2381 }
2382 #endif /* CONFIG_HOTPLUG_SMT */
2383
2384 static const char *smt_states[] = {
2385         [CPU_SMT_ENABLED]               = "on",
2386         [CPU_SMT_DISABLED]              = "off",
2387         [CPU_SMT_FORCE_DISABLED]        = "forceoff",
2388         [CPU_SMT_NOT_SUPPORTED]         = "notsupported",
2389         [CPU_SMT_NOT_IMPLEMENTED]       = "notimplemented",
2390 };
2391
2392 static ssize_t
2393 show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
2394 {
2395         const char *state = smt_states[cpu_smt_control];
2396
2397         return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
2398 }
2399
2400 static ssize_t
2401 store_smt_control(struct device *dev, struct device_attribute *attr,
2402                   const char *buf, size_t count)
2403 {
2404         return __store_smt_control(dev, attr, buf, count);
2405 }
2406 static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2407
2408 static ssize_t
2409 show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2410 {
2411         return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
2412 }
2413 static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2414
2415 static struct attribute *cpuhp_smt_attrs[] = {
2416         &dev_attr_control.attr,
2417         &dev_attr_active.attr,
2418         NULL
2419 };
2420
2421 static const struct attribute_group cpuhp_smt_attr_group = {
2422         .attrs = cpuhp_smt_attrs,
2423         .name = "smt",
2424         NULL
2425 };
2426
2427 static int __init cpu_smt_sysfs_init(void)
2428 {
2429         return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2430                                   &cpuhp_smt_attr_group);
2431 }
2432
2433 static int __init cpuhp_sysfs_init(void)
2434 {
2435         int cpu, ret;
2436
2437         ret = cpu_smt_sysfs_init();
2438         if (ret)
2439                 return ret;
2440
2441         ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2442                                  &cpuhp_cpu_root_attr_group);
2443         if (ret)
2444                 return ret;
2445
2446         for_each_possible_cpu(cpu) {
2447                 struct device *dev = get_cpu_device(cpu);
2448
2449                 if (!dev)
2450                         continue;
2451                 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2452                 if (ret)
2453                         return ret;
2454         }
2455         return 0;
2456 }
2457 device_initcall(cpuhp_sysfs_init);
2458 #endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
2459
2460 /*
2461  * cpu_bit_bitmap[] is a special, "compressed" data structure that
2462  * represents all NR_CPUS bits binary values of 1<<nr.
2463  *
2464  * It is used by cpumask_of() to get a constant address to a CPU
2465  * mask value that has a single bit set only.
2466  */
2467
2468 /* cpu_bit_bitmap[0] is empty - so we can back into it */
2469 #define MASK_DECLARE_1(x)       [x+1][0] = (1UL << (x))
2470 #define MASK_DECLARE_2(x)       MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2471 #define MASK_DECLARE_4(x)       MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2472 #define MASK_DECLARE_8(x)       MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2473
2474 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2475
2476         MASK_DECLARE_8(0),      MASK_DECLARE_8(8),
2477         MASK_DECLARE_8(16),     MASK_DECLARE_8(24),
2478 #if BITS_PER_LONG > 32
2479         MASK_DECLARE_8(32),     MASK_DECLARE_8(40),
2480         MASK_DECLARE_8(48),     MASK_DECLARE_8(56),
2481 #endif
2482 };
2483 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2484
2485 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2486 EXPORT_SYMBOL(cpu_all_bits);
2487
2488 #ifdef CONFIG_INIT_ALL_POSSIBLE
2489 struct cpumask __cpu_possible_mask __read_mostly
2490         = {CPU_BITS_ALL};
2491 #else
2492 struct cpumask __cpu_possible_mask __read_mostly;
2493 #endif
2494 EXPORT_SYMBOL(__cpu_possible_mask);
2495
2496 struct cpumask __cpu_online_mask __read_mostly;
2497 EXPORT_SYMBOL(__cpu_online_mask);
2498
2499 struct cpumask __cpu_present_mask __read_mostly;
2500 EXPORT_SYMBOL(__cpu_present_mask);
2501
2502 struct cpumask __cpu_active_mask __read_mostly;
2503 EXPORT_SYMBOL(__cpu_active_mask);
2504
2505 atomic_t __num_online_cpus __read_mostly;
2506 EXPORT_SYMBOL(__num_online_cpus);
2507
2508 void init_cpu_present(const struct cpumask *src)
2509 {
2510         cpumask_copy(&__cpu_present_mask, src);
2511 }
2512
2513 void init_cpu_possible(const struct cpumask *src)
2514 {
2515         cpumask_copy(&__cpu_possible_mask, src);
2516 }
2517
2518 void init_cpu_online(const struct cpumask *src)
2519 {
2520         cpumask_copy(&__cpu_online_mask, src);
2521 }
2522
2523 void set_cpu_online(unsigned int cpu, bool online)
2524 {
2525         /*
2526          * atomic_inc/dec() is required to handle the horrid abuse of this
2527          * function by the reboot and kexec code which invoke it from
2528          * IPI/NMI broadcasts when shutting down CPUs. Invocation from
2529          * regular CPU hotplug is properly serialized.
2530          *
2531          * Note, that the fact that __num_online_cpus is of type atomic_t
2532          * does not protect readers which are not serialized against
2533          * concurrent hotplug operations.
2534          */
2535         if (online) {
2536                 if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
2537                         atomic_inc(&__num_online_cpus);
2538         } else {
2539                 if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
2540                         atomic_dec(&__num_online_cpus);
2541         }
2542 }
2543
2544 /*
2545  * Activate the first processor.
2546  */
2547 void __init boot_cpu_init(void)
2548 {
2549         int cpu = smp_processor_id();
2550
2551         /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2552         set_cpu_online(cpu, true);
2553         set_cpu_active(cpu, true);
2554         set_cpu_present(cpu, true);
2555         set_cpu_possible(cpu, true);
2556
2557 #ifdef CONFIG_SMP
2558         __boot_cpu_id = cpu;
2559 #endif
2560 }
2561
2562 /*
2563  * Must be called _AFTER_ setting up the per_cpu areas
2564  */
2565 void __init boot_cpu_hotplug_init(void)
2566 {
2567 #ifdef CONFIG_SMP
2568         cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
2569 #endif
2570         this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2571 }
2572
2573 /*
2574  * These are used for a global "mitigations=" cmdline option for toggling
2575  * optional CPU mitigations.
2576  */
2577 enum cpu_mitigations {
2578         CPU_MITIGATIONS_OFF,
2579         CPU_MITIGATIONS_AUTO,
2580         CPU_MITIGATIONS_AUTO_NOSMT,
2581 };
2582
2583 static enum cpu_mitigations cpu_mitigations __ro_after_init =
2584         CPU_MITIGATIONS_AUTO;
2585
2586 static int __init mitigations_parse_cmdline(char *arg)
2587 {
2588         if (!strcmp(arg, "off"))
2589                 cpu_mitigations = CPU_MITIGATIONS_OFF;
2590         else if (!strcmp(arg, "auto"))
2591                 cpu_mitigations = CPU_MITIGATIONS_AUTO;
2592         else if (!strcmp(arg, "auto,nosmt"))
2593                 cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2594         else
2595                 pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2596                         arg);
2597
2598         return 0;
2599 }
2600 early_param("mitigations", mitigations_parse_cmdline);
2601
2602 /* mitigations=off */
2603 bool cpu_mitigations_off(void)
2604 {
2605         return cpu_mitigations == CPU_MITIGATIONS_OFF;
2606 }
2607 EXPORT_SYMBOL_GPL(cpu_mitigations_off);
2608
2609 /* mitigations=auto,nosmt */
2610 bool cpu_mitigations_auto_nosmt(void)
2611 {
2612         return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
2613 }
2614 EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);