kernel/cpu.c

   1 /* CPU control.
   2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
   3  *
   4  * This code is licenced under the GPL.
   5  */
   6 #include <linux/proc_fs.h>
   7 #include <linux/smp.h>
   8 #include <linux/init.h>
   9 #include <linux/notifier.h>
  10 #include <linux/sched/signal.h>
  11 #include <linux/sched/hotplug.h>
  12 #include <linux/sched/task.h>
  13 #include <linux/sched/smt.h>
  14 #include <linux/unistd.h>
  15 #include <linux/cpu.h>
  16 #include <linux/oom.h>
  17 #include <linux/rcupdate.h>
  18 #include <linux/export.h>
  19 #include <linux/bug.h>
  20 #include <linux/kthread.h>
  21 #include <linux/stop_machine.h>
  22 #include <linux/mutex.h>
  23 #include <linux/gfp.h>
  24 #include <linux/suspend.h>
  25 #include <linux/lockdep.h>
  26 #include <linux/tick.h>
  27 #include <linux/irq.h>
  28 #include <linux/nmi.h>
  29 #include <linux/smpboot.h>
  30 #include <linux/relay.h>
  31 #include <linux/slab.h>
  32 #include <linux/percpu-rwsem.h>
  33
  34 #include <trace/events/power.h>
  35 #define CREATE_TRACE_POINTS
  36 #include <trace/events/cpuhp.h>
  37
  38 #include "smpboot.h"
  39
  40 /**
  41  * cpuhp_cpu_state - Per cpu hotplug state storage
  42  * @state:      The current cpu state
  43  * @target:     The target state
  44  * @thread:     Pointer to the hotplug thread
  45  * @should_run: Thread should execute
  46  * @rollback:   Perform a rollback
  47  * @single:     Single callback invocation
  48  * @bringup:    Single callback bringup or teardown selector
  49  * @cb_state:   The state for a single callback (install/uninstall)
  50  * @result:     Result of the operation
  51  * @done_up:    Signal completion to the issuer of the task for cpu-up
  52  * @done_down:  Signal completion to the issuer of the task for cpu-down
  53  */
  54 struct cpuhp_cpu_state {
  55         enum cpuhp_state        state;
  56         enum cpuhp_state        target;
  57         enum cpuhp_state        fail;
  58 #ifdef CONFIG_SMP
  59         struct task_struct      *thread;
  60         bool                    should_run;
  61         bool                    rollback;
  62         bool                    single;
  63         bool                    bringup;
  64         bool                    booted_once;
  65         struct hlist_node       *node;
  66         struct hlist_node       *last;
  67         enum cpuhp_state        cb_state;
  68         int                     result;
  69         struct completion       done_up;
  70         struct completion       done_down;
  71 #endif
  72 };
  73
  74 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
  75         .fail = CPUHP_INVALID,
  76 };
  77
  78 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
  79 static struct lockdep_map cpuhp_state_up_map =
  80         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
  81 static struct lockdep_map cpuhp_state_down_map =
  82         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
  83
  84
  85 static inline void cpuhp_lock_acquire(bool bringup)
  86 {
  87         lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  88 }
  89
  90 static inline void cpuhp_lock_release(bool bringup)
  91 {
  92         lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  93 }
  94 #else
  95
  96 static inline void cpuhp_lock_acquire(bool bringup) { }
  97 static inline void cpuhp_lock_release(bool bringup) { }
  98
  99 #endif
 100
 101 /**
 102  * cpuhp_step - Hotplug state machine step
 103  * @name:       Name of the step
 104  * @startup:    Startup function of the step
 105  * @teardown:   Teardown function of the step
 106  * @cant_stop:  Bringup/teardown can't be stopped at this step
 107  */
 108 struct cpuhp_step {
 109         const char              *name;
 110         union {
 111                 int             (*single)(unsigned int cpu);
 112                 int             (*multi)(unsigned int cpu,
 113                                          struct hlist_node *node);
 114         } startup;
 115         union {
 116                 int             (*single)(unsigned int cpu);
 117                 int             (*multi)(unsigned int cpu,
 118                                          struct hlist_node *node);
 119         } teardown;
 120         struct hlist_head       list;
 121         bool                    cant_stop;
 122         bool                    multi_instance;
 123 };
 124
 125 static DEFINE_MUTEX(cpuhp_state_mutex);
 126 static struct cpuhp_step cpuhp_hp_states[];
 127
 128 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
 129 {
 130         return cpuhp_hp_states + state;
 131 }
 132
 133 /**
 134  * cpuhp_invoke_callback _ Invoke the callbacks for a given state
 135  * @cpu:        The cpu for which the callback should be invoked
 136  * @state:      The state to do callbacks for
 137  * @bringup:    True if the bringup callback should be invoked
 138  * @node:       For multi-instance, do a single entry callback for install/remove
 139  * @lastp:      For multi-instance rollback, remember how far we got
 140  *
 141  * Called from cpu hotplug and from the state register machinery.
 142  */
 143 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
 144                                  bool bringup, struct hlist_node *node,
 145                                  struct hlist_node **lastp)
 146 {
 147         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 148         struct cpuhp_step *step = cpuhp_get_step(state);
 149         int (*cbm)(unsigned int cpu, struct hlist_node *node);
 150         int (*cb)(unsigned int cpu);
 151         int ret, cnt;
 152
 153         if (st->fail == state) {
 154                 st->fail = CPUHP_INVALID;
 155
 156                 if (!(bringup ? step->startup.single : step->teardown.single))
 157                         return 0;
 158
 159                 return -EAGAIN;
 160         }
 161
 162         if (!step->multi_instance) {
 163                 WARN_ON_ONCE(lastp && *lastp);
 164                 cb = bringup ? step->startup.single : step->teardown.single;
 165                 if (!cb)
 166                         return 0;
 167                 trace_cpuhp_enter(cpu, st->target, state, cb);
 168                 ret = cb(cpu);
 169                 trace_cpuhp_exit(cpu, st->state, state, ret);
 170                 return ret;
 171         }
 172         cbm = bringup ? step->startup.multi : step->teardown.multi;
 173         if (!cbm)
 174                 return 0;
 175
 176         /* Single invocation for instance add/remove */
 177         if (node) {
 178                 WARN_ON_ONCE(lastp && *lastp);
 179                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 180                 ret = cbm(cpu, node);
 181                 trace_cpuhp_exit(cpu, st->state, state, ret);
 182                 return ret;
 183         }
 184
 185         /* State transition. Invoke on all instances */
 186         cnt = 0;
 187         hlist_for_each(node, &step->list) {
 188                 if (lastp && node == *lastp)
 189                         break;
 190
 191                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 192                 ret = cbm(cpu, node);
 193                 trace_cpuhp_exit(cpu, st->state, state, ret);
 194                 if (ret) {
 195                         if (!lastp)
 196                                 goto err;
 197
 198                         *lastp = node;
 199                         return ret;
 200                 }
 201                 cnt++;
 202         }
 203         if (lastp)
 204                 *lastp = NULL;
 205         return 0;
 206 err:
 207         /* Rollback the instances if one failed */
 208         cbm = !bringup ? step->startup.multi : step->teardown.multi;
 209         if (!cbm)
 210                 return ret;
 211
 212         hlist_for_each(node, &step->list) {
 213                 if (!cnt--)
 214                         break;
 215
 216                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 217                 ret = cbm(cpu, node);
 218                 trace_cpuhp_exit(cpu, st->state, state, ret);
 219                 /*
 220                  * Rollback must not fail,
 221                  */
 222                 WARN_ON_ONCE(ret);
 223         }
 224         return ret;
 225 }
 226
 227 #ifdef CONFIG_SMP
 228 static bool cpuhp_is_ap_state(enum cpuhp_state state)
 229 {
 230         /*
 231          * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
 232          * purposes as that state is handled explicitly in cpu_down.
 233          */
 234         return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
 235 }
 236
 237 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 238 {
 239         struct completion *done = bringup ? &st->done_up : &st->done_down;
 240         wait_for_completion(done);
 241 }
 242
 243 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 244 {
 245         struct completion *done = bringup ? &st->done_up : &st->done_down;
 246         complete(done);
 247 }
 248
 249 /*
 250  * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
 251  */
 252 static bool cpuhp_is_atomic_state(enum cpuhp_state state)
 253 {
 254         return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
 255 }
 256
 257 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
 258 static DEFINE_MUTEX(cpu_add_remove_lock);
 259 bool cpuhp_tasks_frozen;
 260 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
 261
 262 /*
 263  * The following two APIs (cpu_maps_update_begin/done) must be used when
 264  * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
 265  */
 266 void cpu_maps_update_begin(void)
 267 {
 268         mutex_lock(&cpu_add_remove_lock);
 269 }
 270
 271 void cpu_maps_update_done(void)
 272 {
 273         mutex_unlock(&cpu_add_remove_lock);
 274 }
 275
 276 /*
 277  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
 278  * Should always be manipulated under cpu_add_remove_lock
 279  */
 280 static int cpu_hotplug_disabled;
 281
 282 #ifdef CONFIG_HOTPLUG_CPU
 283
 284 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 285
 286 void cpus_read_lock(void)
 287 {
 288         percpu_down_read(&cpu_hotplug_lock);
 289 }
 290 EXPORT_SYMBOL_GPL(cpus_read_lock);
 291
 292 int cpus_read_trylock(void)
 293 {
 294         return percpu_down_read_trylock(&cpu_hotplug_lock);
 295 }
 296 EXPORT_SYMBOL_GPL(cpus_read_trylock);
 297
 298 void cpus_read_unlock(void)
 299 {
 300         percpu_up_read(&cpu_hotplug_lock);
 301 }
 302 EXPORT_SYMBOL_GPL(cpus_read_unlock);
 303
 304 void cpus_write_lock(void)
 305 {
 306         percpu_down_write(&cpu_hotplug_lock);
 307 }
 308
 309 void cpus_write_unlock(void)
 310 {
 311         percpu_up_write(&cpu_hotplug_lock);
 312 }
 313
 314 void lockdep_assert_cpus_held(void)
 315 {
 316         /*
 317          * We can't have hotplug operations before userspace starts running,
 318          * and some init codepaths will knowingly not take the hotplug lock.
 319          * This is all valid, so mute lockdep until it makes sense to report
 320          * unheld locks.
 321          */
 322         if (system_state < SYSTEM_RUNNING)
 323                 return;
 324
 325         percpu_rwsem_assert_held(&cpu_hotplug_lock);
 326 }
 327
 328 /*
 329  * Wait for currently running CPU hotplug operations to complete (if any) and
 330  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 331  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 332  * hotplug path before performing hotplug operations. So acquiring that lock
 333  * guarantees mutual exclusion from any currently running hotplug operations.
 334  */
 335 void cpu_hotplug_disable(void)
 336 {
 337         cpu_maps_update_begin();
 338         cpu_hotplug_disabled++;
 339         cpu_maps_update_done();
 340 }
 341 EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
 342
 343 static void __cpu_hotplug_enable(void)
 344 {
 345         if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
 346                 return;
 347         cpu_hotplug_disabled--;
 348 }
 349
 350 void cpu_hotplug_enable(void)
 351 {
 352         cpu_maps_update_begin();
 353         __cpu_hotplug_enable();
 354         cpu_maps_update_done();
 355 }
 356 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 357 #endif  /* CONFIG_HOTPLUG_CPU */
 358
 359 /*
 360  * Architectures that need SMT-specific errata handling during SMT hotplug
 361  * should override this.
 362  */
 363 void __weak arch_smt_update(void) { }
 364
 365 #ifdef CONFIG_HOTPLUG_SMT
 366 enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
 367
 368 void __init cpu_smt_disable(bool force)
 369 {
 370         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 371                 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 372                 return;
 373
 374         if (force) {
 375                 pr_info("SMT: Force disabled\n");
 376                 cpu_smt_control = CPU_SMT_FORCE_DISABLED;
 377         } else {
 378                 pr_info("SMT: disabled\n");
 379                 cpu_smt_control = CPU_SMT_DISABLED;
 380         }
 381 }
 382
 383 /*
 384  * The decision whether SMT is supported can only be done after the full
 385  * CPU identification. Called from architecture code.
 386  */
 387 void __init cpu_smt_check_topology(void)
 388 {
 389         if (!topology_smt_supported())
 390                 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
 391 }
 392
 393 static int __init smt_cmdline_disable(char *str)
 394 {
 395         cpu_smt_disable(str && !strcmp(str, "force"));
 396         return 0;
 397 }
 398 early_param("nosmt", smt_cmdline_disable);
 399
 400 static inline bool cpu_smt_allowed(unsigned int cpu)
 401 {
 402         if (cpu_smt_control == CPU_SMT_ENABLED)
 403                 return true;
 404
 405         if (topology_is_primary_thread(cpu))
 406                 return true;
 407
 408         /*
 409          * On x86 it's required to boot all logical CPUs at least once so
 410          * that the init code can get a chance to set CR4.MCE on each
 411          * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any
 412          * core will shutdown the machine.
 413          */
 414         return !per_cpu(cpuhp_state, cpu).booted_once;
 415 }
 416 #else
 417 static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
 418 #endif
 419
 420 static inline enum cpuhp_state
 421 cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 422 {
 423         enum cpuhp_state prev_state = st->state;
 424
 425         st->rollback = false;
 426         st->last = NULL;
 427
 428         st->target = target;
 429         st->single = false;
 430         st->bringup = st->state < target;
 431
 432         return prev_state;
 433 }
 434
 435 static inline void
 436 cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
 437 {
 438         st->rollback = true;
 439
 440         /*
 441          * If we have st->last we need to undo partial multi_instance of this
 442          * state first. Otherwise start undo at the previous state.
 443          */
 444         if (!st->last) {
 445                 if (st->bringup)
 446                         st->state--;
 447                 else
 448                         st->state++;
 449         }
 450
 451         st->target = prev_state;
 452         st->bringup = !st->bringup;
 453 }
 454
 455 /* Regular hotplug invocation of the AP hotplug thread */
 456 static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
 457 {
 458         if (!st->single && st->state == st->target)
 459                 return;
 460
 461         st->result = 0;
 462         /*
 463          * Make sure the above stores are visible before should_run becomes
 464          * true. Paired with the mb() above in cpuhp_thread_fun()
 465          */
 466         smp_mb();
 467         st->should_run = true;
 468         wake_up_process(st->thread);
 469         wait_for_ap_thread(st, st->bringup);
 470 }
 471
 472 static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 473 {
 474         enum cpuhp_state prev_state;
 475         int ret;
 476
 477         prev_state = cpuhp_set_state(st, target);
 478         __cpuhp_kick_ap(st);
 479         if ((ret = st->result)) {
 480                 cpuhp_reset_state(st, prev_state);
 481                 __cpuhp_kick_ap(st);
 482         }
 483
 484         return ret;
 485 }
 486
 487 static int bringup_wait_for_ap(unsigned int cpu)
 488 {
 489         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 490
 491         /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
 492         wait_for_ap_thread(st, true);
 493         if (WARN_ON_ONCE((!cpu_online(cpu))))
 494                 return -ECANCELED;
 495
 496         /* Unpark the stopper thread and the hotplug thread of the target cpu */
 497         stop_machine_unpark(cpu);
 498         kthread_unpark(st->thread);
 499
 500         /*
 501          * SMT soft disabling on X86 requires to bring the CPU out of the
 502          * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
 503          * CPU marked itself as booted_once in cpu_notify_starting() so the
 504          * cpu_smt_allowed() check will now return false if this is not the
 505          * primary sibling.
 506          */
 507         if (!cpu_smt_allowed(cpu))
 508                 return -ECANCELED;
 509
 510         if (st->target <= CPUHP_AP_ONLINE_IDLE)
 511                 return 0;
 512
 513         return cpuhp_kick_ap(st, st->target);
 514 }
 515
 516 static int bringup_cpu(unsigned int cpu)
 517 {
 518         struct task_struct *idle = idle_thread_get(cpu);
 519         int ret;
 520
 521         /*
 522          * Some architectures have to walk the irq descriptors to
 523          * setup the vector space for the cpu which comes online.
 524          * Prevent irq alloc/free across the bringup.
 525          */
 526         irq_lock_sparse();
 527
 528         /* Arch-specific enabling code. */
 529         ret = __cpu_up(cpu, idle);
 530         irq_unlock_sparse();
 531         if (ret)
 532                 return ret;
 533         return bringup_wait_for_ap(cpu);
 534 }
 535
 536 /*
 537  * Hotplug state machine related functions
 538  */
 539
 540 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
 541 {
 542         for (st->state--; st->state > st->target; st->state--)
 543                 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 544 }
 545
 546 static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
 547 {
 548         if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
 549                 return true;
 550         /*
 551          * When CPU hotplug is disabled, then taking the CPU down is not
 552          * possible because takedown_cpu() and the architecture and
 553          * subsystem specific mechanisms are not available. So the CPU
 554          * which would be completely unplugged again needs to stay around
 555          * in the current state.
 556          */
 557         return st->state <= CPUHP_BRINGUP_CPU;
 558 }
 559
 560 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 561                               enum cpuhp_state target)
 562 {
 563         enum cpuhp_state prev_state = st->state;
 564         int ret = 0;
 565
 566         while (st->state < target) {
 567                 st->state++;
 568                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 569                 if (ret) {
 570                         if (can_rollback_cpu(st)) {
 571                                 st->target = prev_state;
 572                                 undo_cpu_up(cpu, st);
 573                         }
 574                         break;
 575                 }
 576         }
 577         return ret;
 578 }
 579
 580 /*
 581  * The cpu hotplug threads manage the bringup and teardown of the cpus
 582  */
 583 static void cpuhp_create(unsigned int cpu)
 584 {
 585         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 586
 587         init_completion(&st->done_up);
 588         init_completion(&st->done_down);
 589 }
 590
 591 static int cpuhp_should_run(unsigned int cpu)
 592 {
 593         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 594
 595         return st->should_run;
 596 }
 597
 598 /*
 599  * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
 600  * callbacks when a state gets [un]installed at runtime.
 601  *
 602  * Each invocation of this function by the smpboot thread does a single AP
 603  * state callback.
 604  *
 605  * It has 3 modes of operation:
 606  *  - single: runs st->cb_state
 607  *  - up:     runs ++st->state, while st->state < st->target
 608  *  - down:   runs st->state--, while st->state > st->target
 609  *
 610  * When complete or on error, should_run is cleared and the completion is fired.
 611  */
 612 static void cpuhp_thread_fun(unsigned int cpu)
 613 {
 614         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 615         bool bringup = st->bringup;
 616         enum cpuhp_state state;
 617
 618         if (WARN_ON_ONCE(!st->should_run))
 619                 return;
 620
 621         /*
 622          * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
 623          * that if we see ->should_run we also see the rest of the state.
 624          */
 625         smp_mb();
 626
 627         cpuhp_lock_acquire(bringup);
 628
 629         if (st->single) {
 630                 state = st->cb_state;
 631                 st->should_run = false;
 632         } else {
 633                 if (bringup) {
 634                         st->state++;
 635                         state = st->state;
 636                         st->should_run = (st->state < st->target);
 637                         WARN_ON_ONCE(st->state > st->target);
 638                 } else {
 639                         state = st->state;
 640                         st->state--;
 641                         st->should_run = (st->state > st->target);
 642                         WARN_ON_ONCE(st->state < st->target);
 643                 }
 644         }
 645
 646         WARN_ON_ONCE(!cpuhp_is_ap_state(state));
 647
 648         if (cpuhp_is_atomic_state(state)) {
 649                 local_irq_disable();
 650                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 651                 local_irq_enable();
 652
 653                 /*
 654                  * STARTING/DYING must not fail!
 655                  */
 656                 WARN_ON_ONCE(st->result);
 657         } else {
 658                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 659         }
 660
 661         if (st->result) {
 662                 /*
 663                  * If we fail on a rollback, we're up a creek without no
 664                  * paddle, no way forward, no way back. We loose, thanks for
 665                  * playing.
 666                  */
 667                 WARN_ON_ONCE(st->rollback);
 668                 st->should_run = false;
 669         }
 670
 671         cpuhp_lock_release(bringup);
 672
 673         if (!st->should_run)
 674                 complete_ap_thread(st, bringup);
 675 }
 676
 677 /* Invoke a single callback on a remote cpu */
 678 static int
 679 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
 680                          struct hlist_node *node)
 681 {
 682         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 683         int ret;
 684
 685         if (!cpu_online(cpu))
 686                 return 0;
 687
 688         cpuhp_lock_acquire(false);
 689         cpuhp_lock_release(false);
 690
 691         cpuhp_lock_acquire(true);
 692         cpuhp_lock_release(true);
 693
 694         /*
 695          * If we are up and running, use the hotplug thread. For early calls
 696          * we invoke the thread function directly.
 697          */
 698         if (!st->thread)
 699                 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
 700
 701         st->rollback = false;
 702         st->last = NULL;
 703
 704         st->node = node;
 705         st->bringup = bringup;
 706         st->cb_state = state;
 707         st->single = true;
 708
 709         __cpuhp_kick_ap(st);
 710
 711         /*
 712          * If we failed and did a partial, do a rollback.
 713          */
 714         if ((ret = st->result) && st->last) {
 715                 st->rollback = true;
 716                 st->bringup = !bringup;
 717
 718                 __cpuhp_kick_ap(st);
 719         }
 720
 721         /*
 722          * Clean up the leftovers so the next hotplug operation wont use stale
 723          * data.
 724          */
 725         st->node = st->last = NULL;
 726         return ret;
 727 }
 728
 729 static int cpuhp_kick_ap_work(unsigned int cpu)
 730 {
 731         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 732         enum cpuhp_state prev_state = st->state;
 733         int ret;
 734
 735         cpuhp_lock_acquire(false);
 736         cpuhp_lock_release(false);
 737
 738         cpuhp_lock_acquire(true);
 739         cpuhp_lock_release(true);
 740
 741         trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
 742         ret = cpuhp_kick_ap(st, st->target);
 743         trace_cpuhp_exit(cpu, st->state, prev_state, ret);
 744
 745         return ret;
 746 }
 747
 748 static struct smp_hotplug_thread cpuhp_threads = {
 749         .store                  = &cpuhp_state.thread,
 750         .create                 = &cpuhp_create,
 751         .thread_should_run      = cpuhp_should_run,
 752         .thread_fn              = cpuhp_thread_fun,
 753         .thread_comm            = "cpuhp/%u",
 754         .selfparking            = true,
 755 };
 756
 757 void __init cpuhp_threads_init(void)
 758 {
 759         BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
 760         kthread_unpark(this_cpu_read(cpuhp_state.thread));
 761 }
 762
 763 #ifdef CONFIG_HOTPLUG_CPU
 764 /**
 765  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
 766  * @cpu: a CPU id
 767  *
 768  * This function walks all processes, finds a valid mm struct for each one and
 769  * then clears a corresponding bit in mm's cpumask.  While this all sounds
 770  * trivial, there are various non-obvious corner cases, which this function
 771  * tries to solve in a safe manner.
 772  *
 773  * Also note that the function uses a somewhat relaxed locking scheme, so it may
 774  * be called only for an already offlined CPU.
 775  */
 776 void clear_tasks_mm_cpumask(int cpu)
 777 {
 778         struct task_struct *p;
 779
 780         /*
 781          * This function is called after the cpu is taken down and marked
 782          * offline, so its not like new tasks will ever get this cpu set in
 783          * their mm mask. -- Peter Zijlstra
 784          * Thus, we may use rcu_read_lock() here, instead of grabbing
 785          * full-fledged tasklist_lock.
 786          */
 787         WARN_ON(cpu_online(cpu));
 788         rcu_read_lock();
 789         for_each_process(p) {
 790                 struct task_struct *t;
 791
 792                 /*
 793                  * Main thread might exit, but other threads may still have
 794                  * a valid mm. Find one.
 795                  */
 796                 t = find_lock_task_mm(p);
 797                 if (!t)
 798                         continue;
 799                 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
 800                 task_unlock(t);
 801         }
 802         rcu_read_unlock();
 803 }
 804
 805 /* Take this CPU down. */
 806 static int take_cpu_down(void *_param)
 807 {
 808         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 809         enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
 810         int err, cpu = smp_processor_id();
 811         int ret;
 812
 813         /* Ensure this CPU doesn't handle any more interrupts. */
 814         err = __cpu_disable();
 815         if (err < 0)
 816                 return err;
 817
 818         /*
 819          * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
 820          * do this step again.
 821          */
 822         WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
 823         st->state--;
 824         /* Invoke the former CPU_DYING callbacks */
 825         for (; st->state > target; st->state--) {
 826                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 827                 /*
 828                  * DYING must not fail!
 829                  */
 830                 WARN_ON_ONCE(ret);
 831         }
 832
 833         /* Give up timekeeping duties */
 834         tick_handover_do_timer();
 835         /* Park the stopper thread */
 836         stop_machine_park(cpu);
 837         return 0;
 838 }
 839
 840 static int takedown_cpu(unsigned int cpu)
 841 {
 842         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 843         int err;
 844
 845         /* Park the smpboot threads */
 846         kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 847
 848         /*
 849          * Prevent irq alloc/free while the dying cpu reorganizes the
 850          * interrupt affinities.
 851          */
 852         irq_lock_sparse();
 853
 854         /*
 855          * So now all preempt/rcu users must observe !cpu_active().
 856          */
 857         err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
 858         if (err) {
 859                 /* CPU refused to die */
 860                 irq_unlock_sparse();
 861                 /* Unpark the hotplug thread so we can rollback there */
 862                 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 863                 return err;
 864         }
 865         BUG_ON(cpu_online(cpu));
 866
 867         /*
 868          * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
 869          * all runnable tasks from the CPU, there's only the idle task left now
 870          * that the migration thread is done doing the stop_machine thing.
 871          *
 872          * Wait for the stop thread to go away.
 873          */
 874         wait_for_ap_thread(st, false);
 875         BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
 876
 877         /* Interrupts are moved away from the dying cpu, reenable alloc/free */
 878         irq_unlock_sparse();
 879
 880         hotplug_cpu__broadcast_tick_pull(cpu);
 881         /* This actually kills the CPU. */
 882         __cpu_die(cpu);
 883
 884         tick_cleanup_dead_cpu(cpu);
 885         rcutree_migrate_callbacks(cpu);
 886         return 0;
 887 }
 888
 889 static void cpuhp_complete_idle_dead(void *arg)
 890 {
 891         struct cpuhp_cpu_state *st = arg;
 892
 893         complete_ap_thread(st, false);
 894 }
 895
 896 void cpuhp_report_idle_dead(void)
 897 {
 898         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 899
 900         BUG_ON(st->state != CPUHP_AP_OFFLINE);
 901         rcu_report_dead(smp_processor_id());
 902         st->state = CPUHP_AP_IDLE_DEAD;
 903         /*
 904          * We cannot call complete after rcu_report_dead() so we delegate it
 905          * to an online cpu.
 906          */
 907         smp_call_function_single(cpumask_first(cpu_online_mask),
 908                                  cpuhp_complete_idle_dead, st, 0);
 909 }
 910
 911 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
 912 {
 913         for (st->state++; st->state < st->target; st->state++)
 914                 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 915 }
 916
 917 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 918                                 enum cpuhp_state target)
 919 {
 920         enum cpuhp_state prev_state = st->state;
 921         int ret = 0;
 922
 923         for (; st->state > target; st->state--) {
 924                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 925                 if (ret) {
 926                         st->target = prev_state;
 927                         if (st->state < prev_state)
 928                                 undo_cpu_down(cpu, st);
 929                         break;
 930                 }
 931         }
 932         return ret;
 933 }
 934
 935 /* Requires cpu_add_remove_lock to be held */
 936 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 937                            enum cpuhp_state target)
 938 {
 939         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 940         int prev_state, ret = 0;
 941
 942         if (num_online_cpus() == 1)
 943                 return -EBUSY;
 944
 945         if (!cpu_present(cpu))
 946                 return -EINVAL;
 947
 948         cpus_write_lock();
 949
 950         cpuhp_tasks_frozen = tasks_frozen;
 951
 952         prev_state = cpuhp_set_state(st, target);
 953         /*
 954          * If the current CPU state is in the range of the AP hotplug thread,
 955          * then we need to kick the thread.
 956          */
 957         if (st->state > CPUHP_TEARDOWN_CPU) {
 958                 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
 959                 ret = cpuhp_kick_ap_work(cpu);
 960                 /*
 961                  * The AP side has done the error rollback already. Just
 962                  * return the error code..
 963                  */
 964                 if (ret)
 965                         goto out;
 966
 967                 /*
 968                  * We might have stopped still in the range of the AP hotplug
 969                  * thread. Nothing to do anymore.
 970                  */
 971                 if (st->state > CPUHP_TEARDOWN_CPU)
 972                         goto out;
 973
 974                 st->target = target;
 975         }
 976         /*
 977          * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
 978          * to do the further cleanups.
 979          */
 980         ret = cpuhp_down_callbacks(cpu, st, target);
 981         if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
 982                 cpuhp_reset_state(st, prev_state);
 983                 __cpuhp_kick_ap(st);
 984         }
 985
 986 out:
 987         cpus_write_unlock();
 988         /*
 989          * Do post unplug cleanup. This is still protected against
 990          * concurrent CPU hotplug via cpu_add_remove_lock.
 991          */
 992         lockup_detector_cleanup();
 993         arch_smt_update();
 994         return ret;
 995 }
 996
 997 static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
 998 {
 999         if (cpu_hotplug_disabled)
1000                 return -EBUSY;
1001         return _cpu_down(cpu, 0, target);
1002 }
1003
1004 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
1005 {
1006         int err;
1007
1008         cpu_maps_update_begin();
1009         err = cpu_down_maps_locked(cpu, target);
1010         cpu_maps_update_done();
1011         return err;
1012 }
1013
1014 int cpu_down(unsigned int cpu)
1015 {
1016         return do_cpu_down(cpu, CPUHP_OFFLINE);
1017 }
1018 EXPORT_SYMBOL(cpu_down);
1019
1020 #else
1021 #define takedown_cpu            NULL
1022 #endif /*CONFIG_HOTPLUG_CPU*/
1023
1024 /**
1025  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1026  * @cpu: cpu that just started
1027  *
1028  * It must be called by the arch code on the new cpu, before the new cpu
1029  * enables interrupts and before the "boot" cpu returns from __cpu_up().
1030  */
1031 void notify_cpu_starting(unsigned int cpu)
1032 {
1033         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1034         enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1035         int ret;
1036
1037         rcu_cpu_starting(cpu);  /* Enables RCU usage on this CPU. */
1038         st->booted_once = true;
1039         while (st->state < target) {
1040                 st->state++;
1041                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1042                 /*
1043                  * STARTING must not fail!
1044                  */
1045                 WARN_ON_ONCE(ret);
1046         }
1047 }
1048
1049 /*
1050  * Called from the idle task. Wake up the controlling task which brings the
1051  * stopper and the hotplug thread of the upcoming CPU up and then delegates
1052  * the rest of the online bringup to the hotplug thread.
1053  */
1054 void cpuhp_online_idle(enum cpuhp_state state)
1055 {
1056         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1057
1058         /* Happens for the boot cpu */
1059         if (state != CPUHP_AP_ONLINE_IDLE)
1060                 return;
1061
1062         st->state = CPUHP_AP_ONLINE_IDLE;
1063         complete_ap_thread(st, true);
1064 }
1065
1066 /* Requires cpu_add_remove_lock to be held */
1067 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1068 {
1069         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1070         struct task_struct *idle;
1071         int ret = 0;
1072
1073         cpus_write_lock();
1074
1075         if (!cpu_present(cpu)) {
1076                 ret = -EINVAL;
1077                 goto out;
1078         }
1079
1080         /*
1081          * The caller of do_cpu_up might have raced with another
1082          * caller. Ignore it for now.
1083          */
1084         if (st->state >= target)
1085                 goto out;
1086
1087         if (st->state == CPUHP_OFFLINE) {
1088                 /* Let it fail before we try to bring the cpu up */
1089                 idle = idle_thread_get(cpu);
1090                 if (IS_ERR(idle)) {
1091                         ret = PTR_ERR(idle);
1092                         goto out;
1093                 }
1094         }
1095
1096         cpuhp_tasks_frozen = tasks_frozen;
1097
1098         cpuhp_set_state(st, target);
1099         /*
1100          * If the current CPU state is in the range of the AP hotplug thread,
1101          * then we need to kick the thread once more.
1102          */
1103         if (st->state > CPUHP_BRINGUP_CPU) {
1104                 ret = cpuhp_kick_ap_work(cpu);
1105                 /*
1106                  * The AP side has done the error rollback already. Just
1107                  * return the error code..
1108                  */
1109                 if (ret)
1110                         goto out;
1111         }
1112
1113         /*
1114          * Try to reach the target state. We max out on the BP at
1115          * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1116          * responsible for bringing it up to the target state.
1117          */
1118         target = min((int)target, CPUHP_BRINGUP_CPU);
1119         ret = cpuhp_up_callbacks(cpu, st, target);
1120 out:
1121         cpus_write_unlock();
1122         arch_smt_update();
1123         return ret;
1124 }
1125
1126 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
1127 {
1128         int err = 0;
1129
1130         if (!cpu_possible(cpu)) {
1131                 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1132                        cpu);
1133 #if defined(CONFIG_IA64)
1134                 pr_err("please check additional_cpus= boot parameter\n");
1135 #endif
1136                 return -EINVAL;
1137         }
1138
1139         err = try_online_node(cpu_to_node(cpu));
1140         if (err)
1141                 return err;
1142
1143         cpu_maps_update_begin();
1144
1145         if (cpu_hotplug_disabled) {
1146                 err = -EBUSY;
1147                 goto out;
1148         }
1149         if (!cpu_smt_allowed(cpu)) {
1150                 err = -EPERM;
1151                 goto out;
1152         }
1153
1154         err = _cpu_up(cpu, 0, target);
1155 out:
1156         cpu_maps_update_done();
1157         return err;
1158 }
1159
1160 int cpu_up(unsigned int cpu)
1161 {
1162         return do_cpu_up(cpu, CPUHP_ONLINE);
1163 }
1164 EXPORT_SYMBOL_GPL(cpu_up);
1165
1166 #ifdef CONFIG_PM_SLEEP_SMP
1167 static cpumask_var_t frozen_cpus;
1168
1169 int freeze_secondary_cpus(int primary)
1170 {
1171         int cpu, error = 0;
1172
1173         cpu_maps_update_begin();
1174         if (!cpu_online(primary))
1175                 primary = cpumask_first(cpu_online_mask);
1176         /*
1177          * We take down all of the non-boot CPUs in one shot to avoid races
1178          * with the userspace trying to use the CPU hotplug at the same time
1179          */
1180         cpumask_clear(frozen_cpus);
1181
1182         pr_info("Disabling non-boot CPUs ...\n");
1183         for_each_online_cpu(cpu) {
1184                 if (cpu == primary)
1185                         continue;
1186                 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1187                 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1188                 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1189                 if (!error)
1190                         cpumask_set_cpu(cpu, frozen_cpus);
1191                 else {
1192                         pr_err("Error taking CPU%d down: %d\n", cpu, error);
1193                         break;
1194                 }
1195         }
1196
1197         if (!error)
1198                 BUG_ON(num_online_cpus() > 1);
1199         else
1200                 pr_err("Non-boot CPUs are not disabled\n");
1201
1202         /*
1203          * Make sure the CPUs won't be enabled by someone else. We need to do
1204          * this even in case of failure as all disable_nonboot_cpus() users are
1205          * supposed to do enable_nonboot_cpus() on the failure path.
1206          */
1207         cpu_hotplug_disabled++;
1208
1209         cpu_maps_update_done();
1210         return error;
1211 }
1212
1213 void __weak arch_enable_nonboot_cpus_begin(void)
1214 {
1215 }
1216
1217 void __weak arch_enable_nonboot_cpus_end(void)
1218 {
1219 }
1220
1221 void enable_nonboot_cpus(void)
1222 {
1223         int cpu, error;
1224
1225         /* Allow everyone to use the CPU hotplug again */
1226         cpu_maps_update_begin();
1227         __cpu_hotplug_enable();
1228         if (cpumask_empty(frozen_cpus))
1229                 goto out;
1230
1231         pr_info("Enabling non-boot CPUs ...\n");
1232
1233         arch_enable_nonboot_cpus_begin();
1234
1235         for_each_cpu(cpu, frozen_cpus) {
1236                 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1237                 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1238                 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1239                 if (!error) {
1240                         pr_info("CPU%d is up\n", cpu);
1241                         continue;
1242                 }
1243                 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1244         }
1245
1246         arch_enable_nonboot_cpus_end();
1247
1248         cpumask_clear(frozen_cpus);
1249 out:
1250         cpu_maps_update_done();
1251 }
1252
1253 static int __init alloc_frozen_cpus(void)
1254 {
1255         if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1256                 return -ENOMEM;
1257         return 0;
1258 }
1259 core_initcall(alloc_frozen_cpus);
1260
1261 /*
1262  * When callbacks for CPU hotplug notifications are being executed, we must
1263  * ensure that the state of the system with respect to the tasks being frozen
1264  * or not, as reported by the notification, remains unchanged *throughout the
1265  * duration* of the execution of the callbacks.
1266  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1267  *
1268  * This synchronization is implemented by mutually excluding regular CPU
1269  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1270  * Hibernate notifications.
1271  */
1272 static int
1273 cpu_hotplug_pm_callback(struct notifier_block *nb,
1274                         unsigned long action, void *ptr)
1275 {
1276         switch (action) {
1277
1278         case PM_SUSPEND_PREPARE:
1279         case PM_HIBERNATION_PREPARE:
1280                 cpu_hotplug_disable();
1281                 break;
1282
1283         case PM_POST_SUSPEND:
1284         case PM_POST_HIBERNATION:
1285                 cpu_hotplug_enable();
1286                 break;
1287
1288         default:
1289                 return NOTIFY_DONE;
1290         }
1291
1292         return NOTIFY_OK;
1293 }
1294
1295
1296 static int __init cpu_hotplug_pm_sync_init(void)
1297 {
1298         /*
1299          * cpu_hotplug_pm_callback has higher priority than x86
1300          * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1301          * to disable cpu hotplug to avoid cpu hotplug race.
1302          */
1303         pm_notifier(cpu_hotplug_pm_callback, 0);
1304         return 0;
1305 }
1306 core_initcall(cpu_hotplug_pm_sync_init);
1307
1308 #endif /* CONFIG_PM_SLEEP_SMP */
1309
1310 int __boot_cpu_id;
1311
1312 #endif /* CONFIG_SMP */
1313
1314 /* Boot processor state steps */
1315 static struct cpuhp_step cpuhp_hp_states[] = {
1316         [CPUHP_OFFLINE] = {
1317                 .name                   = "offline",
1318                 .startup.single         = NULL,
1319                 .teardown.single        = NULL,
1320         },
1321 #ifdef CONFIG_SMP
1322         [CPUHP_CREATE_THREADS]= {
1323                 .name                   = "threads:prepare",
1324                 .startup.single         = smpboot_create_threads,
1325                 .teardown.single        = NULL,
1326                 .cant_stop              = true,
1327         },
1328         [CPUHP_PERF_PREPARE] = {
1329                 .name                   = "perf:prepare",
1330                 .startup.single         = perf_event_init_cpu,
1331                 .teardown.single        = perf_event_exit_cpu,
1332         },
1333         [CPUHP_WORKQUEUE_PREP] = {
1334                 .name                   = "workqueue:prepare",
1335                 .startup.single         = workqueue_prepare_cpu,
1336                 .teardown.single        = NULL,
1337         },
1338         [CPUHP_HRTIMERS_PREPARE] = {
1339                 .name                   = "hrtimers:prepare",
1340                 .startup.single         = hrtimers_prepare_cpu,
1341                 .teardown.single        = hrtimers_dead_cpu,
1342         },
1343         [CPUHP_SMPCFD_PREPARE] = {
1344                 .name                   = "smpcfd:prepare",
1345                 .startup.single         = smpcfd_prepare_cpu,
1346                 .teardown.single        = smpcfd_dead_cpu,
1347         },
1348         [CPUHP_RELAY_PREPARE] = {
1349                 .name                   = "relay:prepare",
1350                 .startup.single         = relay_prepare_cpu,
1351                 .teardown.single        = NULL,
1352         },
1353         [CPUHP_SLAB_PREPARE] = {
1354                 .name                   = "slab:prepare",
1355                 .startup.single         = slab_prepare_cpu,
1356                 .teardown.single        = slab_dead_cpu,
1357         },
1358         [CPUHP_RCUTREE_PREP] = {
1359                 .name                   = "RCU/tree:prepare",
1360                 .startup.single         = rcutree_prepare_cpu,
1361                 .teardown.single        = rcutree_dead_cpu,
1362         },
1363         /*
1364          * On the tear-down path, timers_dead_cpu() must be invoked
1365          * before blk_mq_queue_reinit_notify() from notify_dead(),
1366          * otherwise a RCU stall occurs.
1367          */
1368         [CPUHP_TIMERS_PREPARE] = {
1369                 .name                   = "timers:prepare",
1370                 .startup.single         = timers_prepare_cpu,
1371                 .teardown.single        = timers_dead_cpu,
1372         },
1373         /* Kicks the plugged cpu into life */
1374         [CPUHP_BRINGUP_CPU] = {
1375                 .name                   = "cpu:bringup",
1376                 .startup.single         = bringup_cpu,
1377                 .teardown.single        = NULL,
1378                 .cant_stop              = true,
1379         },
1380         /* Final state before CPU kills itself */
1381         [CPUHP_AP_IDLE_DEAD] = {
1382                 .name                   = "idle:dead",
1383         },
1384         /*
1385          * Last state before CPU enters the idle loop to die. Transient state
1386          * for synchronization.
1387          */
1388         [CPUHP_AP_OFFLINE] = {
1389                 .name                   = "ap:offline",
1390                 .cant_stop              = true,
1391         },
1392         /* First state is scheduler control. Interrupts are disabled */
1393         [CPUHP_AP_SCHED_STARTING] = {
1394                 .name                   = "sched:starting",
1395                 .startup.single         = sched_cpu_starting,
1396                 .teardown.single        = sched_cpu_dying,
1397         },
1398         [CPUHP_AP_RCUTREE_DYING] = {
1399                 .name                   = "RCU/tree:dying",
1400                 .startup.single         = NULL,
1401                 .teardown.single        = rcutree_dying_cpu,
1402         },
1403         [CPUHP_AP_SMPCFD_DYING] = {
1404                 .name                   = "smpcfd:dying",
1405                 .startup.single         = NULL,
1406                 .teardown.single        = smpcfd_dying_cpu,
1407         },
1408         /* Entry state on starting. Interrupts enabled from here on. Transient
1409          * state for synchronsization */
1410         [CPUHP_AP_ONLINE] = {
1411                 .name                   = "ap:online",
1412         },
1413         /*
1414          * Handled on controll processor until the plugged processor manages
1415          * this itself.
1416          */
1417         [CPUHP_TEARDOWN_CPU] = {
1418                 .name                   = "cpu:teardown",
1419                 .startup.single         = NULL,
1420                 .teardown.single        = takedown_cpu,
1421                 .cant_stop              = true,
1422         },
1423         /* Handle smpboot threads park/unpark */
1424         [CPUHP_AP_SMPBOOT_THREADS] = {
1425                 .name                   = "smpboot/threads:online",
1426                 .startup.single         = smpboot_unpark_threads,
1427                 .teardown.single        = smpboot_park_threads,
1428         },
1429         [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1430                 .name                   = "irq/affinity:online",
1431                 .startup.single         = irq_affinity_online_cpu,
1432                 .teardown.single        = NULL,
1433         },
1434         [CPUHP_AP_PERF_ONLINE] = {
1435                 .name                   = "perf:online",
1436                 .startup.single         = perf_event_init_cpu,
1437                 .teardown.single        = perf_event_exit_cpu,
1438         },
1439         [CPUHP_AP_WATCHDOG_ONLINE] = {
1440                 .name                   = "lockup_detector:online",
1441                 .startup.single         = lockup_detector_online_cpu,
1442                 .teardown.single        = lockup_detector_offline_cpu,
1443         },
1444         [CPUHP_AP_WORKQUEUE_ONLINE] = {
1445                 .name                   = "workqueue:online",
1446                 .startup.single         = workqueue_online_cpu,
1447                 .teardown.single        = workqueue_offline_cpu,
1448         },
1449         [CPUHP_AP_RCUTREE_ONLINE] = {
1450                 .name                   = "RCU/tree:online",
1451                 .startup.single         = rcutree_online_cpu,
1452                 .teardown.single        = rcutree_offline_cpu,
1453         },
1454 #endif
1455         /*
1456          * The dynamically registered state space is here
1457          */
1458
1459 #ifdef CONFIG_SMP
1460         /* Last state is scheduler control setting the cpu active */
1461         [CPUHP_AP_ACTIVE] = {
1462                 .name                   = "sched:active",
1463                 .startup.single         = sched_cpu_activate,
1464                 .teardown.single        = sched_cpu_deactivate,
1465         },
1466 #endif
1467
1468         /* CPU is fully up and running. */
1469         [CPUHP_ONLINE] = {
1470                 .name                   = "online",
1471                 .startup.single         = NULL,
1472                 .teardown.single        = NULL,
1473         },
1474 };
1475
1476 /* Sanity check for callbacks */
1477 static int cpuhp_cb_check(enum cpuhp_state state)
1478 {
1479         if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1480                 return -EINVAL;
1481         return 0;
1482 }
1483
1484 /*
1485  * Returns a free for dynamic slot assignment of the Online state. The states
1486  * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1487  * by having no name assigned.
1488  */
1489 static int cpuhp_reserve_state(enum cpuhp_state state)
1490 {
1491         enum cpuhp_state i, end;
1492         struct cpuhp_step *step;
1493
1494         switch (state) {
1495         case CPUHP_AP_ONLINE_DYN:
1496                 step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1497                 end = CPUHP_AP_ONLINE_DYN_END;
1498                 break;
1499         case CPUHP_BP_PREPARE_DYN:
1500                 step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1501                 end = CPUHP_BP_PREPARE_DYN_END;
1502                 break;
1503         default:
1504                 return -EINVAL;
1505         }
1506
1507         for (i = state; i <= end; i++, step++) {
1508                 if (!step->name)
1509                         return i;
1510         }
1511         WARN(1, "No more dynamic states available for CPU hotplug\n");
1512         return -ENOSPC;
1513 }
1514
1515 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1516                                  int (*startup)(unsigned int cpu),
1517                                  int (*teardown)(unsigned int cpu),
1518                                  bool multi_instance)
1519 {
1520         /* (Un)Install the callbacks for further cpu hotplug operations */
1521         struct cpuhp_step *sp;
1522         int ret = 0;
1523
1524         /*
1525          * If name is NULL, then the state gets removed.
1526          *
1527          * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1528          * the first allocation from these dynamic ranges, so the removal
1529          * would trigger a new allocation and clear the wrong (already
1530          * empty) state, leaving the callbacks of the to be cleared state
1531          * dangling, which causes wreckage on the next hotplug operation.
1532          */
1533         if (name && (state == CPUHP_AP_ONLINE_DYN ||
1534                      state == CPUHP_BP_PREPARE_DYN)) {
1535                 ret = cpuhp_reserve_state(state);
1536                 if (ret < 0)
1537                         return ret;
1538                 state = ret;
1539         }
1540         sp = cpuhp_get_step(state);
1541         if (name && sp->name)
1542                 return -EBUSY;
1543
1544         sp->startup.single = startup;
1545         sp->teardown.single = teardown;
1546         sp->name = name;
1547         sp->multi_instance = multi_instance;
1548         INIT_HLIST_HEAD(&sp->list);
1549         return ret;
1550 }
1551
1552 static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1553 {
1554         return cpuhp_get_step(state)->teardown.single;
1555 }
1556
1557 /*
1558  * Call the startup/teardown function for a step either on the AP or
1559  * on the current CPU.
1560  */
1561 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1562                             struct hlist_node *node)
1563 {
1564         struct cpuhp_step *sp = cpuhp_get_step(state);
1565         int ret;
1566
1567         /*
1568          * If there's nothing to do, we done.
1569          * Relies on the union for multi_instance.
1570          */
1571         if ((bringup && !sp->startup.single) ||
1572             (!bringup && !sp->teardown.single))
1573                 return 0;
1574         /*
1575          * The non AP bound callbacks can fail on bringup. On teardown
1576          * e.g. module removal we crash for now.
1577          */
1578 #ifdef CONFIG_SMP
1579         if (cpuhp_is_ap_state(state))
1580                 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1581         else
1582                 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1583 #else
1584         ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1585 #endif
1586         BUG_ON(ret && !bringup);
1587         return ret;
1588 }
1589
1590 /*
1591  * Called from __cpuhp_setup_state on a recoverable failure.
1592  *
1593  * Note: The teardown callbacks for rollback are not allowed to fail!
1594  */
1595 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1596                                    struct hlist_node *node)
1597 {
1598         int cpu;
1599
1600         /* Roll back the already executed steps on the other cpus */
1601         for_each_present_cpu(cpu) {
1602                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1603                 int cpustate = st->state;
1604
1605                 if (cpu >= failedcpu)
1606                         break;
1607
1608                 /* Did we invoke the startup call on that cpu ? */
1609                 if (cpustate >= state)
1610                         cpuhp_issue_call(cpu, state, false, node);
1611         }
1612 }
1613
1614 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1615                                           struct hlist_node *node,
1616                                           bool invoke)
1617 {
1618         struct cpuhp_step *sp;
1619         int cpu;
1620         int ret;
1621
1622         lockdep_assert_cpus_held();
1623
1624         sp = cpuhp_get_step(state);
1625         if (sp->multi_instance == false)
1626                 return -EINVAL;
1627
1628         mutex_lock(&cpuhp_state_mutex);
1629
1630         if (!invoke || !sp->startup.multi)
1631                 goto add_node;
1632
1633         /*
1634          * Try to call the startup callback for each present cpu
1635          * depending on the hotplug state of the cpu.
1636          */
1637         for_each_present_cpu(cpu) {
1638                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1639                 int cpustate = st->state;
1640
1641                 if (cpustate < state)
1642                         continue;
1643
1644                 ret = cpuhp_issue_call(cpu, state, true, node);
1645                 if (ret) {
1646                         if (sp->teardown.multi)
1647                                 cpuhp_rollback_install(cpu, state, node);
1648                         goto unlock;
1649                 }
1650         }
1651 add_node:
1652         ret = 0;
1653         hlist_add_head(node, &sp->list);
1654 unlock:
1655         mutex_unlock(&cpuhp_state_mutex);
1656         return ret;
1657 }
1658
1659 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1660                                bool invoke)
1661 {
1662         int ret;
1663
1664         cpus_read_lock();
1665         ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1666         cpus_read_unlock();
1667         return ret;
1668 }
1669 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1670
1671 /**
1672  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1673  * @state:              The state to setup
1674  * @invoke:             If true, the startup function is invoked for cpus where
1675  *                      cpu state >= @state
1676  * @startup:            startup callback function
1677  * @teardown:           teardown callback function
1678  * @multi_instance:     State is set up for multiple instances which get
1679  *                      added afterwards.
1680  *
1681  * The caller needs to hold cpus read locked while calling this function.
1682  * Returns:
1683  *   On success:
1684  *      Positive state number if @state is CPUHP_AP_ONLINE_DYN
1685  *      0 for all other states
1686  *   On failure: proper (negative) error code
1687  */
1688 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1689                                    const char *name, bool invoke,
1690                                    int (*startup)(unsigned int cpu),
1691                                    int (*teardown)(unsigned int cpu),
1692                                    bool multi_instance)
1693 {
1694         int cpu, ret = 0;
1695         bool dynstate;
1696
1697         lockdep_assert_cpus_held();
1698
1699         if (cpuhp_cb_check(state) || !name)
1700                 return -EINVAL;
1701
1702         mutex_lock(&cpuhp_state_mutex);
1703
1704         ret = cpuhp_store_callbacks(state, name, startup, teardown,
1705                                     multi_instance);
1706
1707         dynstate = state == CPUHP_AP_ONLINE_DYN;
1708         if (ret > 0 && dynstate) {
1709                 state = ret;
1710                 ret = 0;
1711         }
1712
1713         if (ret || !invoke || !startup)
1714                 goto out;
1715
1716         /*
1717          * Try to call the startup callback for each present cpu
1718          * depending on the hotplug state of the cpu.
1719          */
1720         for_each_present_cpu(cpu) {
1721                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1722                 int cpustate = st->state;
1723
1724                 if (cpustate < state)
1725                         continue;
1726
1727                 ret = cpuhp_issue_call(cpu, state, true, NULL);
1728                 if (ret) {
1729                         if (teardown)
1730                                 cpuhp_rollback_install(cpu, state, NULL);
1731                         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1732                         goto out;
1733                 }
1734         }
1735 out:
1736         mutex_unlock(&cpuhp_state_mutex);
1737         /*
1738          * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1739          * dynamically allocated state in case of success.
1740          */
1741         if (!ret && dynstate)
1742                 return state;
1743         return ret;
1744 }
1745 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1746
1747 int __cpuhp_setup_state(enum cpuhp_state state,
1748                         const char *name, bool invoke,
1749                         int (*startup)(unsigned int cpu),
1750                         int (*teardown)(unsigned int cpu),
1751                         bool multi_instance)
1752 {
1753         int ret;
1754
1755         cpus_read_lock();
1756         ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1757                                              teardown, multi_instance);
1758         cpus_read_unlock();
1759         return ret;
1760 }
1761 EXPORT_SYMBOL(__cpuhp_setup_state);
1762
1763 int __cpuhp_state_remove_instance(enum cpuhp_state state,
1764                                   struct hlist_node *node, bool invoke)
1765 {
1766         struct cpuhp_step *sp = cpuhp_get_step(state);
1767         int cpu;
1768
1769         BUG_ON(cpuhp_cb_check(state));
1770
1771         if (!sp->multi_instance)
1772                 return -EINVAL;
1773
1774         cpus_read_lock();
1775         mutex_lock(&cpuhp_state_mutex);
1776
1777         if (!invoke || !cpuhp_get_teardown_cb(state))
1778                 goto remove;
1779         /*
1780          * Call the teardown callback for each present cpu depending
1781          * on the hotplug state of the cpu. This function is not
1782          * allowed to fail currently!
1783          */
1784         for_each_present_cpu(cpu) {
1785                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1786                 int cpustate = st->state;
1787
1788                 if (cpustate >= state)
1789                         cpuhp_issue_call(cpu, state, false, node);
1790         }
1791
1792 remove:
1793         hlist_del(node);
1794         mutex_unlock(&cpuhp_state_mutex);
1795         cpus_read_unlock();
1796
1797         return 0;
1798 }
1799 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1800
1801 /**
1802  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1803  * @state:      The state to remove
1804  * @invoke:     If true, the teardown function is invoked for cpus where
1805  *              cpu state >= @state
1806  *
1807  * The caller needs to hold cpus read locked while calling this function.
1808  * The teardown callback is currently not allowed to fail. Think
1809  * about module removal!
1810  */
1811 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
1812 {
1813         struct cpuhp_step *sp = cpuhp_get_step(state);
1814         int cpu;
1815
1816         BUG_ON(cpuhp_cb_check(state));
1817
1818         lockdep_assert_cpus_held();
1819
1820         mutex_lock(&cpuhp_state_mutex);
1821         if (sp->multi_instance) {
1822                 WARN(!hlist_empty(&sp->list),
1823                      "Error: Removing state %d which has instances left.\n",
1824                      state);
1825                 goto remove;
1826         }
1827
1828         if (!invoke || !cpuhp_get_teardown_cb(state))
1829                 goto remove;
1830
1831         /*
1832          * Call the teardown callback for each present cpu depending
1833          * on the hotplug state of the cpu. This function is not
1834          * allowed to fail currently!
1835          */
1836         for_each_present_cpu(cpu) {
1837                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1838                 int cpustate = st->state;
1839
1840                 if (cpustate >= state)
1841                         cpuhp_issue_call(cpu, state, false, NULL);
1842         }
1843 remove:
1844         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1845         mutex_unlock(&cpuhp_state_mutex);
1846 }
1847 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
1848
1849 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1850 {
1851         cpus_read_lock();
1852         __cpuhp_remove_state_cpuslocked(state, invoke);
1853         cpus_read_unlock();
1854 }
1855 EXPORT_SYMBOL(__cpuhp_remove_state);
1856
1857 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1858 static ssize_t show_cpuhp_state(struct device *dev,
1859                                 struct device_attribute *attr, char *buf)
1860 {
1861         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1862
1863         return sprintf(buf, "%d\n", st->state);
1864 }
1865 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1866
1867 static ssize_t write_cpuhp_target(struct device *dev,
1868                                   struct device_attribute *attr,
1869                                   const char *buf, size_t count)
1870 {
1871         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1872         struct cpuhp_step *sp;
1873         int target, ret;
1874
1875         ret = kstrtoint(buf, 10, &target);
1876         if (ret)
1877                 return ret;
1878
1879 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1880         if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1881                 return -EINVAL;
1882 #else
1883         if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1884                 return -EINVAL;
1885 #endif
1886
1887         ret = lock_device_hotplug_sysfs();
1888         if (ret)
1889                 return ret;
1890
1891         mutex_lock(&cpuhp_state_mutex);
1892         sp = cpuhp_get_step(target);
1893         ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1894         mutex_unlock(&cpuhp_state_mutex);
1895         if (ret)
1896                 goto out;
1897
1898         if (st->state < target)
1899                 ret = do_cpu_up(dev->id, target);
1900         else
1901                 ret = do_cpu_down(dev->id, target);
1902 out:
1903         unlock_device_hotplug();
1904         return ret ? ret : count;
1905 }
1906
1907 static ssize_t show_cpuhp_target(struct device *dev,
1908                                  struct device_attribute *attr, char *buf)
1909 {
1910         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1911
1912         return sprintf(buf, "%d\n", st->target);
1913 }
1914 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
1915
1916
1917 static ssize_t write_cpuhp_fail(struct device *dev,
1918                                 struct device_attribute *attr,
1919                                 const char *buf, size_t count)
1920 {
1921         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1922         struct cpuhp_step *sp;
1923         int fail, ret;
1924
1925         ret = kstrtoint(buf, 10, &fail);
1926         if (ret)
1927                 return ret;
1928
1929         if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
1930                 return -EINVAL;
1931
1932         /*
1933          * Cannot fail STARTING/DYING callbacks.
1934          */
1935         if (cpuhp_is_atomic_state(fail))
1936                 return -EINVAL;
1937
1938         /*
1939          * Cannot fail anything that doesn't have callbacks.
1940          */
1941         mutex_lock(&cpuhp_state_mutex);
1942         sp = cpuhp_get_step(fail);
1943         if (!sp->startup.single && !sp->teardown.single)
1944                 ret = -EINVAL;
1945         mutex_unlock(&cpuhp_state_mutex);
1946         if (ret)
1947                 return ret;
1948
1949         st->fail = fail;
1950
1951         return count;
1952 }
1953
1954 static ssize_t show_cpuhp_fail(struct device *dev,
1955                                struct device_attribute *attr, char *buf)
1956 {
1957         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1958
1959         return sprintf(buf, "%d\n", st->fail);
1960 }
1961
1962 static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
1963
1964 static struct attribute *cpuhp_cpu_attrs[] = {
1965         &dev_attr_state.attr,
1966         &dev_attr_target.attr,
1967         &dev_attr_fail.attr,
1968         NULL
1969 };
1970
1971 static const struct attribute_group cpuhp_cpu_attr_group = {
1972         .attrs = cpuhp_cpu_attrs,
1973         .name = "hotplug",
1974         NULL
1975 };
1976
1977 static ssize_t show_cpuhp_states(struct device *dev,
1978                                  struct device_attribute *attr, char *buf)
1979 {
1980         ssize_t cur, res = 0;
1981         int i;
1982
1983         mutex_lock(&cpuhp_state_mutex);
1984         for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
1985                 struct cpuhp_step *sp = cpuhp_get_step(i);
1986
1987                 if (sp->name) {
1988                         cur = sprintf(buf, "%3d: %s\n", i, sp->name);
1989                         buf += cur;
1990                         res += cur;
1991                 }
1992         }
1993         mutex_unlock(&cpuhp_state_mutex);
1994         return res;
1995 }
1996 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
1997
1998 static struct attribute *cpuhp_cpu_root_attrs[] = {
1999         &dev_attr_states.attr,
2000         NULL
2001 };
2002
2003 static const struct attribute_group cpuhp_cpu_root_attr_group = {
2004         .attrs = cpuhp_cpu_root_attrs,
2005         .name = "hotplug",
2006         NULL
2007 };
2008
2009 #ifdef CONFIG_HOTPLUG_SMT
2010
2011 static const char *smt_states[] = {
2012         [CPU_SMT_ENABLED]               = "on",
2013         [CPU_SMT_DISABLED]              = "off",
2014         [CPU_SMT_FORCE_DISABLED]        = "forceoff",
2015         [CPU_SMT_NOT_SUPPORTED]         = "notsupported",
2016 };
2017
2018 static ssize_t
2019 show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
2020 {
2021         return snprintf(buf, PAGE_SIZE - 2, "%s\n", smt_states[cpu_smt_control]);
2022 }
2023
2024 static void cpuhp_offline_cpu_device(unsigned int cpu)
2025 {
2026         struct device *dev = get_cpu_device(cpu);
2027
2028         dev->offline = true;
2029         /* Tell user space about the state change */
2030         kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2031 }
2032
2033 static void cpuhp_online_cpu_device(unsigned int cpu)
2034 {
2035         struct device *dev = get_cpu_device(cpu);
2036
2037         dev->offline = false;
2038         /* Tell user space about the state change */
2039         kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2040 }
2041
2042 int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2043 {
2044         int cpu, ret = 0;
2045
2046         cpu_maps_update_begin();
2047         for_each_online_cpu(cpu) {
2048                 if (topology_is_primary_thread(cpu))
2049                         continue;
2050                 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2051                 if (ret)
2052                         break;
2053                 /*
2054                  * As this needs to hold the cpu maps lock it's impossible
2055                  * to call device_offline() because that ends up calling
2056                  * cpu_down() which takes cpu maps lock. cpu maps lock
2057                  * needs to be held as this might race against in kernel
2058                  * abusers of the hotplug machinery (thermal management).
2059                  *
2060                  * So nothing would update device:offline state. That would
2061                  * leave the sysfs entry stale and prevent onlining after
2062                  * smt control has been changed to 'off' again. This is
2063                  * called under the sysfs hotplug lock, so it is properly
2064                  * serialized against the regular offline usage.
2065                  */
2066                 cpuhp_offline_cpu_device(cpu);
2067         }
2068         if (!ret) {
2069                 cpu_smt_control = ctrlval;
2070                 arch_smt_update();
2071         }
2072         cpu_maps_update_done();
2073         return ret;
2074 }
2075
2076 int cpuhp_smt_enable(void)
2077 {
2078         int cpu, ret = 0;
2079
2080         cpu_maps_update_begin();
2081         cpu_smt_control = CPU_SMT_ENABLED;
2082         arch_smt_update();
2083         for_each_present_cpu(cpu) {
2084                 /* Skip online CPUs and CPUs on offline nodes */
2085                 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2086                         continue;
2087                 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2088                 if (ret)
2089                         break;
2090                 /* See comment in cpuhp_smt_disable() */
2091                 cpuhp_online_cpu_device(cpu);
2092         }
2093         cpu_maps_update_done();
2094         return ret;
2095 }
2096
2097 static ssize_t
2098 store_smt_control(struct device *dev, struct device_attribute *attr,
2099                   const char *buf, size_t count)
2100 {
2101         int ctrlval, ret;
2102
2103         if (sysfs_streq(buf, "on"))
2104                 ctrlval = CPU_SMT_ENABLED;
2105         else if (sysfs_streq(buf, "off"))
2106                 ctrlval = CPU_SMT_DISABLED;
2107         else if (sysfs_streq(buf, "forceoff"))
2108                 ctrlval = CPU_SMT_FORCE_DISABLED;
2109         else
2110                 return -EINVAL;
2111
2112         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2113                 return -EPERM;
2114
2115         if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2116                 return -ENODEV;
2117
2118         ret = lock_device_hotplug_sysfs();
2119         if (ret)
2120                 return ret;
2121
2122         if (ctrlval != cpu_smt_control) {
2123                 switch (ctrlval) {
2124                 case CPU_SMT_ENABLED:
2125                         ret = cpuhp_smt_enable();
2126                         break;
2127                 case CPU_SMT_DISABLED:
2128                 case CPU_SMT_FORCE_DISABLED:
2129                         ret = cpuhp_smt_disable(ctrlval);
2130                         break;
2131                 }
2132         }
2133
2134         unlock_device_hotplug();
2135         return ret ? ret : count;
2136 }
2137 static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2138
2139 static ssize_t
2140 show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2141 {
2142         bool active = topology_max_smt_threads() > 1;
2143
2144         return snprintf(buf, PAGE_SIZE - 2, "%d\n", active);
2145 }
2146 static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2147
2148 static struct attribute *cpuhp_smt_attrs[] = {
2149         &dev_attr_control.attr,
2150         &dev_attr_active.attr,
2151         NULL
2152 };
2153
2154 static const struct attribute_group cpuhp_smt_attr_group = {
2155         .attrs = cpuhp_smt_attrs,
2156         .name = "smt",
2157         NULL
2158 };
2159
2160 static int __init cpu_smt_state_init(void)
2161 {
2162         return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2163                                   &cpuhp_smt_attr_group);
2164 }
2165
2166 #else
2167 static inline int cpu_smt_state_init(void) { return 0; }
2168 #endif
2169
2170 static int __init cpuhp_sysfs_init(void)
2171 {
2172         int cpu, ret;
2173
2174         ret = cpu_smt_state_init();
2175         if (ret)
2176                 return ret;
2177
2178         ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2179                                  &cpuhp_cpu_root_attr_group);
2180         if (ret)
2181                 return ret;
2182
2183         for_each_possible_cpu(cpu) {
2184                 struct device *dev = get_cpu_device(cpu);
2185
2186                 if (!dev)
2187                         continue;
2188                 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2189                 if (ret)
2190                         return ret;
2191         }
2192         return 0;
2193 }
2194 device_initcall(cpuhp_sysfs_init);
2195 #endif
2196
2197 /*
2198  * cpu_bit_bitmap[] is a special, "compressed" data structure that
2199  * represents all NR_CPUS bits binary values of 1<<nr.
2200  *
2201  * It is used by cpumask_of() to get a constant address to a CPU
2202  * mask value that has a single bit set only.
2203  */
2204
2205 /* cpu_bit_bitmap[0] is empty - so we can back into it */
2206 #define MASK_DECLARE_1(x)       [x+1][0] = (1UL << (x))
2207 #define MASK_DECLARE_2(x)       MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2208 #define MASK_DECLARE_4(x)       MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2209 #define MASK_DECLARE_8(x)       MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2210
2211 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2212
2213         MASK_DECLARE_8(0),      MASK_DECLARE_8(8),
2214         MASK_DECLARE_8(16),     MASK_DECLARE_8(24),
2215 #if BITS_PER_LONG > 32
2216         MASK_DECLARE_8(32),     MASK_DECLARE_8(40),
2217         MASK_DECLARE_8(48),     MASK_DECLARE_8(56),
2218 #endif
2219 };
2220 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2221
2222 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2223 EXPORT_SYMBOL(cpu_all_bits);
2224
2225 #ifdef CONFIG_INIT_ALL_POSSIBLE
2226 struct cpumask __cpu_possible_mask __read_mostly
2227         = {CPU_BITS_ALL};
2228 #else
2229 struct cpumask __cpu_possible_mask __read_mostly;
2230 #endif
2231 EXPORT_SYMBOL(__cpu_possible_mask);
2232
2233 struct cpumask __cpu_online_mask __read_mostly;
2234 EXPORT_SYMBOL(__cpu_online_mask);
2235
2236 struct cpumask __cpu_present_mask __read_mostly;
2237 EXPORT_SYMBOL(__cpu_present_mask);
2238
2239 struct cpumask __cpu_active_mask __read_mostly;
2240 EXPORT_SYMBOL(__cpu_active_mask);
2241
2242 void init_cpu_present(const struct cpumask *src)
2243 {
2244         cpumask_copy(&__cpu_present_mask, src);
2245 }
2246
2247 void init_cpu_possible(const struct cpumask *src)
2248 {
2249         cpumask_copy(&__cpu_possible_mask, src);
2250 }
2251
2252 void init_cpu_online(const struct cpumask *src)
2253 {
2254         cpumask_copy(&__cpu_online_mask, src);
2255 }
2256
2257 /*
2258  * Activate the first processor.
2259  */
2260 void __init boot_cpu_init(void)
2261 {
2262         int cpu = smp_processor_id();
2263
2264         /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2265         set_cpu_online(cpu, true);
2266         set_cpu_active(cpu, true);
2267         set_cpu_present(cpu, true);
2268         set_cpu_possible(cpu, true);
2269
2270 #ifdef CONFIG_SMP
2271         __boot_cpu_id = cpu;
2272 #endif
2273 }
2274
2275 /*
2276  * Must be called _AFTER_ setting up the per_cpu areas
2277  */
2278 void __init boot_cpu_hotplug_init(void)
2279 {
2280 #ifdef CONFIG_SMP
2281         this_cpu_write(cpuhp_state.booted_once, true);
2282 #endif
2283         this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2284 }
2285
2286 /*
2287  * These are used for a global "mitigations=" cmdline option for toggling
2288  * optional CPU mitigations.
2289  */
2290 enum cpu_mitigations {
2291         CPU_MITIGATIONS_OFF,
2292         CPU_MITIGATIONS_AUTO,
2293         CPU_MITIGATIONS_AUTO_NOSMT,
2294 };
2295
2296 static enum cpu_mitigations cpu_mitigations __ro_after_init =
2297         CPU_MITIGATIONS_AUTO;
2298
2299 static int __init mitigations_parse_cmdline(char *arg)
2300 {
2301         if (!strcmp(arg, "off"))
2302                 cpu_mitigations = CPU_MITIGATIONS_OFF;
2303         else if (!strcmp(arg, "auto"))
2304                 cpu_mitigations = CPU_MITIGATIONS_AUTO;
2305         else if (!strcmp(arg, "auto,nosmt"))
2306                 cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2307         else
2308                 pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2309                         arg);
2310
2311         return 0;
2312 }
2313 early_param("mitigations", mitigations_parse_cmdline);
2314
2315 /* mitigations=off */
2316 bool cpu_mitigations_off(void)
2317 {
2318         return cpu_mitigations == CPU_MITIGATIONS_OFF;
2319 }
2320 EXPORT_SYMBOL_GPL(cpu_mitigations_off);
2321
2322 /* mitigations=auto,nosmt */
2323 bool cpu_mitigations_auto_nosmt(void)
2324 {
2325         return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
2326 }
2327 EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);