kernel/cpu.c

   1 /* CPU control.
   2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
   3  *
   4  * This code is licenced under the GPL.
   5  */
   6 #include <linux/proc_fs.h>
   7 #include <linux/smp.h>
   8 #include <linux/init.h>
   9 #include <linux/notifier.h>
  10 #include <linux/sched/signal.h>
  11 #include <linux/sched/hotplug.h>
  12 #include <linux/sched/task.h>
  13 #include <linux/sched/smt.h>
  14 #include <linux/unistd.h>
  15 #include <linux/cpu.h>
  16 #include <linux/oom.h>
  17 #include <linux/rcupdate.h>
  18 #include <linux/export.h>
  19 #include <linux/bug.h>
  20 #include <linux/kthread.h>
  21 #include <linux/stop_machine.h>
  22 #include <linux/mutex.h>
  23 #include <linux/gfp.h>
  24 #include <linux/suspend.h>
  25 #include <linux/lockdep.h>
  26 #include <linux/tick.h>
  27 #include <linux/irq.h>
  28 #include <linux/nmi.h>
  29 #include <linux/smpboot.h>
  30 #include <linux/relay.h>
  31 #include <linux/slab.h>
  32 #include <linux/percpu-rwsem.h>
  33
  34 #include <trace/events/power.h>
  35 #define CREATE_TRACE_POINTS
  36 #include <trace/events/cpuhp.h>
  37
  38 #include "smpboot.h"
  39
  40 /**
  41  * cpuhp_cpu_state - Per cpu hotplug state storage
  42  * @state:      The current cpu state
  43  * @target:     The target state
  44  * @thread:     Pointer to the hotplug thread
  45  * @should_run: Thread should execute
  46  * @rollback:   Perform a rollback
  47  * @single:     Single callback invocation
  48  * @bringup:    Single callback bringup or teardown selector
  49  * @cb_state:   The state for a single callback (install/uninstall)
  50  * @result:     Result of the operation
  51  * @done_up:    Signal completion to the issuer of the task for cpu-up
  52  * @done_down:  Signal completion to the issuer of the task for cpu-down
  53  */
  54 struct cpuhp_cpu_state {
  55         enum cpuhp_state        state;
  56         enum cpuhp_state        target;
  57         enum cpuhp_state        fail;
  58 #ifdef CONFIG_SMP
  59         struct task_struct      *thread;
  60         bool                    should_run;
  61         bool                    rollback;
  62         bool                    single;
  63         bool                    bringup;
  64         bool                    booted_once;
  65         struct hlist_node       *node;
  66         struct hlist_node       *last;
  67         enum cpuhp_state        cb_state;
  68         int                     result;
  69         struct completion       done_up;
  70         struct completion       done_down;
  71 #endif
  72 };
  73
  74 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
  75         .fail = CPUHP_INVALID,
  76 };
  77
  78 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
  79 static struct lockdep_map cpuhp_state_up_map =
  80         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
  81 static struct lockdep_map cpuhp_state_down_map =
  82         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
  83
  84
  85 static void inline cpuhp_lock_acquire(bool bringup)
  86 {
  87         lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  88 }
  89
  90 static void inline cpuhp_lock_release(bool bringup)
  91 {
  92         lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  93 }
  94 #else
  95
  96 static void inline cpuhp_lock_acquire(bool bringup) { }
  97 static void inline cpuhp_lock_release(bool bringup) { }
  98
  99 #endif
 100
 101 /**
 102  * cpuhp_step - Hotplug state machine step
 103  * @name:       Name of the step
 104  * @startup:    Startup function of the step
 105  * @teardown:   Teardown function of the step
 106  * @skip_onerr: Do not invoke the functions on error rollback
 107  *              Will go away once the notifiers are gone
 108  * @cant_stop:  Bringup/teardown can't be stopped at this step
 109  */
 110 struct cpuhp_step {
 111         const char              *name;
 112         union {
 113                 int             (*single)(unsigned int cpu);
 114                 int             (*multi)(unsigned int cpu,
 115                                          struct hlist_node *node);
 116         } startup;
 117         union {
 118                 int             (*single)(unsigned int cpu);
 119                 int             (*multi)(unsigned int cpu,
 120                                          struct hlist_node *node);
 121         } teardown;
 122         struct hlist_head       list;
 123         bool                    skip_onerr;
 124         bool                    cant_stop;
 125         bool                    multi_instance;
 126 };
 127
 128 static DEFINE_MUTEX(cpuhp_state_mutex);
 129 static struct cpuhp_step cpuhp_bp_states[];
 130 static struct cpuhp_step cpuhp_ap_states[];
 131
 132 static bool cpuhp_is_ap_state(enum cpuhp_state state)
 133 {
 134         /*
 135          * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
 136          * purposes as that state is handled explicitly in cpu_down.
 137          */
 138         return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
 139 }
 140
 141 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
 142 {
 143         struct cpuhp_step *sp;
 144
 145         sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
 146         return sp + state;
 147 }
 148
 149 /**
 150  * cpuhp_invoke_callback _ Invoke the callbacks for a given state
 151  * @cpu:        The cpu for which the callback should be invoked
 152  * @state:      The state to do callbacks for
 153  * @bringup:    True if the bringup callback should be invoked
 154  * @node:       For multi-instance, do a single entry callback for install/remove
 155  * @lastp:      For multi-instance rollback, remember how far we got
 156  *
 157  * Called from cpu hotplug and from the state register machinery.
 158  */
 159 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
 160                                  bool bringup, struct hlist_node *node,
 161                                  struct hlist_node **lastp)
 162 {
 163         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 164         struct cpuhp_step *step = cpuhp_get_step(state);
 165         int (*cbm)(unsigned int cpu, struct hlist_node *node);
 166         int (*cb)(unsigned int cpu);
 167         int ret, cnt;
 168
 169         if (st->fail == state) {
 170                 st->fail = CPUHP_INVALID;
 171
 172                 if (!(bringup ? step->startup.single : step->teardown.single))
 173                         return 0;
 174
 175                 return -EAGAIN;
 176         }
 177
 178         if (!step->multi_instance) {
 179                 WARN_ON_ONCE(lastp && *lastp);
 180                 cb = bringup ? step->startup.single : step->teardown.single;
 181                 if (!cb)
 182                         return 0;
 183                 trace_cpuhp_enter(cpu, st->target, state, cb);
 184                 ret = cb(cpu);
 185                 trace_cpuhp_exit(cpu, st->state, state, ret);
 186                 return ret;
 187         }
 188         cbm = bringup ? step->startup.multi : step->teardown.multi;
 189         if (!cbm)
 190                 return 0;
 191
 192         /* Single invocation for instance add/remove */
 193         if (node) {
 194                 WARN_ON_ONCE(lastp && *lastp);
 195                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 196                 ret = cbm(cpu, node);
 197                 trace_cpuhp_exit(cpu, st->state, state, ret);
 198                 return ret;
 199         }
 200
 201         /* State transition. Invoke on all instances */
 202         cnt = 0;
 203         hlist_for_each(node, &step->list) {
 204                 if (lastp && node == *lastp)
 205                         break;
 206
 207                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 208                 ret = cbm(cpu, node);
 209                 trace_cpuhp_exit(cpu, st->state, state, ret);
 210                 if (ret) {
 211                         if (!lastp)
 212                                 goto err;
 213
 214                         *lastp = node;
 215                         return ret;
 216                 }
 217                 cnt++;
 218         }
 219         if (lastp)
 220                 *lastp = NULL;
 221         return 0;
 222 err:
 223         /* Rollback the instances if one failed */
 224         cbm = !bringup ? step->startup.multi : step->teardown.multi;
 225         if (!cbm)
 226                 return ret;
 227
 228         hlist_for_each(node, &step->list) {
 229                 if (!cnt--)
 230                         break;
 231
 232                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 233                 ret = cbm(cpu, node);
 234                 trace_cpuhp_exit(cpu, st->state, state, ret);
 235                 /*
 236                  * Rollback must not fail,
 237                  */
 238                 WARN_ON_ONCE(ret);
 239         }
 240         return ret;
 241 }
 242
 243 #ifdef CONFIG_SMP
 244 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 245 {
 246         struct completion *done = bringup ? &st->done_up : &st->done_down;
 247         wait_for_completion(done);
 248 }
 249
 250 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 251 {
 252         struct completion *done = bringup ? &st->done_up : &st->done_down;
 253         complete(done);
 254 }
 255
 256 /*
 257  * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
 258  */
 259 static bool cpuhp_is_atomic_state(enum cpuhp_state state)
 260 {
 261         return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
 262 }
 263
 264 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
 265 static DEFINE_MUTEX(cpu_add_remove_lock);
 266 bool cpuhp_tasks_frozen;
 267 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
 268
 269 /*
 270  * The following two APIs (cpu_maps_update_begin/done) must be used when
 271  * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
 272  */
 273 void cpu_maps_update_begin(void)
 274 {
 275         mutex_lock(&cpu_add_remove_lock);
 276 }
 277
 278 void cpu_maps_update_done(void)
 279 {
 280         mutex_unlock(&cpu_add_remove_lock);
 281 }
 282
 283 /*
 284  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
 285  * Should always be manipulated under cpu_add_remove_lock
 286  */
 287 static int cpu_hotplug_disabled;
 288
 289 #ifdef CONFIG_HOTPLUG_CPU
 290
 291 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 292
 293 void cpus_read_lock(void)
 294 {
 295         percpu_down_read(&cpu_hotplug_lock);
 296 }
 297 EXPORT_SYMBOL_GPL(cpus_read_lock);
 298
 299 void cpus_read_unlock(void)
 300 {
 301         percpu_up_read(&cpu_hotplug_lock);
 302 }
 303 EXPORT_SYMBOL_GPL(cpus_read_unlock);
 304
 305 void cpus_write_lock(void)
 306 {
 307         percpu_down_write(&cpu_hotplug_lock);
 308 }
 309
 310 void cpus_write_unlock(void)
 311 {
 312         percpu_up_write(&cpu_hotplug_lock);
 313 }
 314
 315 void lockdep_assert_cpus_held(void)
 316 {
 317         percpu_rwsem_assert_held(&cpu_hotplug_lock);
 318 }
 319
 320 /*
 321  * Wait for currently running CPU hotplug operations to complete (if any) and
 322  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 323  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 324  * hotplug path before performing hotplug operations. So acquiring that lock
 325  * guarantees mutual exclusion from any currently running hotplug operations.
 326  */
 327 void cpu_hotplug_disable(void)
 328 {
 329         cpu_maps_update_begin();
 330         cpu_hotplug_disabled++;
 331         cpu_maps_update_done();
 332 }
 333 EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
 334
 335 static void __cpu_hotplug_enable(void)
 336 {
 337         if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
 338                 return;
 339         cpu_hotplug_disabled--;
 340 }
 341
 342 void cpu_hotplug_enable(void)
 343 {
 344         cpu_maps_update_begin();
 345         __cpu_hotplug_enable();
 346         cpu_maps_update_done();
 347 }
 348 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 349 #endif  /* CONFIG_HOTPLUG_CPU */
 350
 351 /*
 352  * Architectures that need SMT-specific errata handling during SMT hotplug
 353  * should override this.
 354  */
 355 void __weak arch_smt_update(void) { }
 356
 357 #ifdef CONFIG_HOTPLUG_SMT
 358 enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
 359
 360 void __init cpu_smt_disable(bool force)
 361 {
 362         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 363                 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 364                 return;
 365
 366         if (force) {
 367                 pr_info("SMT: Force disabled\n");
 368                 cpu_smt_control = CPU_SMT_FORCE_DISABLED;
 369         } else {
 370                 cpu_smt_control = CPU_SMT_DISABLED;
 371         }
 372 }
 373
 374 /*
 375  * The decision whether SMT is supported can only be done after the full
 376  * CPU identification. Called from architecture code.
 377  */
 378 void __init cpu_smt_check_topology(void)
 379 {
 380         if (!topology_smt_supported())
 381                 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
 382 }
 383
 384 static int __init smt_cmdline_disable(char *str)
 385 {
 386         cpu_smt_disable(str && !strcmp(str, "force"));
 387         return 0;
 388 }
 389 early_param("nosmt", smt_cmdline_disable);
 390
 391 static inline bool cpu_smt_allowed(unsigned int cpu)
 392 {
 393         if (cpu_smt_control == CPU_SMT_ENABLED)
 394                 return true;
 395
 396         if (topology_is_primary_thread(cpu))
 397                 return true;
 398
 399         /*
 400          * On x86 it's required to boot all logical CPUs at least once so
 401          * that the init code can get a chance to set CR4.MCE on each
 402          * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any
 403          * core will shutdown the machine.
 404          */
 405         return !per_cpu(cpuhp_state, cpu).booted_once;
 406 }
 407 #else
 408 static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
 409 #endif
 410
 411 static inline enum cpuhp_state
 412 cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 413 {
 414         enum cpuhp_state prev_state = st->state;
 415
 416         st->rollback = false;
 417         st->last = NULL;
 418
 419         st->target = target;
 420         st->single = false;
 421         st->bringup = st->state < target;
 422
 423         return prev_state;
 424 }
 425
 426 static inline void
 427 cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
 428 {
 429         st->rollback = true;
 430
 431         /*
 432          * If we have st->last we need to undo partial multi_instance of this
 433          * state first. Otherwise start undo at the previous state.
 434          */
 435         if (!st->last) {
 436                 if (st->bringup)
 437                         st->state--;
 438                 else
 439                         st->state++;
 440         }
 441
 442         st->target = prev_state;
 443         st->bringup = !st->bringup;
 444 }
 445
 446 /* Regular hotplug invocation of the AP hotplug thread */
 447 static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
 448 {
 449         if (!st->single && st->state == st->target)
 450                 return;
 451
 452         st->result = 0;
 453         /*
 454          * Make sure the above stores are visible before should_run becomes
 455          * true. Paired with the mb() above in cpuhp_thread_fun()
 456          */
 457         smp_mb();
 458         st->should_run = true;
 459         wake_up_process(st->thread);
 460         wait_for_ap_thread(st, st->bringup);
 461 }
 462
 463 static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 464 {
 465         enum cpuhp_state prev_state;
 466         int ret;
 467
 468         prev_state = cpuhp_set_state(st, target);
 469         __cpuhp_kick_ap(st);
 470         if ((ret = st->result)) {
 471                 cpuhp_reset_state(st, prev_state);
 472                 __cpuhp_kick_ap(st);
 473         }
 474
 475         return ret;
 476 }
 477
 478 static int bringup_wait_for_ap(unsigned int cpu)
 479 {
 480         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 481
 482         /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
 483         wait_for_ap_thread(st, true);
 484         if (WARN_ON_ONCE((!cpu_online(cpu))))
 485                 return -ECANCELED;
 486
 487         /* Unpark the stopper thread and the hotplug thread of the target cpu */
 488         stop_machine_unpark(cpu);
 489         kthread_unpark(st->thread);
 490
 491         /*
 492          * SMT soft disabling on X86 requires to bring the CPU out of the
 493          * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
 494          * CPU marked itself as booted_once in cpu_notify_starting() so the
 495          * cpu_smt_allowed() check will now return false if this is not the
 496          * primary sibling.
 497          */
 498         if (!cpu_smt_allowed(cpu))
 499                 return -ECANCELED;
 500
 501         if (st->target <= CPUHP_AP_ONLINE_IDLE)
 502                 return 0;
 503
 504         return cpuhp_kick_ap(st, st->target);
 505 }
 506
 507 static int bringup_cpu(unsigned int cpu)
 508 {
 509         struct task_struct *idle = idle_thread_get(cpu);
 510         int ret;
 511
 512         /*
 513          * Some architectures have to walk the irq descriptors to
 514          * setup the vector space for the cpu which comes online.
 515          * Prevent irq alloc/free across the bringup.
 516          */
 517         irq_lock_sparse();
 518
 519         /* Arch-specific enabling code. */
 520         ret = __cpu_up(cpu, idle);
 521         irq_unlock_sparse();
 522         if (ret)
 523                 return ret;
 524         return bringup_wait_for_ap(cpu);
 525 }
 526
 527 /*
 528  * Hotplug state machine related functions
 529  */
 530
 531 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
 532 {
 533         for (st->state--; st->state > st->target; st->state--) {
 534                 struct cpuhp_step *step = cpuhp_get_step(st->state);
 535
 536                 if (!step->skip_onerr)
 537                         cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 538         }
 539 }
 540
 541 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 542                               enum cpuhp_state target)
 543 {
 544         enum cpuhp_state prev_state = st->state;
 545         int ret = 0;
 546
 547         while (st->state < target) {
 548                 st->state++;
 549                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 550                 if (ret) {
 551                         st->target = prev_state;
 552                         undo_cpu_up(cpu, st);
 553                         break;
 554                 }
 555         }
 556         return ret;
 557 }
 558
 559 /*
 560  * The cpu hotplug threads manage the bringup and teardown of the cpus
 561  */
 562 static void cpuhp_create(unsigned int cpu)
 563 {
 564         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 565
 566         init_completion(&st->done_up);
 567         init_completion(&st->done_down);
 568 }
 569
 570 static int cpuhp_should_run(unsigned int cpu)
 571 {
 572         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 573
 574         return st->should_run;
 575 }
 576
 577 /*
 578  * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
 579  * callbacks when a state gets [un]installed at runtime.
 580  *
 581  * Each invocation of this function by the smpboot thread does a single AP
 582  * state callback.
 583  *
 584  * It has 3 modes of operation:
 585  *  - single: runs st->cb_state
 586  *  - up:     runs ++st->state, while st->state < st->target
 587  *  - down:   runs st->state--, while st->state > st->target
 588  *
 589  * When complete or on error, should_run is cleared and the completion is fired.
 590  */
 591 static void cpuhp_thread_fun(unsigned int cpu)
 592 {
 593         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 594         bool bringup = st->bringup;
 595         enum cpuhp_state state;
 596
 597         if (WARN_ON_ONCE(!st->should_run))
 598                 return;
 599
 600         /*
 601          * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
 602          * that if we see ->should_run we also see the rest of the state.
 603          */
 604         smp_mb();
 605
 606         cpuhp_lock_acquire(bringup);
 607
 608         if (st->single) {
 609                 state = st->cb_state;
 610                 st->should_run = false;
 611         } else {
 612                 if (bringup) {
 613                         st->state++;
 614                         state = st->state;
 615                         st->should_run = (st->state < st->target);
 616                         WARN_ON_ONCE(st->state > st->target);
 617                 } else {
 618                         state = st->state;
 619                         st->state--;
 620                         st->should_run = (st->state > st->target);
 621                         WARN_ON_ONCE(st->state < st->target);
 622                 }
 623         }
 624
 625         WARN_ON_ONCE(!cpuhp_is_ap_state(state));
 626
 627         if (st->rollback) {
 628                 struct cpuhp_step *step = cpuhp_get_step(state);
 629                 if (step->skip_onerr)
 630                         goto next;
 631         }
 632
 633         if (cpuhp_is_atomic_state(state)) {
 634                 local_irq_disable();
 635                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 636                 local_irq_enable();
 637
 638                 /*
 639                  * STARTING/DYING must not fail!
 640                  */
 641                 WARN_ON_ONCE(st->result);
 642         } else {
 643                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 644         }
 645
 646         if (st->result) {
 647                 /*
 648                  * If we fail on a rollback, we're up a creek without no
 649                  * paddle, no way forward, no way back. We loose, thanks for
 650                  * playing.
 651                  */
 652                 WARN_ON_ONCE(st->rollback);
 653                 st->should_run = false;
 654         }
 655
 656 next:
 657         cpuhp_lock_release(bringup);
 658
 659         if (!st->should_run)
 660                 complete_ap_thread(st, bringup);
 661 }
 662
 663 /* Invoke a single callback on a remote cpu */
 664 static int
 665 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
 666                          struct hlist_node *node)
 667 {
 668         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 669         int ret;
 670
 671         if (!cpu_online(cpu))
 672                 return 0;
 673
 674         cpuhp_lock_acquire(false);
 675         cpuhp_lock_release(false);
 676
 677         cpuhp_lock_acquire(true);
 678         cpuhp_lock_release(true);
 679
 680         /*
 681          * If we are up and running, use the hotplug thread. For early calls
 682          * we invoke the thread function directly.
 683          */
 684         if (!st->thread)
 685                 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
 686
 687         st->rollback = false;
 688         st->last = NULL;
 689
 690         st->node = node;
 691         st->bringup = bringup;
 692         st->cb_state = state;
 693         st->single = true;
 694
 695         __cpuhp_kick_ap(st);
 696
 697         /*
 698          * If we failed and did a partial, do a rollback.
 699          */
 700         if ((ret = st->result) && st->last) {
 701                 st->rollback = true;
 702                 st->bringup = !bringup;
 703
 704                 __cpuhp_kick_ap(st);
 705         }
 706
 707         /*
 708          * Clean up the leftovers so the next hotplug operation wont use stale
 709          * data.
 710          */
 711         st->node = st->last = NULL;
 712         return ret;
 713 }
 714
 715 static int cpuhp_kick_ap_work(unsigned int cpu)
 716 {
 717         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 718         enum cpuhp_state prev_state = st->state;
 719         int ret;
 720
 721         cpuhp_lock_acquire(false);
 722         cpuhp_lock_release(false);
 723
 724         cpuhp_lock_acquire(true);
 725         cpuhp_lock_release(true);
 726
 727         trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
 728         ret = cpuhp_kick_ap(st, st->target);
 729         trace_cpuhp_exit(cpu, st->state, prev_state, ret);
 730
 731         return ret;
 732 }
 733
 734 static struct smp_hotplug_thread cpuhp_threads = {
 735         .store                  = &cpuhp_state.thread,
 736         .create                 = &cpuhp_create,
 737         .thread_should_run      = cpuhp_should_run,
 738         .thread_fn              = cpuhp_thread_fun,
 739         .thread_comm            = "cpuhp/%u",
 740         .selfparking            = true,
 741 };
 742
 743 void __init cpuhp_threads_init(void)
 744 {
 745         BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
 746         kthread_unpark(this_cpu_read(cpuhp_state.thread));
 747 }
 748
 749 #ifdef CONFIG_HOTPLUG_CPU
 750 /**
 751  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
 752  * @cpu: a CPU id
 753  *
 754  * This function walks all processes, finds a valid mm struct for each one and
 755  * then clears a corresponding bit in mm's cpumask.  While this all sounds
 756  * trivial, there are various non-obvious corner cases, which this function
 757  * tries to solve in a safe manner.
 758  *
 759  * Also note that the function uses a somewhat relaxed locking scheme, so it may
 760  * be called only for an already offlined CPU.
 761  */
 762 void clear_tasks_mm_cpumask(int cpu)
 763 {
 764         struct task_struct *p;
 765
 766         /*
 767          * This function is called after the cpu is taken down and marked
 768          * offline, so its not like new tasks will ever get this cpu set in
 769          * their mm mask. -- Peter Zijlstra
 770          * Thus, we may use rcu_read_lock() here, instead of grabbing
 771          * full-fledged tasklist_lock.
 772          */
 773         WARN_ON(cpu_online(cpu));
 774         rcu_read_lock();
 775         for_each_process(p) {
 776                 struct task_struct *t;
 777
 778                 /*
 779                  * Main thread might exit, but other threads may still have
 780                  * a valid mm. Find one.
 781                  */
 782                 t = find_lock_task_mm(p);
 783                 if (!t)
 784                         continue;
 785                 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
 786                 task_unlock(t);
 787         }
 788         rcu_read_unlock();
 789 }
 790
 791 /* Take this CPU down. */
 792 static int take_cpu_down(void *_param)
 793 {
 794         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 795         enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
 796         int err, cpu = smp_processor_id();
 797         int ret;
 798
 799         /* Ensure this CPU doesn't handle any more interrupts. */
 800         err = __cpu_disable();
 801         if (err < 0)
 802                 return err;
 803
 804         /*
 805          * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
 806          * do this step again.
 807          */
 808         WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
 809         st->state--;
 810         /* Invoke the former CPU_DYING callbacks */
 811         for (; st->state > target; st->state--) {
 812                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 813                 /*
 814                  * DYING must not fail!
 815                  */
 816                 WARN_ON_ONCE(ret);
 817         }
 818
 819         /* Give up timekeeping duties */
 820         tick_handover_do_timer();
 821         /* Park the stopper thread */
 822         stop_machine_park(cpu);
 823         return 0;
 824 }
 825
 826 static int takedown_cpu(unsigned int cpu)
 827 {
 828         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 829         int err;
 830
 831         /* Park the smpboot threads */
 832         kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 833
 834         /*
 835          * Prevent irq alloc/free while the dying cpu reorganizes the
 836          * interrupt affinities.
 837          */
 838         irq_lock_sparse();
 839
 840         /*
 841          * So now all preempt/rcu users must observe !cpu_active().
 842          */
 843         err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
 844         if (err) {
 845                 /* CPU refused to die */
 846                 irq_unlock_sparse();
 847                 /* Unpark the hotplug thread so we can rollback there */
 848                 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 849                 return err;
 850         }
 851         BUG_ON(cpu_online(cpu));
 852
 853         /*
 854          * The CPUHP_AP_SCHED_MIGRATE_DYING callback will have removed all
 855          * runnable tasks from the cpu, there's only the idle task left now
 856          * that the migration thread is done doing the stop_machine thing.
 857          *
 858          * Wait for the stop thread to go away.
 859          */
 860         wait_for_ap_thread(st, false);
 861         BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
 862
 863         /* Interrupts are moved away from the dying cpu, reenable alloc/free */
 864         irq_unlock_sparse();
 865
 866         hotplug_cpu__broadcast_tick_pull(cpu);
 867         /* This actually kills the CPU. */
 868         __cpu_die(cpu);
 869
 870         tick_cleanup_dead_cpu(cpu);
 871         rcutree_migrate_callbacks(cpu);
 872         return 0;
 873 }
 874
 875 static void cpuhp_complete_idle_dead(void *arg)
 876 {
 877         struct cpuhp_cpu_state *st = arg;
 878
 879         complete_ap_thread(st, false);
 880 }
 881
 882 void cpuhp_report_idle_dead(void)
 883 {
 884         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 885
 886         BUG_ON(st->state != CPUHP_AP_OFFLINE);
 887         rcu_report_dead(smp_processor_id());
 888         st->state = CPUHP_AP_IDLE_DEAD;
 889         /*
 890          * We cannot call complete after rcu_report_dead() so we delegate it
 891          * to an online cpu.
 892          */
 893         smp_call_function_single(cpumask_first(cpu_online_mask),
 894                                  cpuhp_complete_idle_dead, st, 0);
 895 }
 896
 897 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
 898 {
 899         for (st->state++; st->state < st->target; st->state++) {
 900                 struct cpuhp_step *step = cpuhp_get_step(st->state);
 901
 902                 if (!step->skip_onerr)
 903                         cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 904         }
 905 }
 906
 907 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 908                                 enum cpuhp_state target)
 909 {
 910         enum cpuhp_state prev_state = st->state;
 911         int ret = 0;
 912
 913         for (; st->state > target; st->state--) {
 914                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 915                 if (ret) {
 916                         st->target = prev_state;
 917                         if (st->state < prev_state)
 918                                 undo_cpu_down(cpu, st);
 919                         break;
 920                 }
 921         }
 922         return ret;
 923 }
 924
 925 /* Requires cpu_add_remove_lock to be held */
 926 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 927                            enum cpuhp_state target)
 928 {
 929         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 930         int prev_state, ret = 0;
 931
 932         if (num_online_cpus() == 1)
 933                 return -EBUSY;
 934
 935         if (!cpu_present(cpu))
 936                 return -EINVAL;
 937
 938         cpus_write_lock();
 939
 940         cpuhp_tasks_frozen = tasks_frozen;
 941
 942         prev_state = cpuhp_set_state(st, target);
 943         /*
 944          * If the current CPU state is in the range of the AP hotplug thread,
 945          * then we need to kick the thread.
 946          */
 947         if (st->state > CPUHP_TEARDOWN_CPU) {
 948                 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
 949                 ret = cpuhp_kick_ap_work(cpu);
 950                 /*
 951                  * The AP side has done the error rollback already. Just
 952                  * return the error code..
 953                  */
 954                 if (ret)
 955                         goto out;
 956
 957                 /*
 958                  * We might have stopped still in the range of the AP hotplug
 959                  * thread. Nothing to do anymore.
 960                  */
 961                 if (st->state > CPUHP_TEARDOWN_CPU)
 962                         goto out;
 963
 964                 st->target = target;
 965         }
 966         /*
 967          * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
 968          * to do the further cleanups.
 969          */
 970         ret = cpuhp_down_callbacks(cpu, st, target);
 971         if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
 972                 cpuhp_reset_state(st, prev_state);
 973                 __cpuhp_kick_ap(st);
 974         }
 975
 976 out:
 977         cpus_write_unlock();
 978         /*
 979          * Do post unplug cleanup. This is still protected against
 980          * concurrent CPU hotplug via cpu_add_remove_lock.
 981          */
 982         lockup_detector_cleanup();
 983         arch_smt_update();
 984         return ret;
 985 }
 986
 987 static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
 988 {
 989         if (cpu_hotplug_disabled)
 990                 return -EBUSY;
 991         return _cpu_down(cpu, 0, target);
 992 }
 993
 994 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
 995 {
 996         int err;
 997
 998         cpu_maps_update_begin();
 999         err = cpu_down_maps_locked(cpu, target);
1000         cpu_maps_update_done();
1001         return err;
1002 }
1003
1004 int cpu_down(unsigned int cpu)
1005 {
1006         return do_cpu_down(cpu, CPUHP_OFFLINE);
1007 }
1008 EXPORT_SYMBOL(cpu_down);
1009
1010 #else
1011 #define takedown_cpu            NULL
1012 #endif /*CONFIG_HOTPLUG_CPU*/
1013
1014 /**
1015  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1016  * @cpu: cpu that just started
1017  *
1018  * It must be called by the arch code on the new cpu, before the new cpu
1019  * enables interrupts and before the "boot" cpu returns from __cpu_up().
1020  */
1021 void notify_cpu_starting(unsigned int cpu)
1022 {
1023         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1024         enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1025         int ret;
1026
1027         rcu_cpu_starting(cpu);  /* Enables RCU usage on this CPU. */
1028         st->booted_once = true;
1029         while (st->state < target) {
1030                 st->state++;
1031                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1032                 /*
1033                  * STARTING must not fail!
1034                  */
1035                 WARN_ON_ONCE(ret);
1036         }
1037 }
1038
1039 /*
1040  * Called from the idle task. Wake up the controlling task which brings the
1041  * stopper and the hotplug thread of the upcoming CPU up and then delegates
1042  * the rest of the online bringup to the hotplug thread.
1043  */
1044 void cpuhp_online_idle(enum cpuhp_state state)
1045 {
1046         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1047
1048         /* Happens for the boot cpu */
1049         if (state != CPUHP_AP_ONLINE_IDLE)
1050                 return;
1051
1052         st->state = CPUHP_AP_ONLINE_IDLE;
1053         complete_ap_thread(st, true);
1054 }
1055
1056 /* Requires cpu_add_remove_lock to be held */
1057 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1058 {
1059         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1060         struct task_struct *idle;
1061         int ret = 0;
1062
1063         cpus_write_lock();
1064
1065         if (!cpu_present(cpu)) {
1066                 ret = -EINVAL;
1067                 goto out;
1068         }
1069
1070         /*
1071          * The caller of do_cpu_up might have raced with another
1072          * caller. Ignore it for now.
1073          */
1074         if (st->state >= target)
1075                 goto out;
1076
1077         if (st->state == CPUHP_OFFLINE) {
1078                 /* Let it fail before we try to bring the cpu up */
1079                 idle = idle_thread_get(cpu);
1080                 if (IS_ERR(idle)) {
1081                         ret = PTR_ERR(idle);
1082                         goto out;
1083                 }
1084         }
1085
1086         cpuhp_tasks_frozen = tasks_frozen;
1087
1088         cpuhp_set_state(st, target);
1089         /*
1090          * If the current CPU state is in the range of the AP hotplug thread,
1091          * then we need to kick the thread once more.
1092          */
1093         if (st->state > CPUHP_BRINGUP_CPU) {
1094                 ret = cpuhp_kick_ap_work(cpu);
1095                 /*
1096                  * The AP side has done the error rollback already. Just
1097                  * return the error code..
1098                  */
1099                 if (ret)
1100                         goto out;
1101         }
1102
1103         /*
1104          * Try to reach the target state. We max out on the BP at
1105          * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1106          * responsible for bringing it up to the target state.
1107          */
1108         target = min((int)target, CPUHP_BRINGUP_CPU);
1109         ret = cpuhp_up_callbacks(cpu, st, target);
1110 out:
1111         cpus_write_unlock();
1112         arch_smt_update();
1113         return ret;
1114 }
1115
1116 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
1117 {
1118         int err = 0;
1119
1120         if (!cpu_possible(cpu)) {
1121                 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1122                        cpu);
1123 #if defined(CONFIG_IA64)
1124                 pr_err("please check additional_cpus= boot parameter\n");
1125 #endif
1126                 return -EINVAL;
1127         }
1128
1129         err = try_online_node(cpu_to_node(cpu));
1130         if (err)
1131                 return err;
1132
1133         cpu_maps_update_begin();
1134
1135         if (cpu_hotplug_disabled) {
1136                 err = -EBUSY;
1137                 goto out;
1138         }
1139         if (!cpu_smt_allowed(cpu)) {
1140                 err = -EPERM;
1141                 goto out;
1142         }
1143
1144         err = _cpu_up(cpu, 0, target);
1145 out:
1146         cpu_maps_update_done();
1147         return err;
1148 }
1149
1150 int cpu_up(unsigned int cpu)
1151 {
1152         return do_cpu_up(cpu, CPUHP_ONLINE);
1153 }
1154 EXPORT_SYMBOL_GPL(cpu_up);
1155
1156 #ifdef CONFIG_PM_SLEEP_SMP
1157 static cpumask_var_t frozen_cpus;
1158
1159 int freeze_secondary_cpus(int primary)
1160 {
1161         int cpu, error = 0;
1162
1163         cpu_maps_update_begin();
1164         if (!cpu_online(primary))
1165                 primary = cpumask_first(cpu_online_mask);
1166         /*
1167          * We take down all of the non-boot CPUs in one shot to avoid races
1168          * with the userspace trying to use the CPU hotplug at the same time
1169          */
1170         cpumask_clear(frozen_cpus);
1171
1172         pr_info("Disabling non-boot CPUs ...\n");
1173         for_each_online_cpu(cpu) {
1174                 if (cpu == primary)
1175                         continue;
1176                 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1177                 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1178                 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1179                 if (!error)
1180                         cpumask_set_cpu(cpu, frozen_cpus);
1181                 else {
1182                         pr_err("Error taking CPU%d down: %d\n", cpu, error);
1183                         break;
1184                 }
1185         }
1186
1187         if (!error)
1188                 BUG_ON(num_online_cpus() > 1);
1189         else
1190                 pr_err("Non-boot CPUs are not disabled\n");
1191
1192         /*
1193          * Make sure the CPUs won't be enabled by someone else. We need to do
1194          * this even in case of failure as all disable_nonboot_cpus() users are
1195          * supposed to do enable_nonboot_cpus() on the failure path.
1196          */
1197         cpu_hotplug_disabled++;
1198
1199         cpu_maps_update_done();
1200         return error;
1201 }
1202
1203 void __weak arch_enable_nonboot_cpus_begin(void)
1204 {
1205 }
1206
1207 void __weak arch_enable_nonboot_cpus_end(void)
1208 {
1209 }
1210
1211 void enable_nonboot_cpus(void)
1212 {
1213         int cpu, error;
1214
1215         /* Allow everyone to use the CPU hotplug again */
1216         cpu_maps_update_begin();
1217         __cpu_hotplug_enable();
1218         if (cpumask_empty(frozen_cpus))
1219                 goto out;
1220
1221         pr_info("Enabling non-boot CPUs ...\n");
1222
1223         arch_enable_nonboot_cpus_begin();
1224
1225         for_each_cpu(cpu, frozen_cpus) {
1226                 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1227                 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1228                 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1229                 if (!error) {
1230                         pr_info("CPU%d is up\n", cpu);
1231                         continue;
1232                 }
1233                 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1234         }
1235
1236         arch_enable_nonboot_cpus_end();
1237
1238         cpumask_clear(frozen_cpus);
1239 out:
1240         cpu_maps_update_done();
1241 }
1242
1243 static int __init alloc_frozen_cpus(void)
1244 {
1245         if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1246                 return -ENOMEM;
1247         return 0;
1248 }
1249 core_initcall(alloc_frozen_cpus);
1250
1251 /*
1252  * When callbacks for CPU hotplug notifications are being executed, we must
1253  * ensure that the state of the system with respect to the tasks being frozen
1254  * or not, as reported by the notification, remains unchanged *throughout the
1255  * duration* of the execution of the callbacks.
1256  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1257  *
1258  * This synchronization is implemented by mutually excluding regular CPU
1259  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1260  * Hibernate notifications.
1261  */
1262 static int
1263 cpu_hotplug_pm_callback(struct notifier_block *nb,
1264                         unsigned long action, void *ptr)
1265 {
1266         switch (action) {
1267
1268         case PM_SUSPEND_PREPARE:
1269         case PM_HIBERNATION_PREPARE:
1270                 cpu_hotplug_disable();
1271                 break;
1272
1273         case PM_POST_SUSPEND:
1274         case PM_POST_HIBERNATION:
1275                 cpu_hotplug_enable();
1276                 break;
1277
1278         default:
1279                 return NOTIFY_DONE;
1280         }
1281
1282         return NOTIFY_OK;
1283 }
1284
1285
1286 static int __init cpu_hotplug_pm_sync_init(void)
1287 {
1288         /*
1289          * cpu_hotplug_pm_callback has higher priority than x86
1290          * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1291          * to disable cpu hotplug to avoid cpu hotplug race.
1292          */
1293         pm_notifier(cpu_hotplug_pm_callback, 0);
1294         return 0;
1295 }
1296 core_initcall(cpu_hotplug_pm_sync_init);
1297
1298 #endif /* CONFIG_PM_SLEEP_SMP */
1299
1300 int __boot_cpu_id;
1301
1302 #endif /* CONFIG_SMP */
1303
1304 /* Boot processor state steps */
1305 static struct cpuhp_step cpuhp_bp_states[] = {
1306         [CPUHP_OFFLINE] = {
1307                 .name                   = "offline",
1308                 .startup.single         = NULL,
1309                 .teardown.single        = NULL,
1310         },
1311 #ifdef CONFIG_SMP
1312         [CPUHP_CREATE_THREADS]= {
1313                 .name                   = "threads:prepare",
1314                 .startup.single         = smpboot_create_threads,
1315                 .teardown.single        = NULL,
1316                 .cant_stop              = true,
1317         },
1318         [CPUHP_PERF_PREPARE] = {
1319                 .name                   = "perf:prepare",
1320                 .startup.single         = perf_event_init_cpu,
1321                 .teardown.single        = perf_event_exit_cpu,
1322         },
1323         [CPUHP_WORKQUEUE_PREP] = {
1324                 .name                   = "workqueue:prepare",
1325                 .startup.single         = workqueue_prepare_cpu,
1326                 .teardown.single        = NULL,
1327         },
1328         [CPUHP_HRTIMERS_PREPARE] = {
1329                 .name                   = "hrtimers:prepare",
1330                 .startup.single         = hrtimers_prepare_cpu,
1331                 .teardown.single        = hrtimers_dead_cpu,
1332         },
1333         [CPUHP_SMPCFD_PREPARE] = {
1334                 .name                   = "smpcfd:prepare",
1335                 .startup.single         = smpcfd_prepare_cpu,
1336                 .teardown.single        = smpcfd_dead_cpu,
1337         },
1338         [CPUHP_RELAY_PREPARE] = {
1339                 .name                   = "relay:prepare",
1340                 .startup.single         = relay_prepare_cpu,
1341                 .teardown.single        = NULL,
1342         },
1343         [CPUHP_SLAB_PREPARE] = {
1344                 .name                   = "slab:prepare",
1345                 .startup.single         = slab_prepare_cpu,
1346                 .teardown.single        = slab_dead_cpu,
1347         },
1348         [CPUHP_RCUTREE_PREP] = {
1349                 .name                   = "RCU/tree:prepare",
1350                 .startup.single         = rcutree_prepare_cpu,
1351                 .teardown.single        = rcutree_dead_cpu,
1352         },
1353         /*
1354          * On the tear-down path, timers_dead_cpu() must be invoked
1355          * before blk_mq_queue_reinit_notify() from notify_dead(),
1356          * otherwise a RCU stall occurs.
1357          */
1358         [CPUHP_TIMERS_PREPARE] = {
1359                 .name                   = "timers:dead",
1360                 .startup.single         = timers_prepare_cpu,
1361                 .teardown.single        = timers_dead_cpu,
1362         },
1363         /* Kicks the plugged cpu into life */
1364         [CPUHP_BRINGUP_CPU] = {
1365                 .name                   = "cpu:bringup",
1366                 .startup.single         = bringup_cpu,
1367                 .teardown.single        = NULL,
1368                 .cant_stop              = true,
1369         },
1370         /*
1371          * Handled on controll processor until the plugged processor manages
1372          * this itself.
1373          */
1374         [CPUHP_TEARDOWN_CPU] = {
1375                 .name                   = "cpu:teardown",
1376                 .startup.single         = NULL,
1377                 .teardown.single        = takedown_cpu,
1378                 .cant_stop              = true,
1379         },
1380 #else
1381         [CPUHP_BRINGUP_CPU] = { },
1382 #endif
1383 };
1384
1385 /* Application processor state steps */
1386 static struct cpuhp_step cpuhp_ap_states[] = {
1387 #ifdef CONFIG_SMP
1388         /* Final state before CPU kills itself */
1389         [CPUHP_AP_IDLE_DEAD] = {
1390                 .name                   = "idle:dead",
1391         },
1392         /*
1393          * Last state before CPU enters the idle loop to die. Transient state
1394          * for synchronization.
1395          */
1396         [CPUHP_AP_OFFLINE] = {
1397                 .name                   = "ap:offline",
1398                 .cant_stop              = true,
1399         },
1400         /* First state is scheduler control. Interrupts are disabled */
1401         [CPUHP_AP_SCHED_STARTING] = {
1402                 .name                   = "sched:starting",
1403                 .startup.single         = sched_cpu_starting,
1404                 .teardown.single        = sched_cpu_dying,
1405         },
1406         [CPUHP_AP_RCUTREE_DYING] = {
1407                 .name                   = "RCU/tree:dying",
1408                 .startup.single         = NULL,
1409                 .teardown.single        = rcutree_dying_cpu,
1410         },
1411         [CPUHP_AP_SMPCFD_DYING] = {
1412                 .name                   = "smpcfd:dying",
1413                 .startup.single         = NULL,
1414                 .teardown.single        = smpcfd_dying_cpu,
1415         },
1416         /* Entry state on starting. Interrupts enabled from here on. Transient
1417          * state for synchronsization */
1418         [CPUHP_AP_ONLINE] = {
1419                 .name                   = "ap:online",
1420         },
1421         /* Handle smpboot threads park/unpark */
1422         [CPUHP_AP_SMPBOOT_THREADS] = {
1423                 .name                   = "smpboot/threads:online",
1424                 .startup.single         = smpboot_unpark_threads,
1425                 .teardown.single        = smpboot_park_threads,
1426         },
1427         [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1428                 .name                   = "irq/affinity:online",
1429                 .startup.single         = irq_affinity_online_cpu,
1430                 .teardown.single        = NULL,
1431         },
1432         [CPUHP_AP_PERF_ONLINE] = {
1433                 .name                   = "perf:online",
1434                 .startup.single         = perf_event_init_cpu,
1435                 .teardown.single        = perf_event_exit_cpu,
1436         },
1437         [CPUHP_AP_WORKQUEUE_ONLINE] = {
1438                 .name                   = "workqueue:online",
1439                 .startup.single         = workqueue_online_cpu,
1440                 .teardown.single        = workqueue_offline_cpu,
1441         },
1442         [CPUHP_AP_RCUTREE_ONLINE] = {
1443                 .name                   = "RCU/tree:online",
1444                 .startup.single         = rcutree_online_cpu,
1445                 .teardown.single        = rcutree_offline_cpu,
1446         },
1447 #endif
1448         /*
1449          * The dynamically registered state space is here
1450          */
1451
1452 #ifdef CONFIG_SMP
1453         /* Last state is scheduler control setting the cpu active */
1454         [CPUHP_AP_ACTIVE] = {
1455                 .name                   = "sched:active",
1456                 .startup.single         = sched_cpu_activate,
1457                 .teardown.single        = sched_cpu_deactivate,
1458         },
1459 #endif
1460
1461         /* CPU is fully up and running. */
1462         [CPUHP_ONLINE] = {
1463                 .name                   = "online",
1464                 .startup.single         = NULL,
1465                 .teardown.single        = NULL,
1466         },
1467 };
1468
1469 /* Sanity check for callbacks */
1470 static int cpuhp_cb_check(enum cpuhp_state state)
1471 {
1472         if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1473                 return -EINVAL;
1474         return 0;
1475 }
1476
1477 /*
1478  * Returns a free for dynamic slot assignment of the Online state. The states
1479  * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1480  * by having no name assigned.
1481  */
1482 static int cpuhp_reserve_state(enum cpuhp_state state)
1483 {
1484         enum cpuhp_state i, end;
1485         struct cpuhp_step *step;
1486
1487         switch (state) {
1488         case CPUHP_AP_ONLINE_DYN:
1489                 step = cpuhp_ap_states + CPUHP_AP_ONLINE_DYN;
1490                 end = CPUHP_AP_ONLINE_DYN_END;
1491                 break;
1492         case CPUHP_BP_PREPARE_DYN:
1493                 step = cpuhp_bp_states + CPUHP_BP_PREPARE_DYN;
1494                 end = CPUHP_BP_PREPARE_DYN_END;
1495                 break;
1496         default:
1497                 return -EINVAL;
1498         }
1499
1500         for (i = state; i <= end; i++, step++) {
1501                 if (!step->name)
1502                         return i;
1503         }
1504         WARN(1, "No more dynamic states available for CPU hotplug\n");
1505         return -ENOSPC;
1506 }
1507
1508 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1509                                  int (*startup)(unsigned int cpu),
1510                                  int (*teardown)(unsigned int cpu),
1511                                  bool multi_instance)
1512 {
1513         /* (Un)Install the callbacks for further cpu hotplug operations */
1514         struct cpuhp_step *sp;
1515         int ret = 0;
1516
1517         /*
1518          * If name is NULL, then the state gets removed.
1519          *
1520          * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1521          * the first allocation from these dynamic ranges, so the removal
1522          * would trigger a new allocation and clear the wrong (already
1523          * empty) state, leaving the callbacks of the to be cleared state
1524          * dangling, which causes wreckage on the next hotplug operation.
1525          */
1526         if (name && (state == CPUHP_AP_ONLINE_DYN ||
1527                      state == CPUHP_BP_PREPARE_DYN)) {
1528                 ret = cpuhp_reserve_state(state);
1529                 if (ret < 0)
1530                         return ret;
1531                 state = ret;
1532         }
1533         sp = cpuhp_get_step(state);
1534         if (name && sp->name)
1535                 return -EBUSY;
1536
1537         sp->startup.single = startup;
1538         sp->teardown.single = teardown;
1539         sp->name = name;
1540         sp->multi_instance = multi_instance;
1541         INIT_HLIST_HEAD(&sp->list);
1542         return ret;
1543 }
1544
1545 static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1546 {
1547         return cpuhp_get_step(state)->teardown.single;
1548 }
1549
1550 /*
1551  * Call the startup/teardown function for a step either on the AP or
1552  * on the current CPU.
1553  */
1554 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1555                             struct hlist_node *node)
1556 {
1557         struct cpuhp_step *sp = cpuhp_get_step(state);
1558         int ret;
1559
1560         /*
1561          * If there's nothing to do, we done.
1562          * Relies on the union for multi_instance.
1563          */
1564         if ((bringup && !sp->startup.single) ||
1565             (!bringup && !sp->teardown.single))
1566                 return 0;
1567         /*
1568          * The non AP bound callbacks can fail on bringup. On teardown
1569          * e.g. module removal we crash for now.
1570          */
1571 #ifdef CONFIG_SMP
1572         if (cpuhp_is_ap_state(state))
1573                 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1574         else
1575                 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1576 #else
1577         ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1578 #endif
1579         BUG_ON(ret && !bringup);
1580         return ret;
1581 }
1582
1583 /*
1584  * Called from __cpuhp_setup_state on a recoverable failure.
1585  *
1586  * Note: The teardown callbacks for rollback are not allowed to fail!
1587  */
1588 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1589                                    struct hlist_node *node)
1590 {
1591         int cpu;
1592
1593         /* Roll back the already executed steps on the other cpus */
1594         for_each_present_cpu(cpu) {
1595                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1596                 int cpustate = st->state;
1597
1598                 if (cpu >= failedcpu)
1599                         break;
1600
1601                 /* Did we invoke the startup call on that cpu ? */
1602                 if (cpustate >= state)
1603                         cpuhp_issue_call(cpu, state, false, node);
1604         }
1605 }
1606
1607 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1608                                           struct hlist_node *node,
1609                                           bool invoke)
1610 {
1611         struct cpuhp_step *sp;
1612         int cpu;
1613         int ret;
1614
1615         lockdep_assert_cpus_held();
1616
1617         sp = cpuhp_get_step(state);
1618         if (sp->multi_instance == false)
1619                 return -EINVAL;
1620
1621         mutex_lock(&cpuhp_state_mutex);
1622
1623         if (!invoke || !sp->startup.multi)
1624                 goto add_node;
1625
1626         /*
1627          * Try to call the startup callback for each present cpu
1628          * depending on the hotplug state of the cpu.
1629          */
1630         for_each_present_cpu(cpu) {
1631                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1632                 int cpustate = st->state;
1633
1634                 if (cpustate < state)
1635                         continue;
1636
1637                 ret = cpuhp_issue_call(cpu, state, true, node);
1638                 if (ret) {
1639                         if (sp->teardown.multi)
1640                                 cpuhp_rollback_install(cpu, state, node);
1641                         goto unlock;
1642                 }
1643         }
1644 add_node:
1645         ret = 0;
1646         hlist_add_head(node, &sp->list);
1647 unlock:
1648         mutex_unlock(&cpuhp_state_mutex);
1649         return ret;
1650 }
1651
1652 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1653                                bool invoke)
1654 {
1655         int ret;
1656
1657         cpus_read_lock();
1658         ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1659         cpus_read_unlock();
1660         return ret;
1661 }
1662 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1663
1664 /**
1665  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1666  * @state:              The state to setup
1667  * @invoke:             If true, the startup function is invoked for cpus where
1668  *                      cpu state >= @state
1669  * @startup:            startup callback function
1670  * @teardown:           teardown callback function
1671  * @multi_instance:     State is set up for multiple instances which get
1672  *                      added afterwards.
1673  *
1674  * The caller needs to hold cpus read locked while calling this function.
1675  * Returns:
1676  *   On success:
1677  *      Positive state number if @state is CPUHP_AP_ONLINE_DYN
1678  *      0 for all other states
1679  *   On failure: proper (negative) error code
1680  */
1681 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1682                                    const char *name, bool invoke,
1683                                    int (*startup)(unsigned int cpu),
1684                                    int (*teardown)(unsigned int cpu),
1685                                    bool multi_instance)
1686 {
1687         int cpu, ret = 0;
1688         bool dynstate;
1689
1690         lockdep_assert_cpus_held();
1691
1692         if (cpuhp_cb_check(state) || !name)
1693                 return -EINVAL;
1694
1695         mutex_lock(&cpuhp_state_mutex);
1696
1697         ret = cpuhp_store_callbacks(state, name, startup, teardown,
1698                                     multi_instance);
1699
1700         dynstate = state == CPUHP_AP_ONLINE_DYN;
1701         if (ret > 0 && dynstate) {
1702                 state = ret;
1703                 ret = 0;
1704         }
1705
1706         if (ret || !invoke || !startup)
1707                 goto out;
1708
1709         /*
1710          * Try to call the startup callback for each present cpu
1711          * depending on the hotplug state of the cpu.
1712          */
1713         for_each_present_cpu(cpu) {
1714                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1715                 int cpustate = st->state;
1716
1717                 if (cpustate < state)
1718                         continue;
1719
1720                 ret = cpuhp_issue_call(cpu, state, true, NULL);
1721                 if (ret) {
1722                         if (teardown)
1723                                 cpuhp_rollback_install(cpu, state, NULL);
1724                         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1725                         goto out;
1726                 }
1727         }
1728 out:
1729         mutex_unlock(&cpuhp_state_mutex);
1730         /*
1731          * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1732          * dynamically allocated state in case of success.
1733          */
1734         if (!ret && dynstate)
1735                 return state;
1736         return ret;
1737 }
1738 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1739
1740 int __cpuhp_setup_state(enum cpuhp_state state,
1741                         const char *name, bool invoke,
1742                         int (*startup)(unsigned int cpu),
1743                         int (*teardown)(unsigned int cpu),
1744                         bool multi_instance)
1745 {
1746         int ret;
1747
1748         cpus_read_lock();
1749         ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1750                                              teardown, multi_instance);
1751         cpus_read_unlock();
1752         return ret;
1753 }
1754 EXPORT_SYMBOL(__cpuhp_setup_state);
1755
1756 int __cpuhp_state_remove_instance(enum cpuhp_state state,
1757                                   struct hlist_node *node, bool invoke)
1758 {
1759         struct cpuhp_step *sp = cpuhp_get_step(state);
1760         int cpu;
1761
1762         BUG_ON(cpuhp_cb_check(state));
1763
1764         if (!sp->multi_instance)
1765                 return -EINVAL;
1766
1767         cpus_read_lock();
1768         mutex_lock(&cpuhp_state_mutex);
1769
1770         if (!invoke || !cpuhp_get_teardown_cb(state))
1771                 goto remove;
1772         /*
1773          * Call the teardown callback for each present cpu depending
1774          * on the hotplug state of the cpu. This function is not
1775          * allowed to fail currently!
1776          */
1777         for_each_present_cpu(cpu) {
1778                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1779                 int cpustate = st->state;
1780
1781                 if (cpustate >= state)
1782                         cpuhp_issue_call(cpu, state, false, node);
1783         }
1784
1785 remove:
1786         hlist_del(node);
1787         mutex_unlock(&cpuhp_state_mutex);
1788         cpus_read_unlock();
1789
1790         return 0;
1791 }
1792 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1793
1794 /**
1795  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1796  * @state:      The state to remove
1797  * @invoke:     If true, the teardown function is invoked for cpus where
1798  *              cpu state >= @state
1799  *
1800  * The caller needs to hold cpus read locked while calling this function.
1801  * The teardown callback is currently not allowed to fail. Think
1802  * about module removal!
1803  */
1804 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
1805 {
1806         struct cpuhp_step *sp = cpuhp_get_step(state);
1807         int cpu;
1808
1809         BUG_ON(cpuhp_cb_check(state));
1810
1811         lockdep_assert_cpus_held();
1812
1813         mutex_lock(&cpuhp_state_mutex);
1814         if (sp->multi_instance) {
1815                 WARN(!hlist_empty(&sp->list),
1816                      "Error: Removing state %d which has instances left.\n",
1817                      state);
1818                 goto remove;
1819         }
1820
1821         if (!invoke || !cpuhp_get_teardown_cb(state))
1822                 goto remove;
1823
1824         /*
1825          * Call the teardown callback for each present cpu depending
1826          * on the hotplug state of the cpu. This function is not
1827          * allowed to fail currently!
1828          */
1829         for_each_present_cpu(cpu) {
1830                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1831                 int cpustate = st->state;
1832
1833                 if (cpustate >= state)
1834                         cpuhp_issue_call(cpu, state, false, NULL);
1835         }
1836 remove:
1837         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1838         mutex_unlock(&cpuhp_state_mutex);
1839 }
1840 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
1841
1842 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1843 {
1844         cpus_read_lock();
1845         __cpuhp_remove_state_cpuslocked(state, invoke);
1846         cpus_read_unlock();
1847 }
1848 EXPORT_SYMBOL(__cpuhp_remove_state);
1849
1850 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1851 static ssize_t show_cpuhp_state(struct device *dev,
1852                                 struct device_attribute *attr, char *buf)
1853 {
1854         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1855
1856         return sprintf(buf, "%d\n", st->state);
1857 }
1858 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1859
1860 static ssize_t write_cpuhp_target(struct device *dev,
1861                                   struct device_attribute *attr,
1862                                   const char *buf, size_t count)
1863 {
1864         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1865         struct cpuhp_step *sp;
1866         int target, ret;
1867
1868         ret = kstrtoint(buf, 10, &target);
1869         if (ret)
1870                 return ret;
1871
1872 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1873         if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1874                 return -EINVAL;
1875 #else
1876         if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1877                 return -EINVAL;
1878 #endif
1879
1880         ret = lock_device_hotplug_sysfs();
1881         if (ret)
1882                 return ret;
1883
1884         mutex_lock(&cpuhp_state_mutex);
1885         sp = cpuhp_get_step(target);
1886         ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1887         mutex_unlock(&cpuhp_state_mutex);
1888         if (ret)
1889                 goto out;
1890
1891         if (st->state < target)
1892                 ret = do_cpu_up(dev->id, target);
1893         else
1894                 ret = do_cpu_down(dev->id, target);
1895 out:
1896         unlock_device_hotplug();
1897         return ret ? ret : count;
1898 }
1899
1900 static ssize_t show_cpuhp_target(struct device *dev,
1901                                  struct device_attribute *attr, char *buf)
1902 {
1903         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1904
1905         return sprintf(buf, "%d\n", st->target);
1906 }
1907 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
1908
1909
1910 static ssize_t write_cpuhp_fail(struct device *dev,
1911                                 struct device_attribute *attr,
1912                                 const char *buf, size_t count)
1913 {
1914         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1915         struct cpuhp_step *sp;
1916         int fail, ret;
1917
1918         ret = kstrtoint(buf, 10, &fail);
1919         if (ret)
1920                 return ret;
1921
1922         /*
1923          * Cannot fail STARTING/DYING callbacks.
1924          */
1925         if (cpuhp_is_atomic_state(fail))
1926                 return -EINVAL;
1927
1928         /*
1929          * Cannot fail anything that doesn't have callbacks.
1930          */
1931         mutex_lock(&cpuhp_state_mutex);
1932         sp = cpuhp_get_step(fail);
1933         if (!sp->startup.single && !sp->teardown.single)
1934                 ret = -EINVAL;
1935         mutex_unlock(&cpuhp_state_mutex);
1936         if (ret)
1937                 return ret;
1938
1939         st->fail = fail;
1940
1941         return count;
1942 }
1943
1944 static ssize_t show_cpuhp_fail(struct device *dev,
1945                                struct device_attribute *attr, char *buf)
1946 {
1947         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1948
1949         return sprintf(buf, "%d\n", st->fail);
1950 }
1951
1952 static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
1953
1954 static struct attribute *cpuhp_cpu_attrs[] = {
1955         &dev_attr_state.attr,
1956         &dev_attr_target.attr,
1957         &dev_attr_fail.attr,
1958         NULL
1959 };
1960
1961 static const struct attribute_group cpuhp_cpu_attr_group = {
1962         .attrs = cpuhp_cpu_attrs,
1963         .name = "hotplug",
1964         NULL
1965 };
1966
1967 static ssize_t show_cpuhp_states(struct device *dev,
1968                                  struct device_attribute *attr, char *buf)
1969 {
1970         ssize_t cur, res = 0;
1971         int i;
1972
1973         mutex_lock(&cpuhp_state_mutex);
1974         for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
1975                 struct cpuhp_step *sp = cpuhp_get_step(i);
1976
1977                 if (sp->name) {
1978                         cur = sprintf(buf, "%3d: %s\n", i, sp->name);
1979                         buf += cur;
1980                         res += cur;
1981                 }
1982         }
1983         mutex_unlock(&cpuhp_state_mutex);
1984         return res;
1985 }
1986 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
1987
1988 static struct attribute *cpuhp_cpu_root_attrs[] = {
1989         &dev_attr_states.attr,
1990         NULL
1991 };
1992
1993 static const struct attribute_group cpuhp_cpu_root_attr_group = {
1994         .attrs = cpuhp_cpu_root_attrs,
1995         .name = "hotplug",
1996         NULL
1997 };
1998
1999 #ifdef CONFIG_HOTPLUG_SMT
2000
2001 static const char *smt_states[] = {
2002         [CPU_SMT_ENABLED]               = "on",
2003         [CPU_SMT_DISABLED]              = "off",
2004         [CPU_SMT_FORCE_DISABLED]        = "forceoff",
2005         [CPU_SMT_NOT_SUPPORTED]         = "notsupported",
2006 };
2007
2008 static ssize_t
2009 show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
2010 {
2011         return snprintf(buf, PAGE_SIZE - 2, "%s\n", smt_states[cpu_smt_control]);
2012 }
2013
2014 static void cpuhp_offline_cpu_device(unsigned int cpu)
2015 {
2016         struct device *dev = get_cpu_device(cpu);
2017
2018         dev->offline = true;
2019         /* Tell user space about the state change */
2020         kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2021 }
2022
2023 static void cpuhp_online_cpu_device(unsigned int cpu)
2024 {
2025         struct device *dev = get_cpu_device(cpu);
2026
2027         dev->offline = false;
2028         /* Tell user space about the state change */
2029         kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2030 }
2031
2032 static int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2033 {
2034         int cpu, ret = 0;
2035
2036         cpu_maps_update_begin();
2037         for_each_online_cpu(cpu) {
2038                 if (topology_is_primary_thread(cpu))
2039                         continue;
2040                 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2041                 if (ret)
2042                         break;
2043                 /*
2044                  * As this needs to hold the cpu maps lock it's impossible
2045                  * to call device_offline() because that ends up calling
2046                  * cpu_down() which takes cpu maps lock. cpu maps lock
2047                  * needs to be held as this might race against in kernel
2048                  * abusers of the hotplug machinery (thermal management).
2049                  *
2050                  * So nothing would update device:offline state. That would
2051                  * leave the sysfs entry stale and prevent onlining after
2052                  * smt control has been changed to 'off' again. This is
2053                  * called under the sysfs hotplug lock, so it is properly
2054                  * serialized against the regular offline usage.
2055                  */
2056                 cpuhp_offline_cpu_device(cpu);
2057         }
2058         if (!ret) {
2059                 cpu_smt_control = ctrlval;
2060                 arch_smt_update();
2061         }
2062         cpu_maps_update_done();
2063         return ret;
2064 }
2065
2066 static int cpuhp_smt_enable(void)
2067 {
2068         int cpu, ret = 0;
2069
2070         cpu_maps_update_begin();
2071         cpu_smt_control = CPU_SMT_ENABLED;
2072         arch_smt_update();
2073         for_each_present_cpu(cpu) {
2074                 /* Skip online CPUs and CPUs on offline nodes */
2075                 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2076                         continue;
2077                 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2078                 if (ret)
2079                         break;
2080                 /* See comment in cpuhp_smt_disable() */
2081                 cpuhp_online_cpu_device(cpu);
2082         }
2083         cpu_maps_update_done();
2084         return ret;
2085 }
2086
2087 static ssize_t
2088 store_smt_control(struct device *dev, struct device_attribute *attr,
2089                   const char *buf, size_t count)
2090 {
2091         int ctrlval, ret;
2092
2093         if (sysfs_streq(buf, "on"))
2094                 ctrlval = CPU_SMT_ENABLED;
2095         else if (sysfs_streq(buf, "off"))
2096                 ctrlval = CPU_SMT_DISABLED;
2097         else if (sysfs_streq(buf, "forceoff"))
2098                 ctrlval = CPU_SMT_FORCE_DISABLED;
2099         else
2100                 return -EINVAL;
2101
2102         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2103                 return -EPERM;
2104
2105         if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2106                 return -ENODEV;
2107
2108         ret = lock_device_hotplug_sysfs();
2109         if (ret)
2110                 return ret;
2111
2112         if (ctrlval != cpu_smt_control) {
2113                 switch (ctrlval) {
2114                 case CPU_SMT_ENABLED:
2115                         ret = cpuhp_smt_enable();
2116                         break;
2117                 case CPU_SMT_DISABLED:
2118                 case CPU_SMT_FORCE_DISABLED:
2119                         ret = cpuhp_smt_disable(ctrlval);
2120                         break;
2121                 }
2122         }
2123
2124         unlock_device_hotplug();
2125         return ret ? ret : count;
2126 }
2127 static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2128
2129 static ssize_t
2130 show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2131 {
2132         bool active = topology_max_smt_threads() > 1;
2133
2134         return snprintf(buf, PAGE_SIZE - 2, "%d\n", active);
2135 }
2136 static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2137
2138 static struct attribute *cpuhp_smt_attrs[] = {
2139         &dev_attr_control.attr,
2140         &dev_attr_active.attr,
2141         NULL
2142 };
2143
2144 static const struct attribute_group cpuhp_smt_attr_group = {
2145         .attrs = cpuhp_smt_attrs,
2146         .name = "smt",
2147         NULL
2148 };
2149
2150 static int __init cpu_smt_state_init(void)
2151 {
2152         return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2153                                   &cpuhp_smt_attr_group);
2154 }
2155
2156 #else
2157 static inline int cpu_smt_state_init(void) { return 0; }
2158 #endif
2159
2160 static int __init cpuhp_sysfs_init(void)
2161 {
2162         int cpu, ret;
2163
2164         ret = cpu_smt_state_init();
2165         if (ret)
2166                 return ret;
2167
2168         ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2169                                  &cpuhp_cpu_root_attr_group);
2170         if (ret)
2171                 return ret;
2172
2173         for_each_possible_cpu(cpu) {
2174                 struct device *dev = get_cpu_device(cpu);
2175
2176                 if (!dev)
2177                         continue;
2178                 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2179                 if (ret)
2180                         return ret;
2181         }
2182         return 0;
2183 }
2184 device_initcall(cpuhp_sysfs_init);
2185 #endif
2186
2187 /*
2188  * cpu_bit_bitmap[] is a special, "compressed" data structure that
2189  * represents all NR_CPUS bits binary values of 1<<nr.
2190  *
2191  * It is used by cpumask_of() to get a constant address to a CPU
2192  * mask value that has a single bit set only.
2193  */
2194
2195 /* cpu_bit_bitmap[0] is empty - so we can back into it */
2196 #define MASK_DECLARE_1(x)       [x+1][0] = (1UL << (x))
2197 #define MASK_DECLARE_2(x)       MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2198 #define MASK_DECLARE_4(x)       MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2199 #define MASK_DECLARE_8(x)       MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2200
2201 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2202
2203         MASK_DECLARE_8(0),      MASK_DECLARE_8(8),
2204         MASK_DECLARE_8(16),     MASK_DECLARE_8(24),
2205 #if BITS_PER_LONG > 32
2206         MASK_DECLARE_8(32),     MASK_DECLARE_8(40),
2207         MASK_DECLARE_8(48),     MASK_DECLARE_8(56),
2208 #endif
2209 };
2210 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2211
2212 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2213 EXPORT_SYMBOL(cpu_all_bits);
2214
2215 #ifdef CONFIG_INIT_ALL_POSSIBLE
2216 struct cpumask __cpu_possible_mask __read_mostly
2217         = {CPU_BITS_ALL};
2218 #else
2219 struct cpumask __cpu_possible_mask __read_mostly;
2220 #endif
2221 EXPORT_SYMBOL(__cpu_possible_mask);
2222
2223 struct cpumask __cpu_online_mask __read_mostly;
2224 EXPORT_SYMBOL(__cpu_online_mask);
2225
2226 struct cpumask __cpu_present_mask __read_mostly;
2227 EXPORT_SYMBOL(__cpu_present_mask);
2228
2229 struct cpumask __cpu_active_mask __read_mostly;
2230 EXPORT_SYMBOL(__cpu_active_mask);
2231
2232 void init_cpu_present(const struct cpumask *src)
2233 {
2234         cpumask_copy(&__cpu_present_mask, src);
2235 }
2236
2237 void init_cpu_possible(const struct cpumask *src)
2238 {
2239         cpumask_copy(&__cpu_possible_mask, src);
2240 }
2241
2242 void init_cpu_online(const struct cpumask *src)
2243 {
2244         cpumask_copy(&__cpu_online_mask, src);
2245 }
2246
2247 /*
2248  * Activate the first processor.
2249  */
2250 void __init boot_cpu_init(void)
2251 {
2252         int cpu = smp_processor_id();
2253
2254         /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2255         set_cpu_online(cpu, true);
2256         set_cpu_active(cpu, true);
2257         set_cpu_present(cpu, true);
2258         set_cpu_possible(cpu, true);
2259
2260 #ifdef CONFIG_SMP
2261         __boot_cpu_id = cpu;
2262 #endif
2263 }
2264
2265 /*
2266  * Must be called _AFTER_ setting up the per_cpu areas
2267  */
2268 void __init boot_cpu_hotplug_init(void)
2269 {
2270 #ifdef CONFIG_SMP
2271         this_cpu_write(cpuhp_state.booted_once, true);
2272 #endif
2273         this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2274 }