drivers/idle/intel_idle.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_idle.c - native hardware idle loop for modern Intel processors
   4  *
   5  * Copyright (c) 2013 - 2020, Intel Corporation.
   6  * Len Brown <len.brown@intel.com>
   7  * Rafael J. Wysocki <rafael.j.wysocki@intel.com>
   8  */
   9
  10 /*
  11  * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
  12  * in lieu of the legacy ACPI processor_idle driver.  The intent is to
  13  * make Linux more efficient on these processors, as intel_idle knows
  14  * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
  15  */
  16
  17 /*
  18  * Design Assumptions
  19  *
  20  * All CPUs have same idle states as boot CPU
  21  *
  22  * Chipset BM_STS (bus master status) bit is a NOP
  23  *      for preventing entry into deep C-states
  24  *
  25  * CPU will flush caches as needed when entering a C-state via MWAIT
  26  *      (in contrast to entering ACPI C3, in which case the WBINVD
  27  *      instruction needs to be executed to flush the caches)
  28  */
  29
  30 /*
  31  * Known limitations
  32  *
  33  * ACPI has a .suspend hack to turn off deep c-statees during suspend
  34  * to avoid complications with the lapic timer workaround.
  35  * Have not seen issues with suspend, but may need same workaround here.
  36  *
  37  */
  38
  39 /* un-comment DEBUG to enable pr_debug() statements */
  40 /* #define DEBUG */
  41
  42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  43
  44 #include <linux/acpi.h>
  45 #include <linux/kernel.h>
  46 #include <linux/cpuidle.h>
  47 #include <linux/tick.h>
  48 #include <trace/events/power.h>
  49 #include <linux/sched.h>
  50 #include <linux/sched/smt.h>
  51 #include <linux/notifier.h>
  52 #include <linux/cpu.h>
  53 #include <linux/moduleparam.h>
  54 #include <asm/cpu_device_id.h>
  55 #include <asm/intel-family.h>
  56 #include <asm/nospec-branch.h>
  57 #include <asm/mwait.h>
  58 #include <asm/msr.h>
  59 #include <asm/fpu/api.h>
  60
  61 #define INTEL_IDLE_VERSION "0.5.1"
  62
  63 static struct cpuidle_driver intel_idle_driver = {
  64         .name = "intel_idle",
  65         .owner = THIS_MODULE,
  66 };
  67 /* intel_idle.max_cstate=0 disables driver */
  68 static int max_cstate = CPUIDLE_STATE_MAX - 1;
  69 static unsigned int disabled_states_mask __read_mostly;
  70 static unsigned int preferred_states_mask __read_mostly;
  71 static bool force_irq_on __read_mostly;
  72
  73 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
  74
  75 static unsigned long auto_demotion_disable_flags;
  76
  77 static enum {
  78         C1E_PROMOTION_PRESERVE,
  79         C1E_PROMOTION_ENABLE,
  80         C1E_PROMOTION_DISABLE
  81 } c1e_promotion = C1E_PROMOTION_PRESERVE;
  82
  83 struct idle_cpu {
  84         struct cpuidle_state *state_table;
  85
  86         /*
  87          * Hardware C-state auto-demotion may not always be optimal.
  88          * Indicate which enable bits to clear here.
  89          */
  90         unsigned long auto_demotion_disable_flags;
  91         bool byt_auto_demotion_disable_flag;
  92         bool disable_promotion_to_c1e;
  93         bool use_acpi;
  94 };
  95
  96 static const struct idle_cpu *icpu __initdata;
  97 static struct cpuidle_state *cpuidle_state_table __initdata;
  98
  99 static unsigned int mwait_substates __initdata;
 100
 101 /*
 102  * Enable interrupts before entering the C-state. On some platforms and for
 103  * some C-states, this may measurably decrease interrupt latency.
 104  */
 105 #define CPUIDLE_FLAG_IRQ_ENABLE         BIT(14)
 106
 107 /*
 108  * Enable this state by default even if the ACPI _CST does not list it.
 109  */
 110 #define CPUIDLE_FLAG_ALWAYS_ENABLE      BIT(15)
 111
 112 /*
 113  * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
 114  * above.
 115  */
 116 #define CPUIDLE_FLAG_IBRS               BIT(16)
 117
 118 /*
 119  * Initialize large xstate for the C6-state entrance.
 120  */
 121 #define CPUIDLE_FLAG_INIT_XSTATE        BIT(17)
 122
 123 /*
 124  * MWAIT takes an 8-bit "hint" in EAX "suggesting"
 125  * the C-state (top nibble) and sub-state (bottom nibble)
 126  * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
 127  *
 128  * We store the hint at the top of our "flags" for each state.
 129  */
 130 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
 131 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
 132
 133 static __always_inline int __intel_idle(struct cpuidle_device *dev,
 134                                         struct cpuidle_driver *drv, int index)
 135 {
 136         struct cpuidle_state *state = &drv->states[index];
 137         unsigned long eax = flg2MWAIT(state->flags);
 138         unsigned long ecx = 1; /* break on interrupt flag */
 139
 140         mwait_idle_with_hints(eax, ecx);
 141
 142         return index;
 143 }
 144
 145 /**
 146  * intel_idle - Ask the processor to enter the given idle state.
 147  * @dev: cpuidle device of the target CPU.
 148  * @drv: cpuidle driver (assumed to point to intel_idle_driver).
 149  * @index: Target idle state index.
 150  *
 151  * Use the MWAIT instruction to notify the processor that the CPU represented by
 152  * @dev is idle and it can try to enter the idle state corresponding to @index.
 153  *
 154  * If the local APIC timer is not known to be reliable in the target idle state,
 155  * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
 156  *
 157  * Must be called under local_irq_disable().
 158  */
 159 static __cpuidle int intel_idle(struct cpuidle_device *dev,
 160                                 struct cpuidle_driver *drv, int index)
 161 {
 162         return __intel_idle(dev, drv, index);
 163 }
 164
 165 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
 166                                     struct cpuidle_driver *drv, int index)
 167 {
 168         int ret;
 169
 170         raw_local_irq_enable();
 171         ret = __intel_idle(dev, drv, index);
 172         raw_local_irq_disable();
 173
 174         return ret;
 175 }
 176
 177 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
 178                                      struct cpuidle_driver *drv, int index)
 179 {
 180         bool smt_active = sched_smt_active();
 181         u64 spec_ctrl = spec_ctrl_current();
 182         int ret;
 183
 184         if (smt_active)
 185                 native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
 186
 187         ret = __intel_idle(dev, drv, index);
 188
 189         if (smt_active)
 190                 native_wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl);
 191
 192         return ret;
 193 }
 194
 195 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
 196                                        struct cpuidle_driver *drv, int index)
 197 {
 198         fpu_idle_fpregs();
 199         return __intel_idle(dev, drv, index);
 200 }
 201
 202 static __always_inline int __intel_idle_hlt(struct cpuidle_device *dev,
 203                                         struct cpuidle_driver *drv, int index)
 204 {
 205         raw_safe_halt();
 206         raw_local_irq_disable();
 207         return index;
 208 }
 209
 210 /**
 211  * intel_idle_hlt - Ask the processor to enter the given idle state using hlt.
 212  * @dev: cpuidle device of the target CPU.
 213  * @drv: cpuidle driver (assumed to point to intel_idle_driver).
 214  * @index: Target idle state index.
 215  *
 216  * Use the HLT instruction to notify the processor that the CPU represented by
 217  * @dev is idle and it can try to enter the idle state corresponding to @index.
 218  *
 219  * Must be called under local_irq_disable().
 220  */
 221 static __cpuidle int intel_idle_hlt(struct cpuidle_device *dev,
 222                                 struct cpuidle_driver *drv, int index)
 223 {
 224         return __intel_idle_hlt(dev, drv, index);
 225 }
 226
 227 static __cpuidle int intel_idle_hlt_irq_on(struct cpuidle_device *dev,
 228                                    struct cpuidle_driver *drv, int index)
 229 {
 230        int ret;
 231
 232        raw_local_irq_enable();
 233        ret = __intel_idle_hlt(dev, drv, index);
 234        raw_local_irq_disable();
 235
 236        return ret;
 237 }
 238
 239 /**
 240  * intel_idle_s2idle - Ask the processor to enter the given idle state.
 241  * @dev: cpuidle device of the target CPU.
 242  * @drv: cpuidle driver (assumed to point to intel_idle_driver).
 243  * @index: Target idle state index.
 244  *
 245  * Use the MWAIT instruction to notify the processor that the CPU represented by
 246  * @dev is idle and it can try to enter the idle state corresponding to @index.
 247  *
 248  * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
 249  * scheduler tick and suspended scheduler clock on the target CPU.
 250  */
 251 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
 252                                        struct cpuidle_driver *drv, int index)
 253 {
 254         unsigned long ecx = 1; /* break on interrupt flag */
 255         struct cpuidle_state *state = &drv->states[index];
 256         unsigned long eax = flg2MWAIT(state->flags);
 257
 258         if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
 259                 fpu_idle_fpregs();
 260
 261         mwait_idle_with_hints(eax, ecx);
 262
 263         return 0;
 264 }
 265
 266 /*
 267  * States are indexed by the cstate number,
 268  * which is also the index into the MWAIT hint array.
 269  * Thus C0 is a dummy.
 270  */
 271 static struct cpuidle_state nehalem_cstates[] __initdata = {
 272         {
 273                 .name = "C1",
 274                 .desc = "MWAIT 0x00",
 275                 .flags = MWAIT2flg(0x00),
 276                 .exit_latency = 3,
 277                 .target_residency = 6,
 278                 .enter = &intel_idle,
 279                 .enter_s2idle = intel_idle_s2idle, },
 280         {
 281                 .name = "C1E",
 282                 .desc = "MWAIT 0x01",
 283                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 284                 .exit_latency = 10,
 285                 .target_residency = 20,
 286                 .enter = &intel_idle,
 287                 .enter_s2idle = intel_idle_s2idle, },
 288         {
 289                 .name = "C3",
 290                 .desc = "MWAIT 0x10",
 291                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 292                 .exit_latency = 20,
 293                 .target_residency = 80,
 294                 .enter = &intel_idle,
 295                 .enter_s2idle = intel_idle_s2idle, },
 296         {
 297                 .name = "C6",
 298                 .desc = "MWAIT 0x20",
 299                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 300                 .exit_latency = 200,
 301                 .target_residency = 800,
 302                 .enter = &intel_idle,
 303                 .enter_s2idle = intel_idle_s2idle, },
 304         {
 305                 .enter = NULL }
 306 };
 307
 308 static struct cpuidle_state snb_cstates[] __initdata = {
 309         {
 310                 .name = "C1",
 311                 .desc = "MWAIT 0x00",
 312                 .flags = MWAIT2flg(0x00),
 313                 .exit_latency = 2,
 314                 .target_residency = 2,
 315                 .enter = &intel_idle,
 316                 .enter_s2idle = intel_idle_s2idle, },
 317         {
 318                 .name = "C1E",
 319                 .desc = "MWAIT 0x01",
 320                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 321                 .exit_latency = 10,
 322                 .target_residency = 20,
 323                 .enter = &intel_idle,
 324                 .enter_s2idle = intel_idle_s2idle, },
 325         {
 326                 .name = "C3",
 327                 .desc = "MWAIT 0x10",
 328                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 329                 .exit_latency = 80,
 330                 .target_residency = 211,
 331                 .enter = &intel_idle,
 332                 .enter_s2idle = intel_idle_s2idle, },
 333         {
 334                 .name = "C6",
 335                 .desc = "MWAIT 0x20",
 336                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 337                 .exit_latency = 104,
 338                 .target_residency = 345,
 339                 .enter = &intel_idle,
 340                 .enter_s2idle = intel_idle_s2idle, },
 341         {
 342                 .name = "C7",
 343                 .desc = "MWAIT 0x30",
 344                 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
 345                 .exit_latency = 109,
 346                 .target_residency = 345,
 347                 .enter = &intel_idle,
 348                 .enter_s2idle = intel_idle_s2idle, },
 349         {
 350                 .enter = NULL }
 351 };
 352
 353 static struct cpuidle_state byt_cstates[] __initdata = {
 354         {
 355                 .name = "C1",
 356                 .desc = "MWAIT 0x00",
 357                 .flags = MWAIT2flg(0x00),
 358                 .exit_latency = 1,
 359                 .target_residency = 1,
 360                 .enter = &intel_idle,
 361                 .enter_s2idle = intel_idle_s2idle, },
 362         {
 363                 .name = "C6N",
 364                 .desc = "MWAIT 0x58",
 365                 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
 366                 .exit_latency = 300,
 367                 .target_residency = 275,
 368                 .enter = &intel_idle,
 369                 .enter_s2idle = intel_idle_s2idle, },
 370         {
 371                 .name = "C6S",
 372                 .desc = "MWAIT 0x52",
 373                 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
 374                 .exit_latency = 500,
 375                 .target_residency = 560,
 376                 .enter = &intel_idle,
 377                 .enter_s2idle = intel_idle_s2idle, },
 378         {
 379                 .name = "C7",
 380                 .desc = "MWAIT 0x60",
 381                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 382                 .exit_latency = 1200,
 383                 .target_residency = 4000,
 384                 .enter = &intel_idle,
 385                 .enter_s2idle = intel_idle_s2idle, },
 386         {
 387                 .name = "C7S",
 388                 .desc = "MWAIT 0x64",
 389                 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
 390                 .exit_latency = 10000,
 391                 .target_residency = 20000,
 392                 .enter = &intel_idle,
 393                 .enter_s2idle = intel_idle_s2idle, },
 394         {
 395                 .enter = NULL }
 396 };
 397
 398 static struct cpuidle_state cht_cstates[] __initdata = {
 399         {
 400                 .name = "C1",
 401                 .desc = "MWAIT 0x00",
 402                 .flags = MWAIT2flg(0x00),
 403                 .exit_latency = 1,
 404                 .target_residency = 1,
 405                 .enter = &intel_idle,
 406                 .enter_s2idle = intel_idle_s2idle, },
 407         {
 408                 .name = "C6N",
 409                 .desc = "MWAIT 0x58",
 410                 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
 411                 .exit_latency = 80,
 412                 .target_residency = 275,
 413                 .enter = &intel_idle,
 414                 .enter_s2idle = intel_idle_s2idle, },
 415         {
 416                 .name = "C6S",
 417                 .desc = "MWAIT 0x52",
 418                 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
 419                 .exit_latency = 200,
 420                 .target_residency = 560,
 421                 .enter = &intel_idle,
 422                 .enter_s2idle = intel_idle_s2idle, },
 423         {
 424                 .name = "C7",
 425                 .desc = "MWAIT 0x60",
 426                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 427                 .exit_latency = 1200,
 428                 .target_residency = 4000,
 429                 .enter = &intel_idle,
 430                 .enter_s2idle = intel_idle_s2idle, },
 431         {
 432                 .name = "C7S",
 433                 .desc = "MWAIT 0x64",
 434                 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
 435                 .exit_latency = 10000,
 436                 .target_residency = 20000,
 437                 .enter = &intel_idle,
 438                 .enter_s2idle = intel_idle_s2idle, },
 439         {
 440                 .enter = NULL }
 441 };
 442
 443 static struct cpuidle_state ivb_cstates[] __initdata = {
 444         {
 445                 .name = "C1",
 446                 .desc = "MWAIT 0x00",
 447                 .flags = MWAIT2flg(0x00),
 448                 .exit_latency = 1,
 449                 .target_residency = 1,
 450                 .enter = &intel_idle,
 451                 .enter_s2idle = intel_idle_s2idle, },
 452         {
 453                 .name = "C1E",
 454                 .desc = "MWAIT 0x01",
 455                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 456                 .exit_latency = 10,
 457                 .target_residency = 20,
 458                 .enter = &intel_idle,
 459                 .enter_s2idle = intel_idle_s2idle, },
 460         {
 461                 .name = "C3",
 462                 .desc = "MWAIT 0x10",
 463                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 464                 .exit_latency = 59,
 465                 .target_residency = 156,
 466                 .enter = &intel_idle,
 467                 .enter_s2idle = intel_idle_s2idle, },
 468         {
 469                 .name = "C6",
 470                 .desc = "MWAIT 0x20",
 471                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 472                 .exit_latency = 80,
 473                 .target_residency = 300,
 474                 .enter = &intel_idle,
 475                 .enter_s2idle = intel_idle_s2idle, },
 476         {
 477                 .name = "C7",
 478                 .desc = "MWAIT 0x30",
 479                 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
 480                 .exit_latency = 87,
 481                 .target_residency = 300,
 482                 .enter = &intel_idle,
 483                 .enter_s2idle = intel_idle_s2idle, },
 484         {
 485                 .enter = NULL }
 486 };
 487
 488 static struct cpuidle_state ivt_cstates[] __initdata = {
 489         {
 490                 .name = "C1",
 491                 .desc = "MWAIT 0x00",
 492                 .flags = MWAIT2flg(0x00),
 493                 .exit_latency = 1,
 494                 .target_residency = 1,
 495                 .enter = &intel_idle,
 496                 .enter_s2idle = intel_idle_s2idle, },
 497         {
 498                 .name = "C1E",
 499                 .desc = "MWAIT 0x01",
 500                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 501                 .exit_latency = 10,
 502                 .target_residency = 80,
 503                 .enter = &intel_idle,
 504                 .enter_s2idle = intel_idle_s2idle, },
 505         {
 506                 .name = "C3",
 507                 .desc = "MWAIT 0x10",
 508                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 509                 .exit_latency = 59,
 510                 .target_residency = 156,
 511                 .enter = &intel_idle,
 512                 .enter_s2idle = intel_idle_s2idle, },
 513         {
 514                 .name = "C6",
 515                 .desc = "MWAIT 0x20",
 516                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 517                 .exit_latency = 82,
 518                 .target_residency = 300,
 519                 .enter = &intel_idle,
 520                 .enter_s2idle = intel_idle_s2idle, },
 521         {
 522                 .enter = NULL }
 523 };
 524
 525 static struct cpuidle_state ivt_cstates_4s[] __initdata = {
 526         {
 527                 .name = "C1",
 528                 .desc = "MWAIT 0x00",
 529                 .flags = MWAIT2flg(0x00),
 530                 .exit_latency = 1,
 531                 .target_residency = 1,
 532                 .enter = &intel_idle,
 533                 .enter_s2idle = intel_idle_s2idle, },
 534         {
 535                 .name = "C1E",
 536                 .desc = "MWAIT 0x01",
 537                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 538                 .exit_latency = 10,
 539                 .target_residency = 250,
 540                 .enter = &intel_idle,
 541                 .enter_s2idle = intel_idle_s2idle, },
 542         {
 543                 .name = "C3",
 544                 .desc = "MWAIT 0x10",
 545                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 546                 .exit_latency = 59,
 547                 .target_residency = 300,
 548                 .enter = &intel_idle,
 549                 .enter_s2idle = intel_idle_s2idle, },
 550         {
 551                 .name = "C6",
 552                 .desc = "MWAIT 0x20",
 553                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 554                 .exit_latency = 84,
 555                 .target_residency = 400,
 556                 .enter = &intel_idle,
 557                 .enter_s2idle = intel_idle_s2idle, },
 558         {
 559                 .enter = NULL }
 560 };
 561
 562 static struct cpuidle_state ivt_cstates_8s[] __initdata = {
 563         {
 564                 .name = "C1",
 565                 .desc = "MWAIT 0x00",
 566                 .flags = MWAIT2flg(0x00),
 567                 .exit_latency = 1,
 568                 .target_residency = 1,
 569                 .enter = &intel_idle,
 570                 .enter_s2idle = intel_idle_s2idle, },
 571         {
 572                 .name = "C1E",
 573                 .desc = "MWAIT 0x01",
 574                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 575                 .exit_latency = 10,
 576                 .target_residency = 500,
 577                 .enter = &intel_idle,
 578                 .enter_s2idle = intel_idle_s2idle, },
 579         {
 580                 .name = "C3",
 581                 .desc = "MWAIT 0x10",
 582                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 583                 .exit_latency = 59,
 584                 .target_residency = 600,
 585                 .enter = &intel_idle,
 586                 .enter_s2idle = intel_idle_s2idle, },
 587         {
 588                 .name = "C6",
 589                 .desc = "MWAIT 0x20",
 590                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 591                 .exit_latency = 88,
 592                 .target_residency = 700,
 593                 .enter = &intel_idle,
 594                 .enter_s2idle = intel_idle_s2idle, },
 595         {
 596                 .enter = NULL }
 597 };
 598
 599 static struct cpuidle_state hsw_cstates[] __initdata = {
 600         {
 601                 .name = "C1",
 602                 .desc = "MWAIT 0x00",
 603                 .flags = MWAIT2flg(0x00),
 604                 .exit_latency = 2,
 605                 .target_residency = 2,
 606                 .enter = &intel_idle,
 607                 .enter_s2idle = intel_idle_s2idle, },
 608         {
 609                 .name = "C1E",
 610                 .desc = "MWAIT 0x01",
 611                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 612                 .exit_latency = 10,
 613                 .target_residency = 20,
 614                 .enter = &intel_idle,
 615                 .enter_s2idle = intel_idle_s2idle, },
 616         {
 617                 .name = "C3",
 618                 .desc = "MWAIT 0x10",
 619                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 620                 .exit_latency = 33,
 621                 .target_residency = 100,
 622                 .enter = &intel_idle,
 623                 .enter_s2idle = intel_idle_s2idle, },
 624         {
 625                 .name = "C6",
 626                 .desc = "MWAIT 0x20",
 627                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 628                 .exit_latency = 133,
 629                 .target_residency = 400,
 630                 .enter = &intel_idle,
 631                 .enter_s2idle = intel_idle_s2idle, },
 632         {
 633                 .name = "C7s",
 634                 .desc = "MWAIT 0x32",
 635                 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
 636                 .exit_latency = 166,
 637                 .target_residency = 500,
 638                 .enter = &intel_idle,
 639                 .enter_s2idle = intel_idle_s2idle, },
 640         {
 641                 .name = "C8",
 642                 .desc = "MWAIT 0x40",
 643                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 644                 .exit_latency = 300,
 645                 .target_residency = 900,
 646                 .enter = &intel_idle,
 647                 .enter_s2idle = intel_idle_s2idle, },
 648         {
 649                 .name = "C9",
 650                 .desc = "MWAIT 0x50",
 651                 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
 652                 .exit_latency = 600,
 653                 .target_residency = 1800,
 654                 .enter = &intel_idle,
 655                 .enter_s2idle = intel_idle_s2idle, },
 656         {
 657                 .name = "C10",
 658                 .desc = "MWAIT 0x60",
 659                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 660                 .exit_latency = 2600,
 661                 .target_residency = 7700,
 662                 .enter = &intel_idle,
 663                 .enter_s2idle = intel_idle_s2idle, },
 664         {
 665                 .enter = NULL }
 666 };
 667 static struct cpuidle_state bdw_cstates[] __initdata = {
 668         {
 669                 .name = "C1",
 670                 .desc = "MWAIT 0x00",
 671                 .flags = MWAIT2flg(0x00),
 672                 .exit_latency = 2,
 673                 .target_residency = 2,
 674                 .enter = &intel_idle,
 675                 .enter_s2idle = intel_idle_s2idle, },
 676         {
 677                 .name = "C1E",
 678                 .desc = "MWAIT 0x01",
 679                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 680                 .exit_latency = 10,
 681                 .target_residency = 20,
 682                 .enter = &intel_idle,
 683                 .enter_s2idle = intel_idle_s2idle, },
 684         {
 685                 .name = "C3",
 686                 .desc = "MWAIT 0x10",
 687                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 688                 .exit_latency = 40,
 689                 .target_residency = 100,
 690                 .enter = &intel_idle,
 691                 .enter_s2idle = intel_idle_s2idle, },
 692         {
 693                 .name = "C6",
 694                 .desc = "MWAIT 0x20",
 695                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 696                 .exit_latency = 133,
 697                 .target_residency = 400,
 698                 .enter = &intel_idle,
 699                 .enter_s2idle = intel_idle_s2idle, },
 700         {
 701                 .name = "C7s",
 702                 .desc = "MWAIT 0x32",
 703                 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
 704                 .exit_latency = 166,
 705                 .target_residency = 500,
 706                 .enter = &intel_idle,
 707                 .enter_s2idle = intel_idle_s2idle, },
 708         {
 709                 .name = "C8",
 710                 .desc = "MWAIT 0x40",
 711                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 712                 .exit_latency = 300,
 713                 .target_residency = 900,
 714                 .enter = &intel_idle,
 715                 .enter_s2idle = intel_idle_s2idle, },
 716         {
 717                 .name = "C9",
 718                 .desc = "MWAIT 0x50",
 719                 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
 720                 .exit_latency = 600,
 721                 .target_residency = 1800,
 722                 .enter = &intel_idle,
 723                 .enter_s2idle = intel_idle_s2idle, },
 724         {
 725                 .name = "C10",
 726                 .desc = "MWAIT 0x60",
 727                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 728                 .exit_latency = 2600,
 729                 .target_residency = 7700,
 730                 .enter = &intel_idle,
 731                 .enter_s2idle = intel_idle_s2idle, },
 732         {
 733                 .enter = NULL }
 734 };
 735
 736 static struct cpuidle_state skl_cstates[] __initdata = {
 737         {
 738                 .name = "C1",
 739                 .desc = "MWAIT 0x00",
 740                 .flags = MWAIT2flg(0x00),
 741                 .exit_latency = 2,
 742                 .target_residency = 2,
 743                 .enter = &intel_idle,
 744                 .enter_s2idle = intel_idle_s2idle, },
 745         {
 746                 .name = "C1E",
 747                 .desc = "MWAIT 0x01",
 748                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 749                 .exit_latency = 10,
 750                 .target_residency = 20,
 751                 .enter = &intel_idle,
 752                 .enter_s2idle = intel_idle_s2idle, },
 753         {
 754                 .name = "C3",
 755                 .desc = "MWAIT 0x10",
 756                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 757                 .exit_latency = 70,
 758                 .target_residency = 100,
 759                 .enter = &intel_idle,
 760                 .enter_s2idle = intel_idle_s2idle, },
 761         {
 762                 .name = "C6",
 763                 .desc = "MWAIT 0x20",
 764                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 765                 .exit_latency = 85,
 766                 .target_residency = 200,
 767                 .enter = &intel_idle,
 768                 .enter_s2idle = intel_idle_s2idle, },
 769         {
 770                 .name = "C7s",
 771                 .desc = "MWAIT 0x33",
 772                 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 773                 .exit_latency = 124,
 774                 .target_residency = 800,
 775                 .enter = &intel_idle,
 776                 .enter_s2idle = intel_idle_s2idle, },
 777         {
 778                 .name = "C8",
 779                 .desc = "MWAIT 0x40",
 780                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 781                 .exit_latency = 200,
 782                 .target_residency = 800,
 783                 .enter = &intel_idle,
 784                 .enter_s2idle = intel_idle_s2idle, },
 785         {
 786                 .name = "C9",
 787                 .desc = "MWAIT 0x50",
 788                 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 789                 .exit_latency = 480,
 790                 .target_residency = 5000,
 791                 .enter = &intel_idle,
 792                 .enter_s2idle = intel_idle_s2idle, },
 793         {
 794                 .name = "C10",
 795                 .desc = "MWAIT 0x60",
 796                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 797                 .exit_latency = 890,
 798                 .target_residency = 5000,
 799                 .enter = &intel_idle,
 800                 .enter_s2idle = intel_idle_s2idle, },
 801         {
 802                 .enter = NULL }
 803 };
 804
 805 static struct cpuidle_state skx_cstates[] __initdata = {
 806         {
 807                 .name = "C1",
 808                 .desc = "MWAIT 0x00",
 809                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
 810                 .exit_latency = 2,
 811                 .target_residency = 2,
 812                 .enter = &intel_idle,
 813                 .enter_s2idle = intel_idle_s2idle, },
 814         {
 815                 .name = "C1E",
 816                 .desc = "MWAIT 0x01",
 817                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 818                 .exit_latency = 10,
 819                 .target_residency = 20,
 820                 .enter = &intel_idle,
 821                 .enter_s2idle = intel_idle_s2idle, },
 822         {
 823                 .name = "C6",
 824                 .desc = "MWAIT 0x20",
 825                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 826                 .exit_latency = 133,
 827                 .target_residency = 600,
 828                 .enter = &intel_idle,
 829                 .enter_s2idle = intel_idle_s2idle, },
 830         {
 831                 .enter = NULL }
 832 };
 833
 834 static struct cpuidle_state icx_cstates[] __initdata = {
 835         {
 836                 .name = "C1",
 837                 .desc = "MWAIT 0x00",
 838                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
 839                 .exit_latency = 1,
 840                 .target_residency = 1,
 841                 .enter = &intel_idle,
 842                 .enter_s2idle = intel_idle_s2idle, },
 843         {
 844                 .name = "C1E",
 845                 .desc = "MWAIT 0x01",
 846                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 847                 .exit_latency = 4,
 848                 .target_residency = 4,
 849                 .enter = &intel_idle,
 850                 .enter_s2idle = intel_idle_s2idle, },
 851         {
 852                 .name = "C6",
 853                 .desc = "MWAIT 0x20",
 854                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 855                 .exit_latency = 170,
 856                 .target_residency = 600,
 857                 .enter = &intel_idle,
 858                 .enter_s2idle = intel_idle_s2idle, },
 859         {
 860                 .enter = NULL }
 861 };
 862
 863 /*
 864  * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
 865  * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
 866  * But in this case there is effectively no C1, because C1 requests are
 867  * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
 868  * and C1E requests end up with C1, so there is effectively no C1E.
 869  *
 870  * By default we enable C1E and disable C1 by marking it with
 871  * 'CPUIDLE_FLAG_UNUSABLE'.
 872  */
 873 static struct cpuidle_state adl_cstates[] __initdata = {
 874         {
 875                 .name = "C1",
 876                 .desc = "MWAIT 0x00",
 877                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 878                 .exit_latency = 1,
 879                 .target_residency = 1,
 880                 .enter = &intel_idle,
 881                 .enter_s2idle = intel_idle_s2idle, },
 882         {
 883                 .name = "C1E",
 884                 .desc = "MWAIT 0x01",
 885                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 886                 .exit_latency = 2,
 887                 .target_residency = 4,
 888                 .enter = &intel_idle,
 889                 .enter_s2idle = intel_idle_s2idle, },
 890         {
 891                 .name = "C6",
 892                 .desc = "MWAIT 0x20",
 893                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 894                 .exit_latency = 220,
 895                 .target_residency = 600,
 896                 .enter = &intel_idle,
 897                 .enter_s2idle = intel_idle_s2idle, },
 898         {
 899                 .name = "C8",
 900                 .desc = "MWAIT 0x40",
 901                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 902                 .exit_latency = 280,
 903                 .target_residency = 800,
 904                 .enter = &intel_idle,
 905                 .enter_s2idle = intel_idle_s2idle, },
 906         {
 907                 .name = "C10",
 908                 .desc = "MWAIT 0x60",
 909                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 910                 .exit_latency = 680,
 911                 .target_residency = 2000,
 912                 .enter = &intel_idle,
 913                 .enter_s2idle = intel_idle_s2idle, },
 914         {
 915                 .enter = NULL }
 916 };
 917
 918 static struct cpuidle_state adl_l_cstates[] __initdata = {
 919         {
 920                 .name = "C1",
 921                 .desc = "MWAIT 0x00",
 922                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 923                 .exit_latency = 1,
 924                 .target_residency = 1,
 925                 .enter = &intel_idle,
 926                 .enter_s2idle = intel_idle_s2idle, },
 927         {
 928                 .name = "C1E",
 929                 .desc = "MWAIT 0x01",
 930                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 931                 .exit_latency = 2,
 932                 .target_residency = 4,
 933                 .enter = &intel_idle,
 934                 .enter_s2idle = intel_idle_s2idle, },
 935         {
 936                 .name = "C6",
 937                 .desc = "MWAIT 0x20",
 938                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 939                 .exit_latency = 170,
 940                 .target_residency = 500,
 941                 .enter = &intel_idle,
 942                 .enter_s2idle = intel_idle_s2idle, },
 943         {
 944                 .name = "C8",
 945                 .desc = "MWAIT 0x40",
 946                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 947                 .exit_latency = 200,
 948                 .target_residency = 600,
 949                 .enter = &intel_idle,
 950                 .enter_s2idle = intel_idle_s2idle, },
 951         {
 952                 .name = "C10",
 953                 .desc = "MWAIT 0x60",
 954                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 955                 .exit_latency = 230,
 956                 .target_residency = 700,
 957                 .enter = &intel_idle,
 958                 .enter_s2idle = intel_idle_s2idle, },
 959         {
 960                 .enter = NULL }
 961 };
 962
 963 static struct cpuidle_state adl_n_cstates[] __initdata = {
 964         {
 965                 .name = "C1",
 966                 .desc = "MWAIT 0x00",
 967                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 968                 .exit_latency = 1,
 969                 .target_residency = 1,
 970                 .enter = &intel_idle,
 971                 .enter_s2idle = intel_idle_s2idle, },
 972         {
 973                 .name = "C1E",
 974                 .desc = "MWAIT 0x01",
 975                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 976                 .exit_latency = 2,
 977                 .target_residency = 4,
 978                 .enter = &intel_idle,
 979                 .enter_s2idle = intel_idle_s2idle, },
 980         {
 981                 .name = "C6",
 982                 .desc = "MWAIT 0x20",
 983                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 984                 .exit_latency = 195,
 985                 .target_residency = 585,
 986                 .enter = &intel_idle,
 987                 .enter_s2idle = intel_idle_s2idle, },
 988         {
 989                 .name = "C8",
 990                 .desc = "MWAIT 0x40",
 991                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 992                 .exit_latency = 260,
 993                 .target_residency = 1040,
 994                 .enter = &intel_idle,
 995                 .enter_s2idle = intel_idle_s2idle, },
 996         {
 997                 .name = "C10",
 998                 .desc = "MWAIT 0x60",
 999                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1000                 .exit_latency = 660,
1001                 .target_residency = 1980,
1002                 .enter = &intel_idle,
1003                 .enter_s2idle = intel_idle_s2idle, },
1004         {
1005                 .enter = NULL }
1006 };
1007
1008 static struct cpuidle_state spr_cstates[] __initdata = {
1009         {
1010                 .name = "C1",
1011                 .desc = "MWAIT 0x00",
1012                 .flags = MWAIT2flg(0x00),
1013                 .exit_latency = 1,
1014                 .target_residency = 1,
1015                 .enter = &intel_idle,
1016                 .enter_s2idle = intel_idle_s2idle, },
1017         {
1018                 .name = "C1E",
1019                 .desc = "MWAIT 0x01",
1020                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1021                 .exit_latency = 2,
1022                 .target_residency = 4,
1023                 .enter = &intel_idle,
1024                 .enter_s2idle = intel_idle_s2idle, },
1025         {
1026                 .name = "C6",
1027                 .desc = "MWAIT 0x20",
1028                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1029                                            CPUIDLE_FLAG_INIT_XSTATE,
1030                 .exit_latency = 290,
1031                 .target_residency = 800,
1032                 .enter = &intel_idle,
1033                 .enter_s2idle = intel_idle_s2idle, },
1034         {
1035                 .enter = NULL }
1036 };
1037
1038 static struct cpuidle_state atom_cstates[] __initdata = {
1039         {
1040                 .name = "C1E",
1041                 .desc = "MWAIT 0x00",
1042                 .flags = MWAIT2flg(0x00),
1043                 .exit_latency = 10,
1044                 .target_residency = 20,
1045                 .enter = &intel_idle,
1046                 .enter_s2idle = intel_idle_s2idle, },
1047         {
1048                 .name = "C2",
1049                 .desc = "MWAIT 0x10",
1050                 .flags = MWAIT2flg(0x10),
1051                 .exit_latency = 20,
1052                 .target_residency = 80,
1053                 .enter = &intel_idle,
1054                 .enter_s2idle = intel_idle_s2idle, },
1055         {
1056                 .name = "C4",
1057                 .desc = "MWAIT 0x30",
1058                 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1059                 .exit_latency = 100,
1060                 .target_residency = 400,
1061                 .enter = &intel_idle,
1062                 .enter_s2idle = intel_idle_s2idle, },
1063         {
1064                 .name = "C6",
1065                 .desc = "MWAIT 0x52",
1066                 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1067                 .exit_latency = 140,
1068                 .target_residency = 560,
1069                 .enter = &intel_idle,
1070                 .enter_s2idle = intel_idle_s2idle, },
1071         {
1072                 .enter = NULL }
1073 };
1074 static struct cpuidle_state tangier_cstates[] __initdata = {
1075         {
1076                 .name = "C1",
1077                 .desc = "MWAIT 0x00",
1078                 .flags = MWAIT2flg(0x00),
1079                 .exit_latency = 1,
1080                 .target_residency = 4,
1081                 .enter = &intel_idle,
1082                 .enter_s2idle = intel_idle_s2idle, },
1083         {
1084                 .name = "C4",
1085                 .desc = "MWAIT 0x30",
1086                 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1087                 .exit_latency = 100,
1088                 .target_residency = 400,
1089                 .enter = &intel_idle,
1090                 .enter_s2idle = intel_idle_s2idle, },
1091         {
1092                 .name = "C6",
1093                 .desc = "MWAIT 0x52",
1094                 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1095                 .exit_latency = 140,
1096                 .target_residency = 560,
1097                 .enter = &intel_idle,
1098                 .enter_s2idle = intel_idle_s2idle, },
1099         {
1100                 .name = "C7",
1101                 .desc = "MWAIT 0x60",
1102                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1103                 .exit_latency = 1200,
1104                 .target_residency = 4000,
1105                 .enter = &intel_idle,
1106                 .enter_s2idle = intel_idle_s2idle, },
1107         {
1108                 .name = "C9",
1109                 .desc = "MWAIT 0x64",
1110                 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1111                 .exit_latency = 10000,
1112                 .target_residency = 20000,
1113                 .enter = &intel_idle,
1114                 .enter_s2idle = intel_idle_s2idle, },
1115         {
1116                 .enter = NULL }
1117 };
1118 static struct cpuidle_state avn_cstates[] __initdata = {
1119         {
1120                 .name = "C1",
1121                 .desc = "MWAIT 0x00",
1122                 .flags = MWAIT2flg(0x00),
1123                 .exit_latency = 2,
1124                 .target_residency = 2,
1125                 .enter = &intel_idle,
1126                 .enter_s2idle = intel_idle_s2idle, },
1127         {
1128                 .name = "C6",
1129                 .desc = "MWAIT 0x51",
1130                 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
1131                 .exit_latency = 15,
1132                 .target_residency = 45,
1133                 .enter = &intel_idle,
1134                 .enter_s2idle = intel_idle_s2idle, },
1135         {
1136                 .enter = NULL }
1137 };
1138 static struct cpuidle_state knl_cstates[] __initdata = {
1139         {
1140                 .name = "C1",
1141                 .desc = "MWAIT 0x00",
1142                 .flags = MWAIT2flg(0x00),
1143                 .exit_latency = 1,
1144                 .target_residency = 2,
1145                 .enter = &intel_idle,
1146                 .enter_s2idle = intel_idle_s2idle },
1147         {
1148                 .name = "C6",
1149                 .desc = "MWAIT 0x10",
1150                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1151                 .exit_latency = 120,
1152                 .target_residency = 500,
1153                 .enter = &intel_idle,
1154                 .enter_s2idle = intel_idle_s2idle },
1155         {
1156                 .enter = NULL }
1157 };
1158
1159 static struct cpuidle_state bxt_cstates[] __initdata = {
1160         {
1161                 .name = "C1",
1162                 .desc = "MWAIT 0x00",
1163                 .flags = MWAIT2flg(0x00),
1164                 .exit_latency = 2,
1165                 .target_residency = 2,
1166                 .enter = &intel_idle,
1167                 .enter_s2idle = intel_idle_s2idle, },
1168         {
1169                 .name = "C1E",
1170                 .desc = "MWAIT 0x01",
1171                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1172                 .exit_latency = 10,
1173                 .target_residency = 20,
1174                 .enter = &intel_idle,
1175                 .enter_s2idle = intel_idle_s2idle, },
1176         {
1177                 .name = "C6",
1178                 .desc = "MWAIT 0x20",
1179                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1180                 .exit_latency = 133,
1181                 .target_residency = 133,
1182                 .enter = &intel_idle,
1183                 .enter_s2idle = intel_idle_s2idle, },
1184         {
1185                 .name = "C7s",
1186                 .desc = "MWAIT 0x31",
1187                 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1188                 .exit_latency = 155,
1189                 .target_residency = 155,
1190                 .enter = &intel_idle,
1191                 .enter_s2idle = intel_idle_s2idle, },
1192         {
1193                 .name = "C8",
1194                 .desc = "MWAIT 0x40",
1195                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1196                 .exit_latency = 1000,
1197                 .target_residency = 1000,
1198                 .enter = &intel_idle,
1199                 .enter_s2idle = intel_idle_s2idle, },
1200         {
1201                 .name = "C9",
1202                 .desc = "MWAIT 0x50",
1203                 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1204                 .exit_latency = 2000,
1205                 .target_residency = 2000,
1206                 .enter = &intel_idle,
1207                 .enter_s2idle = intel_idle_s2idle, },
1208         {
1209                 .name = "C10",
1210                 .desc = "MWAIT 0x60",
1211                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1212                 .exit_latency = 10000,
1213                 .target_residency = 10000,
1214                 .enter = &intel_idle,
1215                 .enter_s2idle = intel_idle_s2idle, },
1216         {
1217                 .enter = NULL }
1218 };
1219
1220 static struct cpuidle_state dnv_cstates[] __initdata = {
1221         {
1222                 .name = "C1",
1223                 .desc = "MWAIT 0x00",
1224                 .flags = MWAIT2flg(0x00),
1225                 .exit_latency = 2,
1226                 .target_residency = 2,
1227                 .enter = &intel_idle,
1228                 .enter_s2idle = intel_idle_s2idle, },
1229         {
1230                 .name = "C1E",
1231                 .desc = "MWAIT 0x01",
1232                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1233                 .exit_latency = 10,
1234                 .target_residency = 20,
1235                 .enter = &intel_idle,
1236                 .enter_s2idle = intel_idle_s2idle, },
1237         {
1238                 .name = "C6",
1239                 .desc = "MWAIT 0x20",
1240                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1241                 .exit_latency = 50,
1242                 .target_residency = 500,
1243                 .enter = &intel_idle,
1244                 .enter_s2idle = intel_idle_s2idle, },
1245         {
1246                 .enter = NULL }
1247 };
1248
1249 /*
1250  * Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1251  * C6, and this is indicated in the CPUID mwait leaf.
1252  */
1253 static struct cpuidle_state snr_cstates[] __initdata = {
1254         {
1255                 .name = "C1",
1256                 .desc = "MWAIT 0x00",
1257                 .flags = MWAIT2flg(0x00),
1258                 .exit_latency = 2,
1259                 .target_residency = 2,
1260                 .enter = &intel_idle,
1261                 .enter_s2idle = intel_idle_s2idle, },
1262         {
1263                 .name = "C1E",
1264                 .desc = "MWAIT 0x01",
1265                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1266                 .exit_latency = 15,
1267                 .target_residency = 25,
1268                 .enter = &intel_idle,
1269                 .enter_s2idle = intel_idle_s2idle, },
1270         {
1271                 .name = "C6",
1272                 .desc = "MWAIT 0x20",
1273                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1274                 .exit_latency = 130,
1275                 .target_residency = 500,
1276                 .enter = &intel_idle,
1277                 .enter_s2idle = intel_idle_s2idle, },
1278         {
1279                 .enter = NULL }
1280 };
1281
1282 static struct cpuidle_state vmguest_cstates[] __initdata = {
1283         {
1284                 .name = "C1",
1285                 .desc = "HLT",
1286                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
1287                 .exit_latency = 5,
1288                 .target_residency = 10,
1289                 .enter = &intel_idle_hlt, },
1290         {
1291                 .name = "C1L",
1292                 .desc = "Long HLT",
1293                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TLB_FLUSHED,
1294                 .exit_latency = 5,
1295                 .target_residency = 200,
1296                 .enter = &intel_idle_hlt, },
1297         {
1298                 .enter = NULL }
1299 };
1300
1301 static const struct idle_cpu idle_cpu_nehalem __initconst = {
1302         .state_table = nehalem_cstates,
1303         .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1304         .disable_promotion_to_c1e = true,
1305 };
1306
1307 static const struct idle_cpu idle_cpu_nhx __initconst = {
1308         .state_table = nehalem_cstates,
1309         .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1310         .disable_promotion_to_c1e = true,
1311         .use_acpi = true,
1312 };
1313
1314 static const struct idle_cpu idle_cpu_atom __initconst = {
1315         .state_table = atom_cstates,
1316 };
1317
1318 static const struct idle_cpu idle_cpu_tangier __initconst = {
1319         .state_table = tangier_cstates,
1320 };
1321
1322 static const struct idle_cpu idle_cpu_lincroft __initconst = {
1323         .state_table = atom_cstates,
1324         .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1325 };
1326
1327 static const struct idle_cpu idle_cpu_snb __initconst = {
1328         .state_table = snb_cstates,
1329         .disable_promotion_to_c1e = true,
1330 };
1331
1332 static const struct idle_cpu idle_cpu_snx __initconst = {
1333         .state_table = snb_cstates,
1334         .disable_promotion_to_c1e = true,
1335         .use_acpi = true,
1336 };
1337
1338 static const struct idle_cpu idle_cpu_byt __initconst = {
1339         .state_table = byt_cstates,
1340         .disable_promotion_to_c1e = true,
1341         .byt_auto_demotion_disable_flag = true,
1342 };
1343
1344 static const struct idle_cpu idle_cpu_cht __initconst = {
1345         .state_table = cht_cstates,
1346         .disable_promotion_to_c1e = true,
1347         .byt_auto_demotion_disable_flag = true,
1348 };
1349
1350 static const struct idle_cpu idle_cpu_ivb __initconst = {
1351         .state_table = ivb_cstates,
1352         .disable_promotion_to_c1e = true,
1353 };
1354
1355 static const struct idle_cpu idle_cpu_ivt __initconst = {
1356         .state_table = ivt_cstates,
1357         .disable_promotion_to_c1e = true,
1358         .use_acpi = true,
1359 };
1360
1361 static const struct idle_cpu idle_cpu_hsw __initconst = {
1362         .state_table = hsw_cstates,
1363         .disable_promotion_to_c1e = true,
1364 };
1365
1366 static const struct idle_cpu idle_cpu_hsx __initconst = {
1367         .state_table = hsw_cstates,
1368         .disable_promotion_to_c1e = true,
1369         .use_acpi = true,
1370 };
1371
1372 static const struct idle_cpu idle_cpu_bdw __initconst = {
1373         .state_table = bdw_cstates,
1374         .disable_promotion_to_c1e = true,
1375 };
1376
1377 static const struct idle_cpu idle_cpu_bdx __initconst = {
1378         .state_table = bdw_cstates,
1379         .disable_promotion_to_c1e = true,
1380         .use_acpi = true,
1381 };
1382
1383 static const struct idle_cpu idle_cpu_skl __initconst = {
1384         .state_table = skl_cstates,
1385         .disable_promotion_to_c1e = true,
1386 };
1387
1388 static const struct idle_cpu idle_cpu_skx __initconst = {
1389         .state_table = skx_cstates,
1390         .disable_promotion_to_c1e = true,
1391         .use_acpi = true,
1392 };
1393
1394 static const struct idle_cpu idle_cpu_icx __initconst = {
1395         .state_table = icx_cstates,
1396         .disable_promotion_to_c1e = true,
1397         .use_acpi = true,
1398 };
1399
1400 static const struct idle_cpu idle_cpu_adl __initconst = {
1401         .state_table = adl_cstates,
1402 };
1403
1404 static const struct idle_cpu idle_cpu_adl_l __initconst = {
1405         .state_table = adl_l_cstates,
1406 };
1407
1408 static const struct idle_cpu idle_cpu_adl_n __initconst = {
1409         .state_table = adl_n_cstates,
1410 };
1411
1412 static const struct idle_cpu idle_cpu_spr __initconst = {
1413         .state_table = spr_cstates,
1414         .disable_promotion_to_c1e = true,
1415         .use_acpi = true,
1416 };
1417
1418 static const struct idle_cpu idle_cpu_avn __initconst = {
1419         .state_table = avn_cstates,
1420         .disable_promotion_to_c1e = true,
1421         .use_acpi = true,
1422 };
1423
1424 static const struct idle_cpu idle_cpu_knl __initconst = {
1425         .state_table = knl_cstates,
1426         .use_acpi = true,
1427 };
1428
1429 static const struct idle_cpu idle_cpu_bxt __initconst = {
1430         .state_table = bxt_cstates,
1431         .disable_promotion_to_c1e = true,
1432 };
1433
1434 static const struct idle_cpu idle_cpu_dnv __initconst = {
1435         .state_table = dnv_cstates,
1436         .disable_promotion_to_c1e = true,
1437         .use_acpi = true,
1438 };
1439
1440 static const struct idle_cpu idle_cpu_snr __initconst = {
1441         .state_table = snr_cstates,
1442         .disable_promotion_to_c1e = true,
1443         .use_acpi = true,
1444 };
1445
1446 static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1447         X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP,          &idle_cpu_nhx),
1448         X86_MATCH_INTEL_FAM6_MODEL(NEHALEM,             &idle_cpu_nehalem),
1449         X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G,           &idle_cpu_nehalem),
1450         X86_MATCH_INTEL_FAM6_MODEL(WESTMERE,            &idle_cpu_nehalem),
1451         X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP,         &idle_cpu_nhx),
1452         X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX,          &idle_cpu_nhx),
1453         X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL,        &idle_cpu_atom),
1454         X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID,    &idle_cpu_lincroft),
1455         X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX,         &idle_cpu_nhx),
1456         X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE,         &idle_cpu_snb),
1457         X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X,       &idle_cpu_snx),
1458         X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL,       &idle_cpu_atom),
1459         X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT,     &idle_cpu_byt),
1460         X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &idle_cpu_tangier),
1461         X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT,        &idle_cpu_cht),
1462         X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE,           &idle_cpu_ivb),
1463         X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X,         &idle_cpu_ivt),
1464         X86_MATCH_INTEL_FAM6_MODEL(HASWELL,             &idle_cpu_hsw),
1465         X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X,           &idle_cpu_hsx),
1466         X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L,           &idle_cpu_hsw),
1467         X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G,           &idle_cpu_hsw),
1468         X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D,   &idle_cpu_avn),
1469         X86_MATCH_INTEL_FAM6_MODEL(BROADWELL,           &idle_cpu_bdw),
1470         X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G,         &idle_cpu_bdw),
1471         X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X,         &idle_cpu_bdx),
1472         X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D,         &idle_cpu_bdx),
1473         X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L,           &idle_cpu_skl),
1474         X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE,             &idle_cpu_skl),
1475         X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L,          &idle_cpu_skl),
1476         X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE,            &idle_cpu_skl),
1477         X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X,           &idle_cpu_skx),
1478         X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X,           &idle_cpu_icx),
1479         X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D,           &idle_cpu_icx),
1480         X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE,           &idle_cpu_adl),
1481         X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L,         &idle_cpu_adl_l),
1482         X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N,         &idle_cpu_adl_n),
1483         X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X,    &idle_cpu_spr),
1484         X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X,     &idle_cpu_spr),
1485         X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL,        &idle_cpu_knl),
1486         X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM,        &idle_cpu_knl),
1487         X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT,       &idle_cpu_bxt),
1488         X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS,  &idle_cpu_bxt),
1489         X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D,     &idle_cpu_dnv),
1490         X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D,      &idle_cpu_snr),
1491         {}
1492 };
1493
1494 static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1495         X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL),
1496         {}
1497 };
1498
1499 static bool __init intel_idle_max_cstate_reached(int cstate)
1500 {
1501         if (cstate + 1 > max_cstate) {
1502                 pr_info("max_cstate %d reached\n", max_cstate);
1503                 return true;
1504         }
1505         return false;
1506 }
1507
1508 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1509 {
1510         unsigned long eax = flg2MWAIT(state->flags);
1511
1512         if (boot_cpu_has(X86_FEATURE_ARAT))
1513                 return false;
1514
1515         /*
1516          * Switch over to one-shot tick broadcast if the target C-state
1517          * is deeper than C1.
1518          */
1519         return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1520 }
1521
1522 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1523 #include <acpi/processor.h>
1524
1525 static bool no_acpi __read_mostly;
1526 module_param(no_acpi, bool, 0444);
1527 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1528
1529 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1530 module_param_named(use_acpi, force_use_acpi, bool, 0444);
1531 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1532
1533 static struct acpi_processor_power acpi_state_table __initdata;
1534
1535 /**
1536  * intel_idle_cst_usable - Check if the _CST information can be used.
1537  *
1538  * Check if all of the C-states listed by _CST in the max_cstate range are
1539  * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1540  */
1541 static bool __init intel_idle_cst_usable(void)
1542 {
1543         int cstate, limit;
1544
1545         limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1546                       acpi_state_table.count);
1547
1548         for (cstate = 1; cstate < limit; cstate++) {
1549                 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1550
1551                 if (cx->entry_method != ACPI_CSTATE_FFH)
1552                         return false;
1553         }
1554
1555         return true;
1556 }
1557
1558 static bool __init intel_idle_acpi_cst_extract(void)
1559 {
1560         unsigned int cpu;
1561
1562         if (no_acpi) {
1563                 pr_debug("Not allowed to use ACPI _CST\n");
1564                 return false;
1565         }
1566
1567         for_each_possible_cpu(cpu) {
1568                 struct acpi_processor *pr = per_cpu(processors, cpu);
1569
1570                 if (!pr)
1571                         continue;
1572
1573                 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1574                         continue;
1575
1576                 acpi_state_table.count++;
1577
1578                 if (!intel_idle_cst_usable())
1579                         continue;
1580
1581                 if (!acpi_processor_claim_cst_control())
1582                         break;
1583
1584                 return true;
1585         }
1586
1587         acpi_state_table.count = 0;
1588         pr_debug("ACPI _CST not found or not usable\n");
1589         return false;
1590 }
1591
1592 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1593 {
1594         int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1595
1596         /*
1597          * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1598          * the interesting states are ACPI_CSTATE_FFH.
1599          */
1600         for (cstate = 1; cstate < limit; cstate++) {
1601                 struct acpi_processor_cx *cx;
1602                 struct cpuidle_state *state;
1603
1604                 if (intel_idle_max_cstate_reached(cstate - 1))
1605                         break;
1606
1607                 cx = &acpi_state_table.states[cstate];
1608
1609                 state = &drv->states[drv->state_count++];
1610
1611                 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
1612                 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
1613                 state->exit_latency = cx->latency;
1614                 /*
1615                  * For C1-type C-states use the same number for both the exit
1616                  * latency and target residency, because that is the case for
1617                  * C1 in the majority of the static C-states tables above.
1618                  * For the other types of C-states, however, set the target
1619                  * residency to 3 times the exit latency which should lead to
1620                  * a reasonable balance between energy-efficiency and
1621                  * performance in the majority of interesting cases.
1622                  */
1623                 state->target_residency = cx->latency;
1624                 if (cx->type > ACPI_STATE_C1)
1625                         state->target_residency *= 3;
1626
1627                 state->flags = MWAIT2flg(cx->address);
1628                 if (cx->type > ACPI_STATE_C2)
1629                         state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1630
1631                 if (disabled_states_mask & BIT(cstate))
1632                         state->flags |= CPUIDLE_FLAG_OFF;
1633
1634                 if (intel_idle_state_needs_timer_stop(state))
1635                         state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1636
1637                 state->enter = intel_idle;
1638                 state->enter_s2idle = intel_idle_s2idle;
1639         }
1640 }
1641
1642 static bool __init intel_idle_off_by_default(u32 mwait_hint)
1643 {
1644         int cstate, limit;
1645
1646         /*
1647          * If there are no _CST C-states, do not disable any C-states by
1648          * default.
1649          */
1650         if (!acpi_state_table.count)
1651                 return false;
1652
1653         limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1654         /*
1655          * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1656          * the interesting states are ACPI_CSTATE_FFH.
1657          */
1658         for (cstate = 1; cstate < limit; cstate++) {
1659                 if (acpi_state_table.states[cstate].address == mwait_hint)
1660                         return false;
1661         }
1662         return true;
1663 }
1664 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1665 #define force_use_acpi  (false)
1666
1667 static inline bool intel_idle_acpi_cst_extract(void) { return false; }
1668 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
1669 static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; }
1670 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1671
1672 /**
1673  * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1674  *
1675  * Tune IVT multi-socket targets.
1676  * Assumption: num_sockets == (max_package_num + 1).
1677  */
1678 static void __init ivt_idle_state_table_update(void)
1679 {
1680         /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1681         int cpu, package_num, num_sockets = 1;
1682
1683         for_each_online_cpu(cpu) {
1684                 package_num = topology_physical_package_id(cpu);
1685                 if (package_num + 1 > num_sockets) {
1686                         num_sockets = package_num + 1;
1687
1688                         if (num_sockets > 4) {
1689                                 cpuidle_state_table = ivt_cstates_8s;
1690                                 return;
1691                         }
1692                 }
1693         }
1694
1695         if (num_sockets > 2)
1696                 cpuidle_state_table = ivt_cstates_4s;
1697
1698         /* else, 1 and 2 socket systems use default ivt_cstates */
1699 }
1700
1701 /**
1702  * irtl_2_usec - IRTL to microseconds conversion.
1703  * @irtl: IRTL MSR value.
1704  *
1705  * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
1706  */
1707 static unsigned long long __init irtl_2_usec(unsigned long long irtl)
1708 {
1709         static const unsigned int irtl_ns_units[] __initconst = {
1710                 1, 32, 1024, 32768, 1048576, 33554432, 0, 0
1711         };
1712         unsigned long long ns;
1713
1714         if (!irtl)
1715                 return 0;
1716
1717         ns = irtl_ns_units[(irtl >> 10) & 0x7];
1718
1719         return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
1720 }
1721
1722 /**
1723  * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1724  *
1725  * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
1726  * definitive maximum latency and use the same value for target_residency.
1727  */
1728 static void __init bxt_idle_state_table_update(void)
1729 {
1730         unsigned long long msr;
1731         unsigned int usec;
1732
1733         rdmsrl(MSR_PKGC6_IRTL, msr);
1734         usec = irtl_2_usec(msr);
1735         if (usec) {
1736                 bxt_cstates[2].exit_latency = usec;
1737                 bxt_cstates[2].target_residency = usec;
1738         }
1739
1740         rdmsrl(MSR_PKGC7_IRTL, msr);
1741         usec = irtl_2_usec(msr);
1742         if (usec) {
1743                 bxt_cstates[3].exit_latency = usec;
1744                 bxt_cstates[3].target_residency = usec;
1745         }
1746
1747         rdmsrl(MSR_PKGC8_IRTL, msr);
1748         usec = irtl_2_usec(msr);
1749         if (usec) {
1750                 bxt_cstates[4].exit_latency = usec;
1751                 bxt_cstates[4].target_residency = usec;
1752         }
1753
1754         rdmsrl(MSR_PKGC9_IRTL, msr);
1755         usec = irtl_2_usec(msr);
1756         if (usec) {
1757                 bxt_cstates[5].exit_latency = usec;
1758                 bxt_cstates[5].target_residency = usec;
1759         }
1760
1761         rdmsrl(MSR_PKGC10_IRTL, msr);
1762         usec = irtl_2_usec(msr);
1763         if (usec) {
1764                 bxt_cstates[6].exit_latency = usec;
1765                 bxt_cstates[6].target_residency = usec;
1766         }
1767
1768 }
1769
1770 /**
1771  * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1772  *
1773  * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1774  */
1775 static void __init sklh_idle_state_table_update(void)
1776 {
1777         unsigned long long msr;
1778         unsigned int eax, ebx, ecx, edx;
1779
1780
1781         /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1782         if (max_cstate <= 7)
1783                 return;
1784
1785         /* if PC10 not present in CPUID.MWAIT.EDX */
1786         if ((mwait_substates & (0xF << 28)) == 0)
1787                 return;
1788
1789         rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1790
1791         /* PC10 is not enabled in PKG C-state limit */
1792         if ((msr & 0xF) != 8)
1793                 return;
1794
1795         ecx = 0;
1796         cpuid(7, &eax, &ebx, &ecx, &edx);
1797
1798         /* if SGX is present */
1799         if (ebx & (1 << 2)) {
1800
1801                 rdmsrl(MSR_IA32_FEAT_CTL, msr);
1802
1803                 /* if SGX is enabled */
1804                 if (msr & (1 << 18))
1805                         return;
1806         }
1807
1808         skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE;  /* C8-SKL */
1809         skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE;  /* C9-SKL */
1810 }
1811
1812 /**
1813  * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1814  * idle states table.
1815  */
1816 static void __init skx_idle_state_table_update(void)
1817 {
1818         unsigned long long msr;
1819
1820         rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1821
1822         /*
1823          * 000b: C0/C1 (no package C-state support)
1824          * 001b: C2
1825          * 010b: C6 (non-retention)
1826          * 011b: C6 (retention)
1827          * 111b: No Package C state limits.
1828          */
1829         if ((msr & 0x7) < 2) {
1830                 /*
1831                  * Uses the CC6 + PC0 latency and 3 times of
1832                  * latency for target_residency if the PC6
1833                  * is disabled in BIOS. This is consistent
1834                  * with how intel_idle driver uses _CST
1835                  * to set the target_residency.
1836                  */
1837                 skx_cstates[2].exit_latency = 92;
1838                 skx_cstates[2].target_residency = 276;
1839         }
1840 }
1841
1842 /**
1843  * adl_idle_state_table_update - Adjust AlderLake idle states table.
1844  */
1845 static void __init adl_idle_state_table_update(void)
1846 {
1847         /* Check if user prefers C1 over C1E. */
1848         if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) {
1849                 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE;
1850                 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE;
1851
1852                 /* Disable C1E by clearing the "C1E promotion" bit. */
1853                 c1e_promotion = C1E_PROMOTION_DISABLE;
1854                 return;
1855         }
1856
1857         /* Make sure C1E is enabled by default */
1858         c1e_promotion = C1E_PROMOTION_ENABLE;
1859 }
1860
1861 /**
1862  * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
1863  */
1864 static void __init spr_idle_state_table_update(void)
1865 {
1866         unsigned long long msr;
1867
1868         /*
1869          * By default, the C6 state assumes the worst-case scenario of package
1870          * C6. However, if PC6 is disabled, we update the numbers to match
1871          * core C6.
1872          */
1873         rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1874
1875         /* Limit value 2 and above allow for PC6. */
1876         if ((msr & 0x7) < 2) {
1877                 spr_cstates[2].exit_latency = 190;
1878                 spr_cstates[2].target_residency = 600;
1879         }
1880 }
1881
1882 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
1883 {
1884         unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1;
1885         unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
1886                                         MWAIT_SUBSTATE_MASK;
1887
1888         /* Ignore the C-state if there are NO sub-states in CPUID for it. */
1889         if (num_substates == 0)
1890                 return false;
1891
1892         if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1893                 mark_tsc_unstable("TSC halts in idle states deeper than C2");
1894
1895         return true;
1896 }
1897
1898 static void state_update_enter_method(struct cpuidle_state *state, int cstate)
1899 {
1900         if (state->enter == intel_idle_hlt) {
1901                 if (force_irq_on) {
1902                         pr_info("forced intel_idle_irq for state %d\n", cstate);
1903                         state->enter = intel_idle_hlt_irq_on;
1904                 }
1905                 return;
1906         }
1907         if (state->enter == intel_idle_hlt_irq_on)
1908                 return; /* no update scenarios */
1909
1910         if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
1911                 /*
1912                  * Combining with XSTATE with IBRS or IRQ_ENABLE flags
1913                  * is not currently supported but this driver.
1914                  */
1915                 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
1916                 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
1917                 state->enter = intel_idle_xstate;
1918                 return;
1919         }
1920
1921         if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
1922                            state->flags & CPUIDLE_FLAG_IBRS) {
1923                 /*
1924                  * IBRS mitigation requires that C-states are entered
1925                  * with interrupts disabled.
1926                  */
1927                 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
1928                 state->enter = intel_idle_ibrs;
1929                 return;
1930         }
1931
1932         if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
1933                 state->enter = intel_idle_irq;
1934                 return;
1935         }
1936
1937         if (force_irq_on) {
1938                 pr_info("forced intel_idle_irq for state %d\n", cstate);
1939                 state->enter = intel_idle_irq;
1940         }
1941 }
1942
1943 /*
1944  * For mwait based states, we want to verify the cpuid data to see if the state
1945  * is actually supported by this specific CPU.
1946  * For non-mwait based states, this check should be skipped.
1947  */
1948 static bool should_verify_mwait(struct cpuidle_state *state)
1949 {
1950         if (state->enter == intel_idle_hlt)
1951                 return false;
1952         if (state->enter == intel_idle_hlt_irq_on)
1953                 return false;
1954
1955         return true;
1956 }
1957
1958 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
1959 {
1960         int cstate;
1961
1962         switch (boot_cpu_data.x86_model) {
1963         case INTEL_FAM6_IVYBRIDGE_X:
1964                 ivt_idle_state_table_update();
1965                 break;
1966         case INTEL_FAM6_ATOM_GOLDMONT:
1967         case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
1968                 bxt_idle_state_table_update();
1969                 break;
1970         case INTEL_FAM6_SKYLAKE:
1971                 sklh_idle_state_table_update();
1972                 break;
1973         case INTEL_FAM6_SKYLAKE_X:
1974                 skx_idle_state_table_update();
1975                 break;
1976         case INTEL_FAM6_SAPPHIRERAPIDS_X:
1977         case INTEL_FAM6_EMERALDRAPIDS_X:
1978                 spr_idle_state_table_update();
1979                 break;
1980         case INTEL_FAM6_ALDERLAKE:
1981         case INTEL_FAM6_ALDERLAKE_L:
1982         case INTEL_FAM6_ALDERLAKE_N:
1983                 adl_idle_state_table_update();
1984                 break;
1985         }
1986
1987         for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
1988                 struct cpuidle_state *state;
1989                 unsigned int mwait_hint;
1990
1991                 if (intel_idle_max_cstate_reached(cstate))
1992                         break;
1993
1994                 if (!cpuidle_state_table[cstate].enter &&
1995                     !cpuidle_state_table[cstate].enter_s2idle)
1996                         break;
1997
1998                 /* If marked as unusable, skip this state. */
1999                 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
2000                         pr_debug("state %s is disabled\n",
2001                                  cpuidle_state_table[cstate].name);
2002                         continue;
2003                 }
2004
2005                 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
2006                 if (should_verify_mwait(&cpuidle_state_table[cstate]) && !intel_idle_verify_cstate(mwait_hint))
2007                         continue;
2008
2009                 /* Structure copy. */
2010                 drv->states[drv->state_count] = cpuidle_state_table[cstate];
2011                 state = &drv->states[drv->state_count];
2012
2013                 state_update_enter_method(state, cstate);
2014
2015
2016                 if ((disabled_states_mask & BIT(drv->state_count)) ||
2017                     ((icpu->use_acpi || force_use_acpi) &&
2018                      intel_idle_off_by_default(mwait_hint) &&
2019                      !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
2020                         state->flags |= CPUIDLE_FLAG_OFF;
2021
2022                 if (intel_idle_state_needs_timer_stop(state))
2023                         state->flags |= CPUIDLE_FLAG_TIMER_STOP;
2024
2025                 drv->state_count++;
2026         }
2027
2028         if (icpu->byt_auto_demotion_disable_flag) {
2029                 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
2030                 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
2031         }
2032 }
2033
2034 /**
2035  * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2036  * @drv: cpuidle driver structure to initialize.
2037  */
2038 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
2039 {
2040         cpuidle_poll_state_init(drv);
2041
2042         if (disabled_states_mask & BIT(0))
2043                 drv->states[0].flags |= CPUIDLE_FLAG_OFF;
2044
2045         drv->state_count = 1;
2046
2047         if (icpu)
2048                 intel_idle_init_cstates_icpu(drv);
2049         else
2050                 intel_idle_init_cstates_acpi(drv);
2051 }
2052
2053 static void auto_demotion_disable(void)
2054 {
2055         unsigned long long msr_bits;
2056
2057         rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2058         msr_bits &= ~auto_demotion_disable_flags;
2059         wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2060 }
2061
2062 static void c1e_promotion_enable(void)
2063 {
2064         unsigned long long msr_bits;
2065
2066         rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2067         msr_bits |= 0x2;
2068         wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2069 }
2070
2071 static void c1e_promotion_disable(void)
2072 {
2073         unsigned long long msr_bits;
2074
2075         rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2076         msr_bits &= ~0x2;
2077         wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2078 }
2079
2080 /**
2081  * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2082  * @cpu: CPU to initialize.
2083  *
2084  * Register a cpuidle device object for @cpu and update its MSRs in accordance
2085  * with the processor model flags.
2086  */
2087 static int intel_idle_cpu_init(unsigned int cpu)
2088 {
2089         struct cpuidle_device *dev;
2090
2091         dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2092         dev->cpu = cpu;
2093
2094         if (cpuidle_register_device(dev)) {
2095                 pr_debug("cpuidle_register_device %d failed!\n", cpu);
2096                 return -EIO;
2097         }
2098
2099         if (auto_demotion_disable_flags)
2100                 auto_demotion_disable();
2101
2102         if (c1e_promotion == C1E_PROMOTION_ENABLE)
2103                 c1e_promotion_enable();
2104         else if (c1e_promotion == C1E_PROMOTION_DISABLE)
2105                 c1e_promotion_disable();
2106
2107         return 0;
2108 }
2109
2110 static int intel_idle_cpu_online(unsigned int cpu)
2111 {
2112         struct cpuidle_device *dev;
2113
2114         if (!boot_cpu_has(X86_FEATURE_ARAT))
2115                 tick_broadcast_enable();
2116
2117         /*
2118          * Some systems can hotplug a cpu at runtime after
2119          * the kernel has booted, we have to initialize the
2120          * driver in this case
2121          */
2122         dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2123         if (!dev->registered)
2124                 return intel_idle_cpu_init(cpu);
2125
2126         return 0;
2127 }
2128
2129 /**
2130  * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2131  */
2132 static void __init intel_idle_cpuidle_devices_uninit(void)
2133 {
2134         int i;
2135
2136         for_each_online_cpu(i)
2137                 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2138 }
2139
2140 /*
2141  * Match up the latency and break even point of the bare metal (cpu based)
2142  * states with the deepest VM available state.
2143  *
2144  * We only want to do this for the deepest state, the ones that has
2145  * the TLB_FLUSHED flag set on the .
2146  *
2147  * All our short idle states are dominated by vmexit/vmenter latencies,
2148  * not the underlying hardware latencies so we keep our values for these.
2149  */
2150 static void matchup_vm_state_with_baremetal(void)
2151 {
2152         int cstate;
2153
2154         for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
2155                 int matching_cstate;
2156
2157                 if (intel_idle_max_cstate_reached(cstate))
2158                         break;
2159
2160                 if (!cpuidle_state_table[cstate].enter)
2161                         break;
2162
2163                 if (!(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_TLB_FLUSHED))
2164                         continue;
2165
2166                 for (matching_cstate = 0; matching_cstate < CPUIDLE_STATE_MAX; ++matching_cstate) {
2167                         if (!icpu->state_table[matching_cstate].enter)
2168                                 break;
2169                         if (icpu->state_table[matching_cstate].exit_latency > cpuidle_state_table[cstate].exit_latency) {
2170                                 cpuidle_state_table[cstate].exit_latency = icpu->state_table[matching_cstate].exit_latency;
2171                                 cpuidle_state_table[cstate].target_residency = icpu->state_table[matching_cstate].target_residency;
2172                         }
2173                 }
2174
2175         }
2176 }
2177
2178
2179 static int __init intel_idle_vminit(const struct x86_cpu_id *id)
2180 {
2181         int retval;
2182
2183         cpuidle_state_table = vmguest_cstates;
2184
2185         icpu = (const struct idle_cpu *)id->driver_data;
2186
2187         pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
2188                  boot_cpu_data.x86_model);
2189
2190         intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2191         if (!intel_idle_cpuidle_devices)
2192                 return -ENOMEM;
2193
2194         /*
2195          * We don't know exactly what the host will do when we go idle, but as a worst estimate
2196          * we can assume that the exit latency of the deepest host state will be hit for our
2197          * deep (long duration) guest idle state.
2198          * The same logic applies to the break even point for the long duration guest idle state.
2199          * So lets copy these two properties from the table we found for the host CPU type.
2200          */
2201         matchup_vm_state_with_baremetal();
2202
2203         intel_idle_cpuidle_driver_init(&intel_idle_driver);
2204
2205         retval = cpuidle_register_driver(&intel_idle_driver);
2206         if (retval) {
2207                 struct cpuidle_driver *drv = cpuidle_get_driver();
2208                 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2209                        drv ? drv->name : "none");
2210                 goto init_driver_fail;
2211         }
2212
2213         retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2214                                    intel_idle_cpu_online, NULL);
2215         if (retval < 0)
2216                 goto hp_setup_fail;
2217
2218         return 0;
2219 hp_setup_fail:
2220         intel_idle_cpuidle_devices_uninit();
2221         cpuidle_unregister_driver(&intel_idle_driver);
2222 init_driver_fail:
2223         free_percpu(intel_idle_cpuidle_devices);
2224         return retval;
2225 }
2226
2227 static int __init intel_idle_init(void)
2228 {
2229         const struct x86_cpu_id *id;
2230         unsigned int eax, ebx, ecx;
2231         int retval;
2232
2233         /* Do not load intel_idle at all for now if idle= is passed */
2234         if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2235                 return -ENODEV;
2236
2237         if (max_cstate == 0) {
2238                 pr_debug("disabled\n");
2239                 return -EPERM;
2240         }
2241
2242         id = x86_match_cpu(intel_idle_ids);
2243         if (id) {
2244                 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2245                         if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
2246                                 return intel_idle_vminit(id);
2247                         pr_debug("Please enable MWAIT in BIOS SETUP\n");
2248                         return -ENODEV;
2249                 }
2250         } else {
2251                 id = x86_match_cpu(intel_mwait_ids);
2252                 if (!id)
2253                         return -ENODEV;
2254         }
2255
2256         if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
2257                 return -ENODEV;
2258
2259         cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
2260
2261         if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2262             !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2263             !mwait_substates)
2264                         return -ENODEV;
2265
2266         pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2267
2268         icpu = (const struct idle_cpu *)id->driver_data;
2269         if (icpu) {
2270                 cpuidle_state_table = icpu->state_table;
2271                 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
2272                 if (icpu->disable_promotion_to_c1e)
2273                         c1e_promotion = C1E_PROMOTION_DISABLE;
2274                 if (icpu->use_acpi || force_use_acpi)
2275                         intel_idle_acpi_cst_extract();
2276         } else if (!intel_idle_acpi_cst_extract()) {
2277                 return -ENODEV;
2278         }
2279
2280         pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
2281                  boot_cpu_data.x86_model);
2282
2283         intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2284         if (!intel_idle_cpuidle_devices)
2285                 return -ENOMEM;
2286
2287         intel_idle_cpuidle_driver_init(&intel_idle_driver);
2288
2289         retval = cpuidle_register_driver(&intel_idle_driver);
2290         if (retval) {
2291                 struct cpuidle_driver *drv = cpuidle_get_driver();
2292                 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2293                        drv ? drv->name : "none");
2294                 goto init_driver_fail;
2295         }
2296
2297         retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2298                                    intel_idle_cpu_online, NULL);
2299         if (retval < 0)
2300                 goto hp_setup_fail;
2301
2302         pr_debug("Local APIC timer is reliable in %s\n",
2303                  boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2304
2305         return 0;
2306
2307 hp_setup_fail:
2308         intel_idle_cpuidle_devices_uninit();
2309         cpuidle_unregister_driver(&intel_idle_driver);
2310 init_driver_fail:
2311         free_percpu(intel_idle_cpuidle_devices);
2312         return retval;
2313
2314 }
2315 device_initcall(intel_idle_init);
2316
2317 /*
2318  * We are not really modular, but we used to support that.  Meaning we also
2319  * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2320  * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2321  * is the easiest way (currently) to continue doing that.
2322  */
2323 module_param(max_cstate, int, 0444);
2324 /*
2325  * The positions of the bits that are set in this number are the indices of the
2326  * idle states to be disabled by default (as reflected by the names of the
2327  * corresponding idle state directories in sysfs, "state0", "state1" ...
2328  * "state<i>" ..., where <i> is the index of the given state).
2329  */
2330 module_param_named(states_off, disabled_states_mask, uint, 0444);
2331 MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2332 /*
2333  * Some platforms come with mutually exclusive C-states, so that if one is
2334  * enabled, the other C-states must not be used. Example: C1 and C1E on
2335  * Sapphire Rapids platform. This parameter allows for selecting the
2336  * preferred C-states among the groups of mutually exclusive C-states - the
2337  * selected C-states will be registered, the other C-states from the mutually
2338  * exclusive group won't be registered. If the platform has no mutually
2339  * exclusive C-states, this parameter has no effect.
2340  */
2341 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444);
2342 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
2343 /*
2344  * Debugging option that forces the driver to enter all C-states with
2345  * interrupts enabled. Does not apply to C-states with
2346  * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2347  */
2348 module_param(force_irq_on, bool, 0444);