drivers/idle/intel_idle.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_idle.c - native hardware idle loop for modern Intel processors
   4  *
   5  * Copyright (c) 2013 - 2020, Intel Corporation.
   6  * Len Brown <[email protected]>
   7  * Rafael J. Wysocki <[email protected]>
   8  */
   9
  10 /*
  11  * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
  12  * in lieu of the legacy ACPI processor_idle driver.  The intent is to
  13  * make Linux more efficient on these processors, as intel_idle knows
  14  * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
  15  */
  16
  17 /*
  18  * Design Assumptions
  19  *
  20  * All CPUs have same idle states as boot CPU
  21  *
  22  * Chipset BM_STS (bus master status) bit is a NOP
  23  *      for preventing entry into deep C-states
  24  *
  25  * CPU will flush caches as needed when entering a C-state via MWAIT
  26  *      (in contrast to entering ACPI C3, in which case the WBINVD
  27  *      instruction needs to be executed to flush the caches)
  28  */
  29
  30 /*
  31  * Known limitations
  32  *
  33  * ACPI has a .suspend hack to turn off deep c-statees during suspend
  34  * to avoid complications with the lapic timer workaround.
  35  * Have not seen issues with suspend, but may need same workaround here.
  36  *
  37  */
  38
  39 /* un-comment DEBUG to enable pr_debug() statements */
  40 /* #define DEBUG */
  41
  42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  43
  44 #include <linux/acpi.h>
  45 #include <linux/kernel.h>
  46 #include <linux/cpuidle.h>
  47 #include <linux/tick.h>
  48 #include <trace/events/power.h>
  49 #include <linux/sched.h>
  50 #include <linux/sched/smt.h>
  51 #include <linux/notifier.h>
  52 #include <linux/cpu.h>
  53 #include <linux/moduleparam.h>
  54 #include <asm/cpuid.h>
  55 #include <asm/cpu_device_id.h>
  56 #include <asm/intel-family.h>
  57 #include <asm/mwait.h>
  58 #include <asm/spec-ctrl.h>
  59 #include <asm/fpu/api.h>
  60
  61 #define INTEL_IDLE_VERSION "0.5.1"
  62
  63 static struct cpuidle_driver intel_idle_driver = {
  64         .name = "intel_idle",
  65         .owner = THIS_MODULE,
  66 };
  67 /* intel_idle.max_cstate=0 disables driver */
  68 static int max_cstate = CPUIDLE_STATE_MAX - 1;
  69 static unsigned int disabled_states_mask __read_mostly;
  70 static unsigned int preferred_states_mask __read_mostly;
  71 static bool force_irq_on __read_mostly;
  72 static bool ibrs_off __read_mostly;
  73
  74 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
  75
  76 static unsigned long auto_demotion_disable_flags;
  77
  78 static enum {
  79         C1E_PROMOTION_PRESERVE,
  80         C1E_PROMOTION_ENABLE,
  81         C1E_PROMOTION_DISABLE
  82 } c1e_promotion = C1E_PROMOTION_PRESERVE;
  83
  84 struct idle_cpu {
  85         struct cpuidle_state *state_table;
  86
  87         /*
  88          * Hardware C-state auto-demotion may not always be optimal.
  89          * Indicate which enable bits to clear here.
  90          */
  91         unsigned long auto_demotion_disable_flags;
  92         bool byt_auto_demotion_disable_flag;
  93         bool disable_promotion_to_c1e;
  94         bool use_acpi;
  95 };
  96
  97 static const struct idle_cpu *icpu __initdata;
  98 static struct cpuidle_state *cpuidle_state_table __initdata;
  99
 100 static unsigned int mwait_substates __initdata;
 101
 102 /*
 103  * Enable interrupts before entering the C-state. On some platforms and for
 104  * some C-states, this may measurably decrease interrupt latency.
 105  */
 106 #define CPUIDLE_FLAG_IRQ_ENABLE         BIT(14)
 107
 108 /*
 109  * Enable this state by default even if the ACPI _CST does not list it.
 110  */
 111 #define CPUIDLE_FLAG_ALWAYS_ENABLE      BIT(15)
 112
 113 /*
 114  * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
 115  * above.
 116  */
 117 #define CPUIDLE_FLAG_IBRS               BIT(16)
 118
 119 /*
 120  * Initialize large xstate for the C6-state entrance.
 121  */
 122 #define CPUIDLE_FLAG_INIT_XSTATE        BIT(17)
 123
 124 /*
 125  * Ignore the sub-state when matching mwait hints between the ACPI _CST and
 126  * custom tables.
 127  */
 128 #define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18)
 129
 130 /*
 131  * MWAIT takes an 8-bit "hint" in EAX "suggesting"
 132  * the C-state (top nibble) and sub-state (bottom nibble)
 133  * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
 134  *
 135  * We store the hint at the top of our "flags" for each state.
 136  */
 137 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
 138 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
 139
 140 static __always_inline int __intel_idle(struct cpuidle_device *dev,
 141                                         struct cpuidle_driver *drv,
 142                                         int index, bool irqoff)
 143 {
 144         struct cpuidle_state *state = &drv->states[index];
 145         unsigned long eax = flg2MWAIT(state->flags);
 146         unsigned long ecx = 1*irqoff; /* break on interrupt flag */
 147
 148         mwait_idle_with_hints(eax, ecx);
 149
 150         return index;
 151 }
 152
 153 /**
 154  * intel_idle - Ask the processor to enter the given idle state.
 155  * @dev: cpuidle device of the target CPU.
 156  * @drv: cpuidle driver (assumed to point to intel_idle_driver).
 157  * @index: Target idle state index.
 158  *
 159  * Use the MWAIT instruction to notify the processor that the CPU represented by
 160  * @dev is idle and it can try to enter the idle state corresponding to @index.
 161  *
 162  * If the local APIC timer is not known to be reliable in the target idle state,
 163  * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
 164  *
 165  * Must be called under local_irq_disable().
 166  */
 167 static __cpuidle int intel_idle(struct cpuidle_device *dev,
 168                                 struct cpuidle_driver *drv, int index)
 169 {
 170         return __intel_idle(dev, drv, index, true);
 171 }
 172
 173 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
 174                                     struct cpuidle_driver *drv, int index)
 175 {
 176         return __intel_idle(dev, drv, index, false);
 177 }
 178
 179 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
 180                                      struct cpuidle_driver *drv, int index)
 181 {
 182         bool smt_active = sched_smt_active();
 183         u64 spec_ctrl = spec_ctrl_current();
 184         int ret;
 185
 186         if (smt_active)
 187                 __update_spec_ctrl(0);
 188
 189         ret = __intel_idle(dev, drv, index, true);
 190
 191         if (smt_active)
 192                 __update_spec_ctrl(spec_ctrl);
 193
 194         return ret;
 195 }
 196
 197 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
 198                                        struct cpuidle_driver *drv, int index)
 199 {
 200         fpu_idle_fpregs();
 201         return __intel_idle(dev, drv, index, true);
 202 }
 203
 204 /**
 205  * intel_idle_s2idle - Ask the processor to enter the given idle state.
 206  * @dev: cpuidle device of the target CPU.
 207  * @drv: cpuidle driver (assumed to point to intel_idle_driver).
 208  * @index: Target idle state index.
 209  *
 210  * Use the MWAIT instruction to notify the processor that the CPU represented by
 211  * @dev is idle and it can try to enter the idle state corresponding to @index.
 212  *
 213  * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
 214  * scheduler tick and suspended scheduler clock on the target CPU.
 215  */
 216 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
 217                                        struct cpuidle_driver *drv, int index)
 218 {
 219         unsigned long ecx = 1; /* break on interrupt flag */
 220         struct cpuidle_state *state = &drv->states[index];
 221         unsigned long eax = flg2MWAIT(state->flags);
 222
 223         if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
 224                 fpu_idle_fpregs();
 225
 226         mwait_idle_with_hints(eax, ecx);
 227
 228         return 0;
 229 }
 230
 231 /*
 232  * States are indexed by the cstate number,
 233  * which is also the index into the MWAIT hint array.
 234  * Thus C0 is a dummy.
 235  */
 236 static struct cpuidle_state nehalem_cstates[] __initdata = {
 237         {
 238                 .name = "C1",
 239                 .desc = "MWAIT 0x00",
 240                 .flags = MWAIT2flg(0x00),
 241                 .exit_latency = 3,
 242                 .target_residency = 6,
 243                 .enter = &intel_idle,
 244                 .enter_s2idle = intel_idle_s2idle, },
 245         {
 246                 .name = "C1E",
 247                 .desc = "MWAIT 0x01",
 248                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 249                 .exit_latency = 10,
 250                 .target_residency = 20,
 251                 .enter = &intel_idle,
 252                 .enter_s2idle = intel_idle_s2idle, },
 253         {
 254                 .name = "C3",
 255                 .desc = "MWAIT 0x10",
 256                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 257                 .exit_latency = 20,
 258                 .target_residency = 80,
 259                 .enter = &intel_idle,
 260                 .enter_s2idle = intel_idle_s2idle, },
 261         {
 262                 .name = "C6",
 263                 .desc = "MWAIT 0x20",
 264                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 265                 .exit_latency = 200,
 266                 .target_residency = 800,
 267                 .enter = &intel_idle,
 268                 .enter_s2idle = intel_idle_s2idle, },
 269         {
 270                 .enter = NULL }
 271 };
 272
 273 static struct cpuidle_state snb_cstates[] __initdata = {
 274         {
 275                 .name = "C1",
 276                 .desc = "MWAIT 0x00",
 277                 .flags = MWAIT2flg(0x00),
 278                 .exit_latency = 2,
 279                 .target_residency = 2,
 280                 .enter = &intel_idle,
 281                 .enter_s2idle = intel_idle_s2idle, },
 282         {
 283                 .name = "C1E",
 284                 .desc = "MWAIT 0x01",
 285                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 286                 .exit_latency = 10,
 287                 .target_residency = 20,
 288                 .enter = &intel_idle,
 289                 .enter_s2idle = intel_idle_s2idle, },
 290         {
 291                 .name = "C3",
 292                 .desc = "MWAIT 0x10",
 293                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 294                 .exit_latency = 80,
 295                 .target_residency = 211,
 296                 .enter = &intel_idle,
 297                 .enter_s2idle = intel_idle_s2idle, },
 298         {
 299                 .name = "C6",
 300                 .desc = "MWAIT 0x20",
 301                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 302                 .exit_latency = 104,
 303                 .target_residency = 345,
 304                 .enter = &intel_idle,
 305                 .enter_s2idle = intel_idle_s2idle, },
 306         {
 307                 .name = "C7",
 308                 .desc = "MWAIT 0x30",
 309                 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
 310                 .exit_latency = 109,
 311                 .target_residency = 345,
 312                 .enter = &intel_idle,
 313                 .enter_s2idle = intel_idle_s2idle, },
 314         {
 315                 .enter = NULL }
 316 };
 317
 318 static struct cpuidle_state byt_cstates[] __initdata = {
 319         {
 320                 .name = "C1",
 321                 .desc = "MWAIT 0x00",
 322                 .flags = MWAIT2flg(0x00),
 323                 .exit_latency = 1,
 324                 .target_residency = 1,
 325                 .enter = &intel_idle,
 326                 .enter_s2idle = intel_idle_s2idle, },
 327         {
 328                 .name = "C6N",
 329                 .desc = "MWAIT 0x58",
 330                 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
 331                 .exit_latency = 300,
 332                 .target_residency = 275,
 333                 .enter = &intel_idle,
 334                 .enter_s2idle = intel_idle_s2idle, },
 335         {
 336                 .name = "C6S",
 337                 .desc = "MWAIT 0x52",
 338                 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
 339                 .exit_latency = 500,
 340                 .target_residency = 560,
 341                 .enter = &intel_idle,
 342                 .enter_s2idle = intel_idle_s2idle, },
 343         {
 344                 .name = "C7",
 345                 .desc = "MWAIT 0x60",
 346                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 347                 .exit_latency = 1200,
 348                 .target_residency = 4000,
 349                 .enter = &intel_idle,
 350                 .enter_s2idle = intel_idle_s2idle, },
 351         {
 352                 .name = "C7S",
 353                 .desc = "MWAIT 0x64",
 354                 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
 355                 .exit_latency = 10000,
 356                 .target_residency = 20000,
 357                 .enter = &intel_idle,
 358                 .enter_s2idle = intel_idle_s2idle, },
 359         {
 360                 .enter = NULL }
 361 };
 362
 363 static struct cpuidle_state cht_cstates[] __initdata = {
 364         {
 365                 .name = "C1",
 366                 .desc = "MWAIT 0x00",
 367                 .flags = MWAIT2flg(0x00),
 368                 .exit_latency = 1,
 369                 .target_residency = 1,
 370                 .enter = &intel_idle,
 371                 .enter_s2idle = intel_idle_s2idle, },
 372         {
 373                 .name = "C6N",
 374                 .desc = "MWAIT 0x58",
 375                 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
 376                 .exit_latency = 80,
 377                 .target_residency = 275,
 378                 .enter = &intel_idle,
 379                 .enter_s2idle = intel_idle_s2idle, },
 380         {
 381                 .name = "C6S",
 382                 .desc = "MWAIT 0x52",
 383                 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
 384                 .exit_latency = 200,
 385                 .target_residency = 560,
 386                 .enter = &intel_idle,
 387                 .enter_s2idle = intel_idle_s2idle, },
 388         {
 389                 .name = "C7",
 390                 .desc = "MWAIT 0x60",
 391                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 392                 .exit_latency = 1200,
 393                 .target_residency = 4000,
 394                 .enter = &intel_idle,
 395                 .enter_s2idle = intel_idle_s2idle, },
 396         {
 397                 .name = "C7S",
 398                 .desc = "MWAIT 0x64",
 399                 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
 400                 .exit_latency = 10000,
 401                 .target_residency = 20000,
 402                 .enter = &intel_idle,
 403                 .enter_s2idle = intel_idle_s2idle, },
 404         {
 405                 .enter = NULL }
 406 };
 407
 408 static struct cpuidle_state ivb_cstates[] __initdata = {
 409         {
 410                 .name = "C1",
 411                 .desc = "MWAIT 0x00",
 412                 .flags = MWAIT2flg(0x00),
 413                 .exit_latency = 1,
 414                 .target_residency = 1,
 415                 .enter = &intel_idle,
 416                 .enter_s2idle = intel_idle_s2idle, },
 417         {
 418                 .name = "C1E",
 419                 .desc = "MWAIT 0x01",
 420                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 421                 .exit_latency = 10,
 422                 .target_residency = 20,
 423                 .enter = &intel_idle,
 424                 .enter_s2idle = intel_idle_s2idle, },
 425         {
 426                 .name = "C3",
 427                 .desc = "MWAIT 0x10",
 428                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 429                 .exit_latency = 59,
 430                 .target_residency = 156,
 431                 .enter = &intel_idle,
 432                 .enter_s2idle = intel_idle_s2idle, },
 433         {
 434                 .name = "C6",
 435                 .desc = "MWAIT 0x20",
 436                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 437                 .exit_latency = 80,
 438                 .target_residency = 300,
 439                 .enter = &intel_idle,
 440                 .enter_s2idle = intel_idle_s2idle, },
 441         {
 442                 .name = "C7",
 443                 .desc = "MWAIT 0x30",
 444                 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
 445                 .exit_latency = 87,
 446                 .target_residency = 300,
 447                 .enter = &intel_idle,
 448                 .enter_s2idle = intel_idle_s2idle, },
 449         {
 450                 .enter = NULL }
 451 };
 452
 453 static struct cpuidle_state ivt_cstates[] __initdata = {
 454         {
 455                 .name = "C1",
 456                 .desc = "MWAIT 0x00",
 457                 .flags = MWAIT2flg(0x00),
 458                 .exit_latency = 1,
 459                 .target_residency = 1,
 460                 .enter = &intel_idle,
 461                 .enter_s2idle = intel_idle_s2idle, },
 462         {
 463                 .name = "C1E",
 464                 .desc = "MWAIT 0x01",
 465                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 466                 .exit_latency = 10,
 467                 .target_residency = 80,
 468                 .enter = &intel_idle,
 469                 .enter_s2idle = intel_idle_s2idle, },
 470         {
 471                 .name = "C3",
 472                 .desc = "MWAIT 0x10",
 473                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 474                 .exit_latency = 59,
 475                 .target_residency = 156,
 476                 .enter = &intel_idle,
 477                 .enter_s2idle = intel_idle_s2idle, },
 478         {
 479                 .name = "C6",
 480                 .desc = "MWAIT 0x20",
 481                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 482                 .exit_latency = 82,
 483                 .target_residency = 300,
 484                 .enter = &intel_idle,
 485                 .enter_s2idle = intel_idle_s2idle, },
 486         {
 487                 .enter = NULL }
 488 };
 489
 490 static struct cpuidle_state ivt_cstates_4s[] __initdata = {
 491         {
 492                 .name = "C1",
 493                 .desc = "MWAIT 0x00",
 494                 .flags = MWAIT2flg(0x00),
 495                 .exit_latency = 1,
 496                 .target_residency = 1,
 497                 .enter = &intel_idle,
 498                 .enter_s2idle = intel_idle_s2idle, },
 499         {
 500                 .name = "C1E",
 501                 .desc = "MWAIT 0x01",
 502                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 503                 .exit_latency = 10,
 504                 .target_residency = 250,
 505                 .enter = &intel_idle,
 506                 .enter_s2idle = intel_idle_s2idle, },
 507         {
 508                 .name = "C3",
 509                 .desc = "MWAIT 0x10",
 510                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 511                 .exit_latency = 59,
 512                 .target_residency = 300,
 513                 .enter = &intel_idle,
 514                 .enter_s2idle = intel_idle_s2idle, },
 515         {
 516                 .name = "C6",
 517                 .desc = "MWAIT 0x20",
 518                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 519                 .exit_latency = 84,
 520                 .target_residency = 400,
 521                 .enter = &intel_idle,
 522                 .enter_s2idle = intel_idle_s2idle, },
 523         {
 524                 .enter = NULL }
 525 };
 526
 527 static struct cpuidle_state ivt_cstates_8s[] __initdata = {
 528         {
 529                 .name = "C1",
 530                 .desc = "MWAIT 0x00",
 531                 .flags = MWAIT2flg(0x00),
 532                 .exit_latency = 1,
 533                 .target_residency = 1,
 534                 .enter = &intel_idle,
 535                 .enter_s2idle = intel_idle_s2idle, },
 536         {
 537                 .name = "C1E",
 538                 .desc = "MWAIT 0x01",
 539                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 540                 .exit_latency = 10,
 541                 .target_residency = 500,
 542                 .enter = &intel_idle,
 543                 .enter_s2idle = intel_idle_s2idle, },
 544         {
 545                 .name = "C3",
 546                 .desc = "MWAIT 0x10",
 547                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 548                 .exit_latency = 59,
 549                 .target_residency = 600,
 550                 .enter = &intel_idle,
 551                 .enter_s2idle = intel_idle_s2idle, },
 552         {
 553                 .name = "C6",
 554                 .desc = "MWAIT 0x20",
 555                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 556                 .exit_latency = 88,
 557                 .target_residency = 700,
 558                 .enter = &intel_idle,
 559                 .enter_s2idle = intel_idle_s2idle, },
 560         {
 561                 .enter = NULL }
 562 };
 563
 564 static struct cpuidle_state hsw_cstates[] __initdata = {
 565         {
 566                 .name = "C1",
 567                 .desc = "MWAIT 0x00",
 568                 .flags = MWAIT2flg(0x00),
 569                 .exit_latency = 2,
 570                 .target_residency = 2,
 571                 .enter = &intel_idle,
 572                 .enter_s2idle = intel_idle_s2idle, },
 573         {
 574                 .name = "C1E",
 575                 .desc = "MWAIT 0x01",
 576                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 577                 .exit_latency = 10,
 578                 .target_residency = 20,
 579                 .enter = &intel_idle,
 580                 .enter_s2idle = intel_idle_s2idle, },
 581         {
 582                 .name = "C3",
 583                 .desc = "MWAIT 0x10",
 584                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 585                 .exit_latency = 33,
 586                 .target_residency = 100,
 587                 .enter = &intel_idle,
 588                 .enter_s2idle = intel_idle_s2idle, },
 589         {
 590                 .name = "C6",
 591                 .desc = "MWAIT 0x20",
 592                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 593                 .exit_latency = 133,
 594                 .target_residency = 400,
 595                 .enter = &intel_idle,
 596                 .enter_s2idle = intel_idle_s2idle, },
 597         {
 598                 .name = "C7s",
 599                 .desc = "MWAIT 0x32",
 600                 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
 601                 .exit_latency = 166,
 602                 .target_residency = 500,
 603                 .enter = &intel_idle,
 604                 .enter_s2idle = intel_idle_s2idle, },
 605         {
 606                 .name = "C8",
 607                 .desc = "MWAIT 0x40",
 608                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 609                 .exit_latency = 300,
 610                 .target_residency = 900,
 611                 .enter = &intel_idle,
 612                 .enter_s2idle = intel_idle_s2idle, },
 613         {
 614                 .name = "C9",
 615                 .desc = "MWAIT 0x50",
 616                 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
 617                 .exit_latency = 600,
 618                 .target_residency = 1800,
 619                 .enter = &intel_idle,
 620                 .enter_s2idle = intel_idle_s2idle, },
 621         {
 622                 .name = "C10",
 623                 .desc = "MWAIT 0x60",
 624                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 625                 .exit_latency = 2600,
 626                 .target_residency = 7700,
 627                 .enter = &intel_idle,
 628                 .enter_s2idle = intel_idle_s2idle, },
 629         {
 630                 .enter = NULL }
 631 };
 632 static struct cpuidle_state bdw_cstates[] __initdata = {
 633         {
 634                 .name = "C1",
 635                 .desc = "MWAIT 0x00",
 636                 .flags = MWAIT2flg(0x00),
 637                 .exit_latency = 2,
 638                 .target_residency = 2,
 639                 .enter = &intel_idle,
 640                 .enter_s2idle = intel_idle_s2idle, },
 641         {
 642                 .name = "C1E",
 643                 .desc = "MWAIT 0x01",
 644                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 645                 .exit_latency = 10,
 646                 .target_residency = 20,
 647                 .enter = &intel_idle,
 648                 .enter_s2idle = intel_idle_s2idle, },
 649         {
 650                 .name = "C3",
 651                 .desc = "MWAIT 0x10",
 652                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 653                 .exit_latency = 40,
 654                 .target_residency = 100,
 655                 .enter = &intel_idle,
 656                 .enter_s2idle = intel_idle_s2idle, },
 657         {
 658                 .name = "C6",
 659                 .desc = "MWAIT 0x20",
 660                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 661                 .exit_latency = 133,
 662                 .target_residency = 400,
 663                 .enter = &intel_idle,
 664                 .enter_s2idle = intel_idle_s2idle, },
 665         {
 666                 .name = "C7s",
 667                 .desc = "MWAIT 0x32",
 668                 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
 669                 .exit_latency = 166,
 670                 .target_residency = 500,
 671                 .enter = &intel_idle,
 672                 .enter_s2idle = intel_idle_s2idle, },
 673         {
 674                 .name = "C8",
 675                 .desc = "MWAIT 0x40",
 676                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 677                 .exit_latency = 300,
 678                 .target_residency = 900,
 679                 .enter = &intel_idle,
 680                 .enter_s2idle = intel_idle_s2idle, },
 681         {
 682                 .name = "C9",
 683                 .desc = "MWAIT 0x50",
 684                 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
 685                 .exit_latency = 600,
 686                 .target_residency = 1800,
 687                 .enter = &intel_idle,
 688                 .enter_s2idle = intel_idle_s2idle, },
 689         {
 690                 .name = "C10",
 691                 .desc = "MWAIT 0x60",
 692                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 693                 .exit_latency = 2600,
 694                 .target_residency = 7700,
 695                 .enter = &intel_idle,
 696                 .enter_s2idle = intel_idle_s2idle, },
 697         {
 698                 .enter = NULL }
 699 };
 700
 701 static struct cpuidle_state skl_cstates[] __initdata = {
 702         {
 703                 .name = "C1",
 704                 .desc = "MWAIT 0x00",
 705                 .flags = MWAIT2flg(0x00),
 706                 .exit_latency = 2,
 707                 .target_residency = 2,
 708                 .enter = &intel_idle,
 709                 .enter_s2idle = intel_idle_s2idle, },
 710         {
 711                 .name = "C1E",
 712                 .desc = "MWAIT 0x01",
 713                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 714                 .exit_latency = 10,
 715                 .target_residency = 20,
 716                 .enter = &intel_idle,
 717                 .enter_s2idle = intel_idle_s2idle, },
 718         {
 719                 .name = "C3",
 720                 .desc = "MWAIT 0x10",
 721                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 722                 .exit_latency = 70,
 723                 .target_residency = 100,
 724                 .enter = &intel_idle,
 725                 .enter_s2idle = intel_idle_s2idle, },
 726         {
 727                 .name = "C6",
 728                 .desc = "MWAIT 0x20",
 729                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 730                 .exit_latency = 85,
 731                 .target_residency = 200,
 732                 .enter = &intel_idle,
 733                 .enter_s2idle = intel_idle_s2idle, },
 734         {
 735                 .name = "C7s",
 736                 .desc = "MWAIT 0x33",
 737                 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 738                 .exit_latency = 124,
 739                 .target_residency = 800,
 740                 .enter = &intel_idle,
 741                 .enter_s2idle = intel_idle_s2idle, },
 742         {
 743                 .name = "C8",
 744                 .desc = "MWAIT 0x40",
 745                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 746                 .exit_latency = 200,
 747                 .target_residency = 800,
 748                 .enter = &intel_idle,
 749                 .enter_s2idle = intel_idle_s2idle, },
 750         {
 751                 .name = "C9",
 752                 .desc = "MWAIT 0x50",
 753                 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 754                 .exit_latency = 480,
 755                 .target_residency = 5000,
 756                 .enter = &intel_idle,
 757                 .enter_s2idle = intel_idle_s2idle, },
 758         {
 759                 .name = "C10",
 760                 .desc = "MWAIT 0x60",
 761                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 762                 .exit_latency = 890,
 763                 .target_residency = 5000,
 764                 .enter = &intel_idle,
 765                 .enter_s2idle = intel_idle_s2idle, },
 766         {
 767                 .enter = NULL }
 768 };
 769
 770 static struct cpuidle_state skx_cstates[] __initdata = {
 771         {
 772                 .name = "C1",
 773                 .desc = "MWAIT 0x00",
 774                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
 775                 .exit_latency = 2,
 776                 .target_residency = 2,
 777                 .enter = &intel_idle,
 778                 .enter_s2idle = intel_idle_s2idle, },
 779         {
 780                 .name = "C1E",
 781                 .desc = "MWAIT 0x01",
 782                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 783                 .exit_latency = 10,
 784                 .target_residency = 20,
 785                 .enter = &intel_idle,
 786                 .enter_s2idle = intel_idle_s2idle, },
 787         {
 788                 .name = "C6",
 789                 .desc = "MWAIT 0x20",
 790                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 791                 .exit_latency = 133,
 792                 .target_residency = 600,
 793                 .enter = &intel_idle,
 794                 .enter_s2idle = intel_idle_s2idle, },
 795         {
 796                 .enter = NULL }
 797 };
 798
 799 static struct cpuidle_state icx_cstates[] __initdata = {
 800         {
 801                 .name = "C1",
 802                 .desc = "MWAIT 0x00",
 803                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
 804                 .exit_latency = 1,
 805                 .target_residency = 1,
 806                 .enter = &intel_idle,
 807                 .enter_s2idle = intel_idle_s2idle, },
 808         {
 809                 .name = "C1E",
 810                 .desc = "MWAIT 0x01",
 811                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 812                 .exit_latency = 4,
 813                 .target_residency = 4,
 814                 .enter = &intel_idle,
 815                 .enter_s2idle = intel_idle_s2idle, },
 816         {
 817                 .name = "C6",
 818                 .desc = "MWAIT 0x20",
 819                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 820                 .exit_latency = 170,
 821                 .target_residency = 600,
 822                 .enter = &intel_idle,
 823                 .enter_s2idle = intel_idle_s2idle, },
 824         {
 825                 .enter = NULL }
 826 };
 827
 828 /*
 829  * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
 830  * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
 831  * But in this case there is effectively no C1, because C1 requests are
 832  * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
 833  * and C1E requests end up with C1, so there is effectively no C1E.
 834  *
 835  * By default we enable C1E and disable C1 by marking it with
 836  * 'CPUIDLE_FLAG_UNUSABLE'.
 837  */
 838 static struct cpuidle_state adl_cstates[] __initdata = {
 839         {
 840                 .name = "C1",
 841                 .desc = "MWAIT 0x00",
 842                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 843                 .exit_latency = 1,
 844                 .target_residency = 1,
 845                 .enter = &intel_idle,
 846                 .enter_s2idle = intel_idle_s2idle, },
 847         {
 848                 .name = "C1E",
 849                 .desc = "MWAIT 0x01",
 850                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 851                 .exit_latency = 2,
 852                 .target_residency = 4,
 853                 .enter = &intel_idle,
 854                 .enter_s2idle = intel_idle_s2idle, },
 855         {
 856                 .name = "C6",
 857                 .desc = "MWAIT 0x20",
 858                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 859                 .exit_latency = 220,
 860                 .target_residency = 600,
 861                 .enter = &intel_idle,
 862                 .enter_s2idle = intel_idle_s2idle, },
 863         {
 864                 .name = "C8",
 865                 .desc = "MWAIT 0x40",
 866                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 867                 .exit_latency = 280,
 868                 .target_residency = 800,
 869                 .enter = &intel_idle,
 870                 .enter_s2idle = intel_idle_s2idle, },
 871         {
 872                 .name = "C10",
 873                 .desc = "MWAIT 0x60",
 874                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 875                 .exit_latency = 680,
 876                 .target_residency = 2000,
 877                 .enter = &intel_idle,
 878                 .enter_s2idle = intel_idle_s2idle, },
 879         {
 880                 .enter = NULL }
 881 };
 882
 883 static struct cpuidle_state adl_l_cstates[] __initdata = {
 884         {
 885                 .name = "C1",
 886                 .desc = "MWAIT 0x00",
 887                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 888                 .exit_latency = 1,
 889                 .target_residency = 1,
 890                 .enter = &intel_idle,
 891                 .enter_s2idle = intel_idle_s2idle, },
 892         {
 893                 .name = "C1E",
 894                 .desc = "MWAIT 0x01",
 895                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 896                 .exit_latency = 2,
 897                 .target_residency = 4,
 898                 .enter = &intel_idle,
 899                 .enter_s2idle = intel_idle_s2idle, },
 900         {
 901                 .name = "C6",
 902                 .desc = "MWAIT 0x20",
 903                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 904                 .exit_latency = 170,
 905                 .target_residency = 500,
 906                 .enter = &intel_idle,
 907                 .enter_s2idle = intel_idle_s2idle, },
 908         {
 909                 .name = "C8",
 910                 .desc = "MWAIT 0x40",
 911                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 912                 .exit_latency = 200,
 913                 .target_residency = 600,
 914                 .enter = &intel_idle,
 915                 .enter_s2idle = intel_idle_s2idle, },
 916         {
 917                 .name = "C10",
 918                 .desc = "MWAIT 0x60",
 919                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 920                 .exit_latency = 230,
 921                 .target_residency = 700,
 922                 .enter = &intel_idle,
 923                 .enter_s2idle = intel_idle_s2idle, },
 924         {
 925                 .enter = NULL }
 926 };
 927
 928 static struct cpuidle_state mtl_l_cstates[] __initdata = {
 929         {
 930                 .name = "C1E",
 931                 .desc = "MWAIT 0x01",
 932                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 933                 .exit_latency = 1,
 934                 .target_residency = 1,
 935                 .enter = &intel_idle,
 936                 .enter_s2idle = intel_idle_s2idle, },
 937         {
 938                 .name = "C6",
 939                 .desc = "MWAIT 0x20",
 940                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 941                 .exit_latency = 140,
 942                 .target_residency = 420,
 943                 .enter = &intel_idle,
 944                 .enter_s2idle = intel_idle_s2idle, },
 945         {
 946                 .name = "C10",
 947                 .desc = "MWAIT 0x60",
 948                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 949                 .exit_latency = 310,
 950                 .target_residency = 930,
 951                 .enter = &intel_idle,
 952                 .enter_s2idle = intel_idle_s2idle, },
 953         {
 954                 .enter = NULL }
 955 };
 956
 957 static struct cpuidle_state gmt_cstates[] __initdata = {
 958         {
 959                 .name = "C1",
 960                 .desc = "MWAIT 0x00",
 961                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 962                 .exit_latency = 1,
 963                 .target_residency = 1,
 964                 .enter = &intel_idle,
 965                 .enter_s2idle = intel_idle_s2idle, },
 966         {
 967                 .name = "C1E",
 968                 .desc = "MWAIT 0x01",
 969                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 970                 .exit_latency = 2,
 971                 .target_residency = 4,
 972                 .enter = &intel_idle,
 973                 .enter_s2idle = intel_idle_s2idle, },
 974         {
 975                 .name = "C6",
 976                 .desc = "MWAIT 0x20",
 977                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 978                 .exit_latency = 195,
 979                 .target_residency = 585,
 980                 .enter = &intel_idle,
 981                 .enter_s2idle = intel_idle_s2idle, },
 982         {
 983                 .name = "C8",
 984                 .desc = "MWAIT 0x40",
 985                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 986                 .exit_latency = 260,
 987                 .target_residency = 1040,
 988                 .enter = &intel_idle,
 989                 .enter_s2idle = intel_idle_s2idle, },
 990         {
 991                 .name = "C10",
 992                 .desc = "MWAIT 0x60",
 993                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 994                 .exit_latency = 660,
 995                 .target_residency = 1980,
 996                 .enter = &intel_idle,
 997                 .enter_s2idle = intel_idle_s2idle, },
 998         {
 999                 .enter = NULL }
1000 };
1001
1002 static struct cpuidle_state spr_cstates[] __initdata = {
1003         {
1004                 .name = "C1",
1005                 .desc = "MWAIT 0x00",
1006                 .flags = MWAIT2flg(0x00),
1007                 .exit_latency = 1,
1008                 .target_residency = 1,
1009                 .enter = &intel_idle,
1010                 .enter_s2idle = intel_idle_s2idle, },
1011         {
1012                 .name = "C1E",
1013                 .desc = "MWAIT 0x01",
1014                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1015                 .exit_latency = 2,
1016                 .target_residency = 4,
1017                 .enter = &intel_idle,
1018                 .enter_s2idle = intel_idle_s2idle, },
1019         {
1020                 .name = "C6",
1021                 .desc = "MWAIT 0x20",
1022                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1023                                            CPUIDLE_FLAG_INIT_XSTATE,
1024                 .exit_latency = 290,
1025                 .target_residency = 800,
1026                 .enter = &intel_idle,
1027                 .enter_s2idle = intel_idle_s2idle, },
1028         {
1029                 .enter = NULL }
1030 };
1031
1032 static struct cpuidle_state gnr_cstates[] __initdata = {
1033         {
1034                 .name = "C1",
1035                 .desc = "MWAIT 0x00",
1036                 .flags = MWAIT2flg(0x00),
1037                 .exit_latency = 1,
1038                 .target_residency = 1,
1039                 .enter = &intel_idle,
1040                 .enter_s2idle = intel_idle_s2idle, },
1041         {
1042                 .name = "C1E",
1043                 .desc = "MWAIT 0x01",
1044                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1045                 .exit_latency = 4,
1046                 .target_residency = 4,
1047                 .enter = &intel_idle,
1048                 .enter_s2idle = intel_idle_s2idle, },
1049         {
1050                 .name = "C6",
1051                 .desc = "MWAIT 0x20",
1052                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1053                                            CPUIDLE_FLAG_INIT_XSTATE |
1054                                            CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1055                 .exit_latency = 170,
1056                 .target_residency = 650,
1057                 .enter = &intel_idle,
1058                 .enter_s2idle = intel_idle_s2idle, },
1059         {
1060                 .name = "C6P",
1061                 .desc = "MWAIT 0x21",
1062                 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1063                                            CPUIDLE_FLAG_INIT_XSTATE |
1064                                            CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1065                 .exit_latency = 210,
1066                 .target_residency = 1000,
1067                 .enter = &intel_idle,
1068                 .enter_s2idle = intel_idle_s2idle, },
1069         {
1070                 .enter = NULL }
1071 };
1072
1073 static struct cpuidle_state gnrd_cstates[] __initdata = {
1074         {
1075                 .name = "C1",
1076                 .desc = "MWAIT 0x00",
1077                 .flags = MWAIT2flg(0x00),
1078                 .exit_latency = 1,
1079                 .target_residency = 1,
1080                 .enter = &intel_idle,
1081                 .enter_s2idle = intel_idle_s2idle, },
1082         {
1083                 .name = "C1E",
1084                 .desc = "MWAIT 0x01",
1085                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1086                 .exit_latency = 4,
1087                 .target_residency = 4,
1088                 .enter = &intel_idle,
1089                 .enter_s2idle = intel_idle_s2idle, },
1090         {
1091                 .name = "C6",
1092                 .desc = "MWAIT 0x20",
1093                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1094                                            CPUIDLE_FLAG_INIT_XSTATE |
1095                                            CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1096                 .exit_latency = 220,
1097                 .target_residency = 650,
1098                 .enter = &intel_idle,
1099                 .enter_s2idle = intel_idle_s2idle, },
1100         {
1101                 .name = "C6P",
1102                 .desc = "MWAIT 0x21",
1103                 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1104                                            CPUIDLE_FLAG_INIT_XSTATE |
1105                                            CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1106                 .exit_latency = 240,
1107                 .target_residency = 750,
1108                 .enter = &intel_idle,
1109                 .enter_s2idle = intel_idle_s2idle, },
1110         {
1111                 .enter = NULL }
1112 };
1113
1114 static struct cpuidle_state atom_cstates[] __initdata = {
1115         {
1116                 .name = "C1E",
1117                 .desc = "MWAIT 0x00",
1118                 .flags = MWAIT2flg(0x00),
1119                 .exit_latency = 10,
1120                 .target_residency = 20,
1121                 .enter = &intel_idle,
1122                 .enter_s2idle = intel_idle_s2idle, },
1123         {
1124                 .name = "C2",
1125                 .desc = "MWAIT 0x10",
1126                 .flags = MWAIT2flg(0x10),
1127                 .exit_latency = 20,
1128                 .target_residency = 80,
1129                 .enter = &intel_idle,
1130                 .enter_s2idle = intel_idle_s2idle, },
1131         {
1132                 .name = "C4",
1133                 .desc = "MWAIT 0x30",
1134                 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1135                 .exit_latency = 100,
1136                 .target_residency = 400,
1137                 .enter = &intel_idle,
1138                 .enter_s2idle = intel_idle_s2idle, },
1139         {
1140                 .name = "C6",
1141                 .desc = "MWAIT 0x52",
1142                 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1143                 .exit_latency = 140,
1144                 .target_residency = 560,
1145                 .enter = &intel_idle,
1146                 .enter_s2idle = intel_idle_s2idle, },
1147         {
1148                 .enter = NULL }
1149 };
1150 static struct cpuidle_state tangier_cstates[] __initdata = {
1151         {
1152                 .name = "C1",
1153                 .desc = "MWAIT 0x00",
1154                 .flags = MWAIT2flg(0x00),
1155                 .exit_latency = 1,
1156                 .target_residency = 4,
1157                 .enter = &intel_idle,
1158                 .enter_s2idle = intel_idle_s2idle, },
1159         {
1160                 .name = "C4",
1161                 .desc = "MWAIT 0x30",
1162                 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1163                 .exit_latency = 100,
1164                 .target_residency = 400,
1165                 .enter = &intel_idle,
1166                 .enter_s2idle = intel_idle_s2idle, },
1167         {
1168                 .name = "C6",
1169                 .desc = "MWAIT 0x52",
1170                 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1171                 .exit_latency = 140,
1172                 .target_residency = 560,
1173                 .enter = &intel_idle,
1174                 .enter_s2idle = intel_idle_s2idle, },
1175         {
1176                 .name = "C7",
1177                 .desc = "MWAIT 0x60",
1178                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1179                 .exit_latency = 1200,
1180                 .target_residency = 4000,
1181                 .enter = &intel_idle,
1182                 .enter_s2idle = intel_idle_s2idle, },
1183         {
1184                 .name = "C9",
1185                 .desc = "MWAIT 0x64",
1186                 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1187                 .exit_latency = 10000,
1188                 .target_residency = 20000,
1189                 .enter = &intel_idle,
1190                 .enter_s2idle = intel_idle_s2idle, },
1191         {
1192                 .enter = NULL }
1193 };
1194 static struct cpuidle_state avn_cstates[] __initdata = {
1195         {
1196                 .name = "C1",
1197                 .desc = "MWAIT 0x00",
1198                 .flags = MWAIT2flg(0x00),
1199                 .exit_latency = 2,
1200                 .target_residency = 2,
1201                 .enter = &intel_idle,
1202                 .enter_s2idle = intel_idle_s2idle, },
1203         {
1204                 .name = "C6",
1205                 .desc = "MWAIT 0x51",
1206                 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
1207                 .exit_latency = 15,
1208                 .target_residency = 45,
1209                 .enter = &intel_idle,
1210                 .enter_s2idle = intel_idle_s2idle, },
1211         {
1212                 .enter = NULL }
1213 };
1214 static struct cpuidle_state knl_cstates[] __initdata = {
1215         {
1216                 .name = "C1",
1217                 .desc = "MWAIT 0x00",
1218                 .flags = MWAIT2flg(0x00),
1219                 .exit_latency = 1,
1220                 .target_residency = 2,
1221                 .enter = &intel_idle,
1222                 .enter_s2idle = intel_idle_s2idle },
1223         {
1224                 .name = "C6",
1225                 .desc = "MWAIT 0x10",
1226                 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1227                 .exit_latency = 120,
1228                 .target_residency = 500,
1229                 .enter = &intel_idle,
1230                 .enter_s2idle = intel_idle_s2idle },
1231         {
1232                 .enter = NULL }
1233 };
1234
1235 static struct cpuidle_state bxt_cstates[] __initdata = {
1236         {
1237                 .name = "C1",
1238                 .desc = "MWAIT 0x00",
1239                 .flags = MWAIT2flg(0x00),
1240                 .exit_latency = 2,
1241                 .target_residency = 2,
1242                 .enter = &intel_idle,
1243                 .enter_s2idle = intel_idle_s2idle, },
1244         {
1245                 .name = "C1E",
1246                 .desc = "MWAIT 0x01",
1247                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1248                 .exit_latency = 10,
1249                 .target_residency = 20,
1250                 .enter = &intel_idle,
1251                 .enter_s2idle = intel_idle_s2idle, },
1252         {
1253                 .name = "C6",
1254                 .desc = "MWAIT 0x20",
1255                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1256                 .exit_latency = 133,
1257                 .target_residency = 133,
1258                 .enter = &intel_idle,
1259                 .enter_s2idle = intel_idle_s2idle, },
1260         {
1261                 .name = "C7s",
1262                 .desc = "MWAIT 0x31",
1263                 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1264                 .exit_latency = 155,
1265                 .target_residency = 155,
1266                 .enter = &intel_idle,
1267                 .enter_s2idle = intel_idle_s2idle, },
1268         {
1269                 .name = "C8",
1270                 .desc = "MWAIT 0x40",
1271                 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1272                 .exit_latency = 1000,
1273                 .target_residency = 1000,
1274                 .enter = &intel_idle,
1275                 .enter_s2idle = intel_idle_s2idle, },
1276         {
1277                 .name = "C9",
1278                 .desc = "MWAIT 0x50",
1279                 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1280                 .exit_latency = 2000,
1281                 .target_residency = 2000,
1282                 .enter = &intel_idle,
1283                 .enter_s2idle = intel_idle_s2idle, },
1284         {
1285                 .name = "C10",
1286                 .desc = "MWAIT 0x60",
1287                 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1288                 .exit_latency = 10000,
1289                 .target_residency = 10000,
1290                 .enter = &intel_idle,
1291                 .enter_s2idle = intel_idle_s2idle, },
1292         {
1293                 .enter = NULL }
1294 };
1295
1296 static struct cpuidle_state dnv_cstates[] __initdata = {
1297         {
1298                 .name = "C1",
1299                 .desc = "MWAIT 0x00",
1300                 .flags = MWAIT2flg(0x00),
1301                 .exit_latency = 2,
1302                 .target_residency = 2,
1303                 .enter = &intel_idle,
1304                 .enter_s2idle = intel_idle_s2idle, },
1305         {
1306                 .name = "C1E",
1307                 .desc = "MWAIT 0x01",
1308                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1309                 .exit_latency = 10,
1310                 .target_residency = 20,
1311                 .enter = &intel_idle,
1312                 .enter_s2idle = intel_idle_s2idle, },
1313         {
1314                 .name = "C6",
1315                 .desc = "MWAIT 0x20",
1316                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1317                 .exit_latency = 50,
1318                 .target_residency = 500,
1319                 .enter = &intel_idle,
1320                 .enter_s2idle = intel_idle_s2idle, },
1321         {
1322                 .enter = NULL }
1323 };
1324
1325 /*
1326  * Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1327  * C6, and this is indicated in the CPUID mwait leaf.
1328  */
1329 static struct cpuidle_state snr_cstates[] __initdata = {
1330         {
1331                 .name = "C1",
1332                 .desc = "MWAIT 0x00",
1333                 .flags = MWAIT2flg(0x00),
1334                 .exit_latency = 2,
1335                 .target_residency = 2,
1336                 .enter = &intel_idle,
1337                 .enter_s2idle = intel_idle_s2idle, },
1338         {
1339                 .name = "C1E",
1340                 .desc = "MWAIT 0x01",
1341                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1342                 .exit_latency = 15,
1343                 .target_residency = 25,
1344                 .enter = &intel_idle,
1345                 .enter_s2idle = intel_idle_s2idle, },
1346         {
1347                 .name = "C6",
1348                 .desc = "MWAIT 0x20",
1349                 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1350                 .exit_latency = 130,
1351                 .target_residency = 500,
1352                 .enter = &intel_idle,
1353                 .enter_s2idle = intel_idle_s2idle, },
1354         {
1355                 .enter = NULL }
1356 };
1357
1358 static struct cpuidle_state grr_cstates[] __initdata = {
1359         {
1360                 .name = "C1",
1361                 .desc = "MWAIT 0x00",
1362                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1363                 .exit_latency = 1,
1364                 .target_residency = 1,
1365                 .enter = &intel_idle,
1366                 .enter_s2idle = intel_idle_s2idle, },
1367         {
1368                 .name = "C1E",
1369                 .desc = "MWAIT 0x01",
1370                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1371                 .exit_latency = 2,
1372                 .target_residency = 10,
1373                 .enter = &intel_idle,
1374                 .enter_s2idle = intel_idle_s2idle, },
1375         {
1376                 .name = "C6S",
1377                 .desc = "MWAIT 0x22",
1378                 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1379                 .exit_latency = 140,
1380                 .target_residency = 500,
1381                 .enter = &intel_idle,
1382                 .enter_s2idle = intel_idle_s2idle, },
1383         {
1384                 .enter = NULL }
1385 };
1386
1387 static struct cpuidle_state srf_cstates[] __initdata = {
1388         {
1389                 .name = "C1",
1390                 .desc = "MWAIT 0x00",
1391                 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1392                 .exit_latency = 1,
1393                 .target_residency = 1,
1394                 .enter = &intel_idle,
1395                 .enter_s2idle = intel_idle_s2idle, },
1396         {
1397                 .name = "C1E",
1398                 .desc = "MWAIT 0x01",
1399                 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1400                 .exit_latency = 2,
1401                 .target_residency = 10,
1402                 .enter = &intel_idle,
1403                 .enter_s2idle = intel_idle_s2idle, },
1404         {
1405                 .name = "C6S",
1406                 .desc = "MWAIT 0x22",
1407                 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED |
1408                                            CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1409                 .exit_latency = 270,
1410                 .target_residency = 700,
1411                 .enter = &intel_idle,
1412                 .enter_s2idle = intel_idle_s2idle, },
1413         {
1414                 .name = "C6SP",
1415                 .desc = "MWAIT 0x23",
1416                 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED |
1417                                            CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1418                 .exit_latency = 310,
1419                 .target_residency = 900,
1420                 .enter = &intel_idle,
1421                 .enter_s2idle = intel_idle_s2idle, },
1422         {
1423                 .enter = NULL }
1424 };
1425
1426 static const struct idle_cpu idle_cpu_nehalem __initconst = {
1427         .state_table = nehalem_cstates,
1428         .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1429         .disable_promotion_to_c1e = true,
1430 };
1431
1432 static const struct idle_cpu idle_cpu_nhx __initconst = {
1433         .state_table = nehalem_cstates,
1434         .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1435         .disable_promotion_to_c1e = true,
1436         .use_acpi = true,
1437 };
1438
1439 static const struct idle_cpu idle_cpu_atom __initconst = {
1440         .state_table = atom_cstates,
1441 };
1442
1443 static const struct idle_cpu idle_cpu_tangier __initconst = {
1444         .state_table = tangier_cstates,
1445 };
1446
1447 static const struct idle_cpu idle_cpu_lincroft __initconst = {
1448         .state_table = atom_cstates,
1449         .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1450 };
1451
1452 static const struct idle_cpu idle_cpu_snb __initconst = {
1453         .state_table = snb_cstates,
1454         .disable_promotion_to_c1e = true,
1455 };
1456
1457 static const struct idle_cpu idle_cpu_snx __initconst = {
1458         .state_table = snb_cstates,
1459         .disable_promotion_to_c1e = true,
1460         .use_acpi = true,
1461 };
1462
1463 static const struct idle_cpu idle_cpu_byt __initconst = {
1464         .state_table = byt_cstates,
1465         .disable_promotion_to_c1e = true,
1466         .byt_auto_demotion_disable_flag = true,
1467 };
1468
1469 static const struct idle_cpu idle_cpu_cht __initconst = {
1470         .state_table = cht_cstates,
1471         .disable_promotion_to_c1e = true,
1472         .byt_auto_demotion_disable_flag = true,
1473 };
1474
1475 static const struct idle_cpu idle_cpu_ivb __initconst = {
1476         .state_table = ivb_cstates,
1477         .disable_promotion_to_c1e = true,
1478 };
1479
1480 static const struct idle_cpu idle_cpu_ivt __initconst = {
1481         .state_table = ivt_cstates,
1482         .disable_promotion_to_c1e = true,
1483         .use_acpi = true,
1484 };
1485
1486 static const struct idle_cpu idle_cpu_hsw __initconst = {
1487         .state_table = hsw_cstates,
1488         .disable_promotion_to_c1e = true,
1489 };
1490
1491 static const struct idle_cpu idle_cpu_hsx __initconst = {
1492         .state_table = hsw_cstates,
1493         .disable_promotion_to_c1e = true,
1494         .use_acpi = true,
1495 };
1496
1497 static const struct idle_cpu idle_cpu_bdw __initconst = {
1498         .state_table = bdw_cstates,
1499         .disable_promotion_to_c1e = true,
1500 };
1501
1502 static const struct idle_cpu idle_cpu_bdx __initconst = {
1503         .state_table = bdw_cstates,
1504         .disable_promotion_to_c1e = true,
1505         .use_acpi = true,
1506 };
1507
1508 static const struct idle_cpu idle_cpu_skl __initconst = {
1509         .state_table = skl_cstates,
1510         .disable_promotion_to_c1e = true,
1511 };
1512
1513 static const struct idle_cpu idle_cpu_skx __initconst = {
1514         .state_table = skx_cstates,
1515         .disable_promotion_to_c1e = true,
1516         .use_acpi = true,
1517 };
1518
1519 static const struct idle_cpu idle_cpu_icx __initconst = {
1520         .state_table = icx_cstates,
1521         .disable_promotion_to_c1e = true,
1522         .use_acpi = true,
1523 };
1524
1525 static const struct idle_cpu idle_cpu_adl __initconst = {
1526         .state_table = adl_cstates,
1527 };
1528
1529 static const struct idle_cpu idle_cpu_adl_l __initconst = {
1530         .state_table = adl_l_cstates,
1531 };
1532
1533 static const struct idle_cpu idle_cpu_mtl_l __initconst = {
1534         .state_table = mtl_l_cstates,
1535 };
1536
1537 static const struct idle_cpu idle_cpu_gmt __initconst = {
1538         .state_table = gmt_cstates,
1539 };
1540
1541 static const struct idle_cpu idle_cpu_spr __initconst = {
1542         .state_table = spr_cstates,
1543         .disable_promotion_to_c1e = true,
1544         .use_acpi = true,
1545 };
1546
1547 static const struct idle_cpu idle_cpu_gnr __initconst = {
1548         .state_table = gnr_cstates,
1549         .disable_promotion_to_c1e = true,
1550         .use_acpi = true,
1551 };
1552
1553 static const struct idle_cpu idle_cpu_gnrd __initconst = {
1554         .state_table = gnrd_cstates,
1555         .disable_promotion_to_c1e = true,
1556         .use_acpi = true,
1557 };
1558
1559 static const struct idle_cpu idle_cpu_avn __initconst = {
1560         .state_table = avn_cstates,
1561         .disable_promotion_to_c1e = true,
1562         .use_acpi = true,
1563 };
1564
1565 static const struct idle_cpu idle_cpu_knl __initconst = {
1566         .state_table = knl_cstates,
1567         .use_acpi = true,
1568 };
1569
1570 static const struct idle_cpu idle_cpu_bxt __initconst = {
1571         .state_table = bxt_cstates,
1572         .disable_promotion_to_c1e = true,
1573 };
1574
1575 static const struct idle_cpu idle_cpu_dnv __initconst = {
1576         .state_table = dnv_cstates,
1577         .disable_promotion_to_c1e = true,
1578         .use_acpi = true,
1579 };
1580
1581 static const struct idle_cpu idle_cpu_tmt __initconst = {
1582         .disable_promotion_to_c1e = true,
1583 };
1584
1585 static const struct idle_cpu idle_cpu_snr __initconst = {
1586         .state_table = snr_cstates,
1587         .disable_promotion_to_c1e = true,
1588         .use_acpi = true,
1589 };
1590
1591 static const struct idle_cpu idle_cpu_grr __initconst = {
1592         .state_table = grr_cstates,
1593         .disable_promotion_to_c1e = true,
1594         .use_acpi = true,
1595 };
1596
1597 static const struct idle_cpu idle_cpu_srf __initconst = {
1598         .state_table = srf_cstates,
1599         .disable_promotion_to_c1e = true,
1600         .use_acpi = true,
1601 };
1602
1603 static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1604         X86_MATCH_VFM(INTEL_NEHALEM_EP,         &idle_cpu_nhx),
1605         X86_MATCH_VFM(INTEL_NEHALEM,            &idle_cpu_nehalem),
1606         X86_MATCH_VFM(INTEL_NEHALEM_G,          &idle_cpu_nehalem),
1607         X86_MATCH_VFM(INTEL_WESTMERE,           &idle_cpu_nehalem),
1608         X86_MATCH_VFM(INTEL_WESTMERE_EP,        &idle_cpu_nhx),
1609         X86_MATCH_VFM(INTEL_NEHALEM_EX,         &idle_cpu_nhx),
1610         X86_MATCH_VFM(INTEL_ATOM_BONNELL,       &idle_cpu_atom),
1611         X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID,   &idle_cpu_lincroft),
1612         X86_MATCH_VFM(INTEL_WESTMERE_EX,        &idle_cpu_nhx),
1613         X86_MATCH_VFM(INTEL_SANDYBRIDGE,        &idle_cpu_snb),
1614         X86_MATCH_VFM(INTEL_SANDYBRIDGE_X,      &idle_cpu_snx),
1615         X86_MATCH_VFM(INTEL_ATOM_SALTWELL,      &idle_cpu_atom),
1616         X86_MATCH_VFM(INTEL_ATOM_SILVERMONT,    &idle_cpu_byt),
1617         X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier),
1618         X86_MATCH_VFM(INTEL_ATOM_AIRMONT,       &idle_cpu_cht),
1619         X86_MATCH_VFM(INTEL_IVYBRIDGE,          &idle_cpu_ivb),
1620         X86_MATCH_VFM(INTEL_IVYBRIDGE_X,        &idle_cpu_ivt),
1621         X86_MATCH_VFM(INTEL_HASWELL,            &idle_cpu_hsw),
1622         X86_MATCH_VFM(INTEL_HASWELL_X,          &idle_cpu_hsx),
1623         X86_MATCH_VFM(INTEL_HASWELL_L,          &idle_cpu_hsw),
1624         X86_MATCH_VFM(INTEL_HASWELL_G,          &idle_cpu_hsw),
1625         X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D,  &idle_cpu_avn),
1626         X86_MATCH_VFM(INTEL_BROADWELL,          &idle_cpu_bdw),
1627         X86_MATCH_VFM(INTEL_BROADWELL_G,        &idle_cpu_bdw),
1628         X86_MATCH_VFM(INTEL_BROADWELL_X,        &idle_cpu_bdx),
1629         X86_MATCH_VFM(INTEL_BROADWELL_D,        &idle_cpu_bdx),
1630         X86_MATCH_VFM(INTEL_SKYLAKE_L,          &idle_cpu_skl),
1631         X86_MATCH_VFM(INTEL_SKYLAKE,            &idle_cpu_skl),
1632         X86_MATCH_VFM(INTEL_KABYLAKE_L,         &idle_cpu_skl),
1633         X86_MATCH_VFM(INTEL_KABYLAKE,           &idle_cpu_skl),
1634         X86_MATCH_VFM(INTEL_SKYLAKE_X,          &idle_cpu_skx),
1635         X86_MATCH_VFM(INTEL_ICELAKE_X,          &idle_cpu_icx),
1636         X86_MATCH_VFM(INTEL_ICELAKE_D,          &idle_cpu_icx),
1637         X86_MATCH_VFM(INTEL_ALDERLAKE,          &idle_cpu_adl),
1638         X86_MATCH_VFM(INTEL_ALDERLAKE_L,        &idle_cpu_adl_l),
1639         X86_MATCH_VFM(INTEL_METEORLAKE_L,       &idle_cpu_mtl_l),
1640         X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,     &idle_cpu_gmt),
1641         X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X,   &idle_cpu_spr),
1642         X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,    &idle_cpu_spr),
1643         X86_MATCH_VFM(INTEL_GRANITERAPIDS_X,    &idle_cpu_gnr),
1644         X86_MATCH_VFM(INTEL_GRANITERAPIDS_D,    &idle_cpu_gnrd),
1645         X86_MATCH_VFM(INTEL_XEON_PHI_KNL,       &idle_cpu_knl),
1646         X86_MATCH_VFM(INTEL_XEON_PHI_KNM,       &idle_cpu_knl),
1647         X86_MATCH_VFM(INTEL_ATOM_GOLDMONT,      &idle_cpu_bxt),
1648         X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt),
1649         X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D,    &idle_cpu_dnv),
1650         X86_MATCH_VFM(INTEL_ATOM_TREMONT,       &idle_cpu_tmt),
1651         X86_MATCH_VFM(INTEL_ATOM_TREMONT_L,     &idle_cpu_tmt),
1652         X86_MATCH_VFM(INTEL_ATOM_TREMONT_D,     &idle_cpu_snr),
1653         X86_MATCH_VFM(INTEL_ATOM_CRESTMONT,     &idle_cpu_grr),
1654         X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X,   &idle_cpu_srf),
1655         X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X,    &idle_cpu_srf),
1656         {}
1657 };
1658
1659 static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1660         X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL),
1661         {}
1662 };
1663
1664 static bool __init intel_idle_max_cstate_reached(int cstate)
1665 {
1666         if (cstate + 1 > max_cstate) {
1667                 pr_info("max_cstate %d reached\n", max_cstate);
1668                 return true;
1669         }
1670         return false;
1671 }
1672
1673 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1674 {
1675         unsigned long eax = flg2MWAIT(state->flags);
1676
1677         if (boot_cpu_has(X86_FEATURE_ARAT))
1678                 return false;
1679
1680         /*
1681          * Switch over to one-shot tick broadcast if the target C-state
1682          * is deeper than C1.
1683          */
1684         return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1685 }
1686
1687 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1688 #include <acpi/processor.h>
1689
1690 static bool no_acpi __read_mostly;
1691 module_param(no_acpi, bool, 0444);
1692 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1693
1694 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1695 module_param_named(use_acpi, force_use_acpi, bool, 0444);
1696 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1697
1698 static struct acpi_processor_power acpi_state_table __initdata;
1699
1700 /**
1701  * intel_idle_cst_usable - Check if the _CST information can be used.
1702  *
1703  * Check if all of the C-states listed by _CST in the max_cstate range are
1704  * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1705  */
1706 static bool __init intel_idle_cst_usable(void)
1707 {
1708         int cstate, limit;
1709
1710         limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1711                       acpi_state_table.count);
1712
1713         for (cstate = 1; cstate < limit; cstate++) {
1714                 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1715
1716                 if (cx->entry_method != ACPI_CSTATE_FFH)
1717                         return false;
1718         }
1719
1720         return true;
1721 }
1722
1723 static bool __init intel_idle_acpi_cst_extract(void)
1724 {
1725         unsigned int cpu;
1726
1727         if (no_acpi) {
1728                 pr_debug("Not allowed to use ACPI _CST\n");
1729                 return false;
1730         }
1731
1732         for_each_possible_cpu(cpu) {
1733                 struct acpi_processor *pr = per_cpu(processors, cpu);
1734
1735                 if (!pr)
1736                         continue;
1737
1738                 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1739                         continue;
1740
1741                 acpi_state_table.count++;
1742
1743                 if (!intel_idle_cst_usable())
1744                         continue;
1745
1746                 if (!acpi_processor_claim_cst_control())
1747                         break;
1748
1749                 return true;
1750         }
1751
1752         acpi_state_table.count = 0;
1753         pr_debug("ACPI _CST not found or not usable\n");
1754         return false;
1755 }
1756
1757 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1758 {
1759         int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1760
1761         /*
1762          * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1763          * the interesting states are ACPI_CSTATE_FFH.
1764          */
1765         for (cstate = 1; cstate < limit; cstate++) {
1766                 struct acpi_processor_cx *cx;
1767                 struct cpuidle_state *state;
1768
1769                 if (intel_idle_max_cstate_reached(cstate - 1))
1770                         break;
1771
1772                 cx = &acpi_state_table.states[cstate];
1773
1774                 state = &drv->states[drv->state_count++];
1775
1776                 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
1777                 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
1778                 state->exit_latency = cx->latency;
1779                 /*
1780                  * For C1-type C-states use the same number for both the exit
1781                  * latency and target residency, because that is the case for
1782                  * C1 in the majority of the static C-states tables above.
1783                  * For the other types of C-states, however, set the target
1784                  * residency to 3 times the exit latency which should lead to
1785                  * a reasonable balance between energy-efficiency and
1786                  * performance in the majority of interesting cases.
1787                  */
1788                 state->target_residency = cx->latency;
1789                 if (cx->type > ACPI_STATE_C1)
1790                         state->target_residency *= 3;
1791
1792                 state->flags = MWAIT2flg(cx->address);
1793                 if (cx->type > ACPI_STATE_C2)
1794                         state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1795
1796                 if (disabled_states_mask & BIT(cstate))
1797                         state->flags |= CPUIDLE_FLAG_OFF;
1798
1799                 if (intel_idle_state_needs_timer_stop(state))
1800                         state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1801
1802                 state->enter = intel_idle;
1803                 state->enter_s2idle = intel_idle_s2idle;
1804         }
1805 }
1806
1807 static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
1808 {
1809         int cstate, limit;
1810
1811         /*
1812          * If there are no _CST C-states, do not disable any C-states by
1813          * default.
1814          */
1815         if (!acpi_state_table.count)
1816                 return false;
1817
1818         limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1819         /*
1820          * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1821          * the interesting states are ACPI_CSTATE_FFH.
1822          */
1823         for (cstate = 1; cstate < limit; cstate++) {
1824                 u32 acpi_hint = acpi_state_table.states[cstate].address;
1825                 u32 table_hint = mwait_hint;
1826
1827                 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) {
1828                         acpi_hint &= ~MWAIT_SUBSTATE_MASK;
1829                         table_hint &= ~MWAIT_SUBSTATE_MASK;
1830                 }
1831
1832                 if (acpi_hint == table_hint)
1833                         return false;
1834         }
1835         return true;
1836 }
1837 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1838 #define force_use_acpi  (false)
1839
1840 static inline bool intel_idle_acpi_cst_extract(void) { return false; }
1841 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
1842 static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
1843 {
1844         return false;
1845 }
1846 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1847
1848 /**
1849  * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1850  *
1851  * Tune IVT multi-socket targets.
1852  * Assumption: num_sockets == (max_package_num + 1).
1853  */
1854 static void __init ivt_idle_state_table_update(void)
1855 {
1856         /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1857         int cpu, package_num, num_sockets = 1;
1858
1859         for_each_online_cpu(cpu) {
1860                 package_num = topology_physical_package_id(cpu);
1861                 if (package_num + 1 > num_sockets) {
1862                         num_sockets = package_num + 1;
1863
1864                         if (num_sockets > 4) {
1865                                 cpuidle_state_table = ivt_cstates_8s;
1866                                 return;
1867                         }
1868                 }
1869         }
1870
1871         if (num_sockets > 2)
1872                 cpuidle_state_table = ivt_cstates_4s;
1873
1874         /* else, 1 and 2 socket systems use default ivt_cstates */
1875 }
1876
1877 /**
1878  * irtl_2_usec - IRTL to microseconds conversion.
1879  * @irtl: IRTL MSR value.
1880  *
1881  * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
1882  */
1883 static unsigned long long __init irtl_2_usec(unsigned long long irtl)
1884 {
1885         static const unsigned int irtl_ns_units[] __initconst = {
1886                 1, 32, 1024, 32768, 1048576, 33554432, 0, 0
1887         };
1888         unsigned long long ns;
1889
1890         if (!irtl)
1891                 return 0;
1892
1893         ns = irtl_ns_units[(irtl >> 10) & 0x7];
1894
1895         return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
1896 }
1897
1898 /**
1899  * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1900  *
1901  * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
1902  * definitive maximum latency and use the same value for target_residency.
1903  */
1904 static void __init bxt_idle_state_table_update(void)
1905 {
1906         unsigned long long msr;
1907         unsigned int usec;
1908
1909         rdmsrl(MSR_PKGC6_IRTL, msr);
1910         usec = irtl_2_usec(msr);
1911         if (usec) {
1912                 bxt_cstates[2].exit_latency = usec;
1913                 bxt_cstates[2].target_residency = usec;
1914         }
1915
1916         rdmsrl(MSR_PKGC7_IRTL, msr);
1917         usec = irtl_2_usec(msr);
1918         if (usec) {
1919                 bxt_cstates[3].exit_latency = usec;
1920                 bxt_cstates[3].target_residency = usec;
1921         }
1922
1923         rdmsrl(MSR_PKGC8_IRTL, msr);
1924         usec = irtl_2_usec(msr);
1925         if (usec) {
1926                 bxt_cstates[4].exit_latency = usec;
1927                 bxt_cstates[4].target_residency = usec;
1928         }
1929
1930         rdmsrl(MSR_PKGC9_IRTL, msr);
1931         usec = irtl_2_usec(msr);
1932         if (usec) {
1933                 bxt_cstates[5].exit_latency = usec;
1934                 bxt_cstates[5].target_residency = usec;
1935         }
1936
1937         rdmsrl(MSR_PKGC10_IRTL, msr);
1938         usec = irtl_2_usec(msr);
1939         if (usec) {
1940                 bxt_cstates[6].exit_latency = usec;
1941                 bxt_cstates[6].target_residency = usec;
1942         }
1943
1944 }
1945
1946 /**
1947  * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1948  *
1949  * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1950  */
1951 static void __init sklh_idle_state_table_update(void)
1952 {
1953         unsigned long long msr;
1954         unsigned int eax, ebx, ecx, edx;
1955
1956
1957         /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1958         if (max_cstate <= 7)
1959                 return;
1960
1961         /* if PC10 not present in CPUID.MWAIT.EDX */
1962         if ((mwait_substates & (0xF << 28)) == 0)
1963                 return;
1964
1965         rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1966
1967         /* PC10 is not enabled in PKG C-state limit */
1968         if ((msr & 0xF) != 8)
1969                 return;
1970
1971         ecx = 0;
1972         cpuid(7, &eax, &ebx, &ecx, &edx);
1973
1974         /* if SGX is present */
1975         if (ebx & (1 << 2)) {
1976
1977                 rdmsrl(MSR_IA32_FEAT_CTL, msr);
1978
1979                 /* if SGX is enabled */
1980                 if (msr & (1 << 18))
1981                         return;
1982         }
1983
1984         skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE;  /* C8-SKL */
1985         skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE;  /* C9-SKL */
1986 }
1987
1988 /**
1989  * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1990  * idle states table.
1991  */
1992 static void __init skx_idle_state_table_update(void)
1993 {
1994         unsigned long long msr;
1995
1996         rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1997
1998         /*
1999          * 000b: C0/C1 (no package C-state support)
2000          * 001b: C2
2001          * 010b: C6 (non-retention)
2002          * 011b: C6 (retention)
2003          * 111b: No Package C state limits.
2004          */
2005         if ((msr & 0x7) < 2) {
2006                 /*
2007                  * Uses the CC6 + PC0 latency and 3 times of
2008                  * latency for target_residency if the PC6
2009                  * is disabled in BIOS. This is consistent
2010                  * with how intel_idle driver uses _CST
2011                  * to set the target_residency.
2012                  */
2013                 skx_cstates[2].exit_latency = 92;
2014                 skx_cstates[2].target_residency = 276;
2015         }
2016 }
2017
2018 /**
2019  * adl_idle_state_table_update - Adjust AlderLake idle states table.
2020  */
2021 static void __init adl_idle_state_table_update(void)
2022 {
2023         /* Check if user prefers C1 over C1E. */
2024         if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) {
2025                 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE;
2026                 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE;
2027
2028                 /* Disable C1E by clearing the "C1E promotion" bit. */
2029                 c1e_promotion = C1E_PROMOTION_DISABLE;
2030                 return;
2031         }
2032
2033         /* Make sure C1E is enabled by default */
2034         c1e_promotion = C1E_PROMOTION_ENABLE;
2035 }
2036
2037 /**
2038  * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
2039  */
2040 static void __init spr_idle_state_table_update(void)
2041 {
2042         unsigned long long msr;
2043
2044         /*
2045          * By default, the C6 state assumes the worst-case scenario of package
2046          * C6. However, if PC6 is disabled, we update the numbers to match
2047          * core C6.
2048          */
2049         rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
2050
2051         /* Limit value 2 and above allow for PC6. */
2052         if ((msr & 0x7) < 2) {
2053                 spr_cstates[2].exit_latency = 190;
2054                 spr_cstates[2].target_residency = 600;
2055         }
2056 }
2057
2058 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
2059 {
2060         unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) &
2061                                         MWAIT_CSTATE_MASK;
2062         unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
2063                                         MWAIT_SUBSTATE_MASK;
2064
2065         /* Ignore the C-state if there are NO sub-states in CPUID for it. */
2066         if (num_substates == 0)
2067                 return false;
2068
2069         if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
2070                 mark_tsc_unstable("TSC halts in idle states deeper than C2");
2071
2072         return true;
2073 }
2074
2075 static void state_update_enter_method(struct cpuidle_state *state, int cstate)
2076 {
2077         if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
2078                 /*
2079                  * Combining with XSTATE with IBRS or IRQ_ENABLE flags
2080                  * is not currently supported but this driver.
2081                  */
2082                 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
2083                 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2084                 state->enter = intel_idle_xstate;
2085                 return;
2086         }
2087
2088         if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
2089                         ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {
2090                 /*
2091                  * IBRS mitigation requires that C-states are entered
2092                  * with interrupts disabled.
2093                  */
2094                 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))
2095                         state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;
2096                 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2097                 state->enter = intel_idle_ibrs;
2098                 return;
2099         }
2100
2101         if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
2102                 state->enter = intel_idle_irq;
2103                 return;
2104         }
2105
2106         if (force_irq_on) {
2107                 pr_info("forced intel_idle_irq for state %d\n", cstate);
2108                 state->enter = intel_idle_irq;
2109         }
2110 }
2111
2112 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
2113 {
2114         int cstate;
2115
2116         switch (boot_cpu_data.x86_vfm) {
2117         case INTEL_IVYBRIDGE_X:
2118                 ivt_idle_state_table_update();
2119                 break;
2120         case INTEL_ATOM_GOLDMONT:
2121         case INTEL_ATOM_GOLDMONT_PLUS:
2122                 bxt_idle_state_table_update();
2123                 break;
2124         case INTEL_SKYLAKE:
2125                 sklh_idle_state_table_update();
2126                 break;
2127         case INTEL_SKYLAKE_X:
2128                 skx_idle_state_table_update();
2129                 break;
2130         case INTEL_SAPPHIRERAPIDS_X:
2131         case INTEL_EMERALDRAPIDS_X:
2132                 spr_idle_state_table_update();
2133                 break;
2134         case INTEL_ALDERLAKE:
2135         case INTEL_ALDERLAKE_L:
2136         case INTEL_ATOM_GRACEMONT:
2137                 adl_idle_state_table_update();
2138                 break;
2139         }
2140
2141         for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
2142                 struct cpuidle_state *state;
2143                 unsigned int mwait_hint;
2144
2145                 if (intel_idle_max_cstate_reached(cstate))
2146                         break;
2147
2148                 if (!cpuidle_state_table[cstate].enter &&
2149                     !cpuidle_state_table[cstate].enter_s2idle)
2150                         break;
2151
2152                 /* If marked as unusable, skip this state. */
2153                 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
2154                         pr_debug("state %s is disabled\n",
2155                                  cpuidle_state_table[cstate].name);
2156                         continue;
2157                 }
2158
2159                 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
2160                 if (!intel_idle_verify_cstate(mwait_hint))
2161                         continue;
2162
2163                 /* Structure copy. */
2164                 drv->states[drv->state_count] = cpuidle_state_table[cstate];
2165                 state = &drv->states[drv->state_count];
2166
2167                 state_update_enter_method(state, cstate);
2168
2169
2170                 if ((disabled_states_mask & BIT(drv->state_count)) ||
2171                     ((icpu->use_acpi || force_use_acpi) &&
2172                      intel_idle_off_by_default(state->flags, mwait_hint) &&
2173                      !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
2174                         state->flags |= CPUIDLE_FLAG_OFF;
2175
2176                 if (intel_idle_state_needs_timer_stop(state))
2177                         state->flags |= CPUIDLE_FLAG_TIMER_STOP;
2178
2179                 drv->state_count++;
2180         }
2181
2182         if (icpu->byt_auto_demotion_disable_flag) {
2183                 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
2184                 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
2185         }
2186 }
2187
2188 /**
2189  * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2190  * @drv: cpuidle driver structure to initialize.
2191  */
2192 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
2193 {
2194         cpuidle_poll_state_init(drv);
2195
2196         if (disabled_states_mask & BIT(0))
2197                 drv->states[0].flags |= CPUIDLE_FLAG_OFF;
2198
2199         drv->state_count = 1;
2200
2201         if (icpu && icpu->state_table)
2202                 intel_idle_init_cstates_icpu(drv);
2203         else
2204                 intel_idle_init_cstates_acpi(drv);
2205 }
2206
2207 static void auto_demotion_disable(void)
2208 {
2209         unsigned long long msr_bits;
2210
2211         rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2212         msr_bits &= ~auto_demotion_disable_flags;
2213         wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2214 }
2215
2216 static void c1e_promotion_enable(void)
2217 {
2218         unsigned long long msr_bits;
2219
2220         rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2221         msr_bits |= 0x2;
2222         wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2223 }
2224
2225 static void c1e_promotion_disable(void)
2226 {
2227         unsigned long long msr_bits;
2228
2229         rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2230         msr_bits &= ~0x2;
2231         wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2232 }
2233
2234 /**
2235  * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2236  * @cpu: CPU to initialize.
2237  *
2238  * Register a cpuidle device object for @cpu and update its MSRs in accordance
2239  * with the processor model flags.
2240  */
2241 static int intel_idle_cpu_init(unsigned int cpu)
2242 {
2243         struct cpuidle_device *dev;
2244
2245         dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2246         dev->cpu = cpu;
2247
2248         if (cpuidle_register_device(dev)) {
2249                 pr_debug("cpuidle_register_device %d failed!\n", cpu);
2250                 return -EIO;
2251         }
2252
2253         if (auto_demotion_disable_flags)
2254                 auto_demotion_disable();
2255
2256         if (c1e_promotion == C1E_PROMOTION_ENABLE)
2257                 c1e_promotion_enable();
2258         else if (c1e_promotion == C1E_PROMOTION_DISABLE)
2259                 c1e_promotion_disable();
2260
2261         return 0;
2262 }
2263
2264 static int intel_idle_cpu_online(unsigned int cpu)
2265 {
2266         struct cpuidle_device *dev;
2267
2268         if (!boot_cpu_has(X86_FEATURE_ARAT))
2269                 tick_broadcast_enable();
2270
2271         /*
2272          * Some systems can hotplug a cpu at runtime after
2273          * the kernel has booted, we have to initialize the
2274          * driver in this case
2275          */
2276         dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2277         if (!dev->registered)
2278                 return intel_idle_cpu_init(cpu);
2279
2280         return 0;
2281 }
2282
2283 /**
2284  * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2285  */
2286 static void __init intel_idle_cpuidle_devices_uninit(void)
2287 {
2288         int i;
2289
2290         for_each_online_cpu(i)
2291                 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2292 }
2293
2294 static int __init intel_idle_init(void)
2295 {
2296         const struct x86_cpu_id *id;
2297         unsigned int eax, ebx, ecx;
2298         int retval;
2299
2300         /* Do not load intel_idle at all for now if idle= is passed */
2301         if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2302                 return -ENODEV;
2303
2304         if (max_cstate == 0) {
2305                 pr_debug("disabled\n");
2306                 return -EPERM;
2307         }
2308
2309         id = x86_match_cpu(intel_idle_ids);
2310         if (id) {
2311                 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2312                         pr_debug("Please enable MWAIT in BIOS SETUP\n");
2313                         return -ENODEV;
2314                 }
2315         } else {
2316                 id = x86_match_cpu(intel_mwait_ids);
2317                 if (!id)
2318                         return -ENODEV;
2319         }
2320
2321         cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates);
2322
2323         if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2324             !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2325             !mwait_substates)
2326                         return -ENODEV;
2327
2328         pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2329
2330         icpu = (const struct idle_cpu *)id->driver_data;
2331         if (icpu) {
2332                 if (icpu->state_table)
2333                         cpuidle_state_table = icpu->state_table;
2334                 else if (!intel_idle_acpi_cst_extract())
2335                         return -ENODEV;
2336
2337                 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
2338                 if (icpu->disable_promotion_to_c1e)
2339                         c1e_promotion = C1E_PROMOTION_DISABLE;
2340                 if (icpu->use_acpi || force_use_acpi)
2341                         intel_idle_acpi_cst_extract();
2342         } else if (!intel_idle_acpi_cst_extract()) {
2343                 return -ENODEV;
2344         }
2345
2346         pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
2347                  boot_cpu_data.x86_model);
2348
2349         intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2350         if (!intel_idle_cpuidle_devices)
2351                 return -ENOMEM;
2352
2353         intel_idle_cpuidle_driver_init(&intel_idle_driver);
2354
2355         retval = cpuidle_register_driver(&intel_idle_driver);
2356         if (retval) {
2357                 struct cpuidle_driver *drv = cpuidle_get_driver();
2358                 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2359                        drv ? drv->name : "none");
2360                 goto init_driver_fail;
2361         }
2362
2363         retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2364                                    intel_idle_cpu_online, NULL);
2365         if (retval < 0)
2366                 goto hp_setup_fail;
2367
2368         pr_debug("Local APIC timer is reliable in %s\n",
2369                  boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2370
2371         return 0;
2372
2373 hp_setup_fail:
2374         intel_idle_cpuidle_devices_uninit();
2375         cpuidle_unregister_driver(&intel_idle_driver);
2376 init_driver_fail:
2377         free_percpu(intel_idle_cpuidle_devices);
2378         return retval;
2379
2380 }
2381 device_initcall(intel_idle_init);
2382
2383 /*
2384  * We are not really modular, but we used to support that.  Meaning we also
2385  * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2386  * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2387  * is the easiest way (currently) to continue doing that.
2388  */
2389 module_param(max_cstate, int, 0444);
2390 /*
2391  * The positions of the bits that are set in this number are the indices of the
2392  * idle states to be disabled by default (as reflected by the names of the
2393  * corresponding idle state directories in sysfs, "state0", "state1" ...
2394  * "state<i>" ..., where <i> is the index of the given state).
2395  */
2396 module_param_named(states_off, disabled_states_mask, uint, 0444);
2397 MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2398 /*
2399  * Some platforms come with mutually exclusive C-states, so that if one is
2400  * enabled, the other C-states must not be used. Example: C1 and C1E on
2401  * Sapphire Rapids platform. This parameter allows for selecting the
2402  * preferred C-states among the groups of mutually exclusive C-states - the
2403  * selected C-states will be registered, the other C-states from the mutually
2404  * exclusive group won't be registered. If the platform has no mutually
2405  * exclusive C-states, this parameter has no effect.
2406  */
2407 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444);
2408 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
2409 /*
2410  * Debugging option that forces the driver to enter all C-states with
2411  * interrupts enabled. Does not apply to C-states with
2412  * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2413  */
2414 module_param(force_irq_on, bool, 0444);
2415 /*
2416  * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and
2417  * CPUIDLE_FLAG_IRQ_ENABLE isn't set.
2418  */
2419 module_param(ibrs_off, bool, 0444);
2420 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");