drivers/cpufreq/intel_pstate.c

   1 /*
   2  * intel_pstate.c: Native P state management for Intel processors
   3  *
   4  * (C) Copyright 2012 Intel Corporation
   5  * Author: Dirk Brandewie <[email protected]>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; version 2
  10  * of the License.
  11  */
  12
  13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  14
  15 #include <linux/kernel.h>
  16 #include <linux/kernel_stat.h>
  17 #include <linux/module.h>
  18 #include <linux/ktime.h>
  19 #include <linux/hrtimer.h>
  20 #include <linux/tick.h>
  21 #include <linux/slab.h>
  22 #include <linux/sched/cpufreq.h>
  23 #include <linux/list.h>
  24 #include <linux/cpu.h>
  25 #include <linux/cpufreq.h>
  26 #include <linux/sysfs.h>
  27 #include <linux/types.h>
  28 #include <linux/fs.h>
  29 #include <linux/acpi.h>
  30 #include <linux/vmalloc.h>
  31 #include <trace/events/power.h>
  32
  33 #include <asm/div64.h>
  34 #include <asm/msr.h>
  35 #include <asm/cpu_device_id.h>
  36 #include <asm/cpufeature.h>
  37 #include <asm/intel-family.h>
  38
  39 #define INTEL_PSTATE_SAMPLING_INTERVAL  (10 * NSEC_PER_MSEC)
  40
  41 #define INTEL_CPUFREQ_TRANSITION_LATENCY        20000
  42 #define INTEL_CPUFREQ_TRANSITION_DELAY          500
  43
  44 #ifdef CONFIG_ACPI
  45 #include <acpi/processor.h>
  46 #include <acpi/cppc_acpi.h>
  47 #endif
  48
  49 #define FRAC_BITS 8
  50 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
  51 #define fp_toint(X) ((X) >> FRAC_BITS)
  52
  53 #define EXT_BITS 6
  54 #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
  55 #define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS)
  56 #define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS)
  57
  58 static inline int32_t mul_fp(int32_t x, int32_t y)
  59 {
  60         return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
  61 }
  62
  63 static inline int32_t div_fp(s64 x, s64 y)
  64 {
  65         return div64_s64((int64_t)x << FRAC_BITS, y);
  66 }
  67
  68 static inline int ceiling_fp(int32_t x)
  69 {
  70         int mask, ret;
  71
  72         ret = fp_toint(x);
  73         mask = (1 << FRAC_BITS) - 1;
  74         if (x & mask)
  75                 ret += 1;
  76         return ret;
  77 }
  78
  79 static inline int32_t percent_fp(int percent)
  80 {
  81         return div_fp(percent, 100);
  82 }
  83
  84 static inline u64 mul_ext_fp(u64 x, u64 y)
  85 {
  86         return (x * y) >> EXT_FRAC_BITS;
  87 }
  88
  89 static inline u64 div_ext_fp(u64 x, u64 y)
  90 {
  91         return div64_u64(x << EXT_FRAC_BITS, y);
  92 }
  93
  94 static inline int32_t percent_ext_fp(int percent)
  95 {
  96         return div_ext_fp(percent, 100);
  97 }
  98
  99 /**
 100  * struct sample -      Store performance sample
 101  * @core_avg_perf:      Ratio of APERF/MPERF which is the actual average
 102  *                      performance during last sample period
 103  * @busy_scaled:        Scaled busy value which is used to calculate next
 104  *                      P state. This can be different than core_avg_perf
 105  *                      to account for cpu idle period
 106  * @aperf:              Difference of actual performance frequency clock count
 107  *                      read from APERF MSR between last and current sample
 108  * @mperf:              Difference of maximum performance frequency clock count
 109  *                      read from MPERF MSR between last and current sample
 110  * @tsc:                Difference of time stamp counter between last and
 111  *                      current sample
 112  * @time:               Current time from scheduler
 113  *
 114  * This structure is used in the cpudata structure to store performance sample
 115  * data for choosing next P State.
 116  */
 117 struct sample {
 118         int32_t core_avg_perf;
 119         int32_t busy_scaled;
 120         u64 aperf;
 121         u64 mperf;
 122         u64 tsc;
 123         u64 time;
 124 };
 125
 126 /**
 127  * struct pstate_data - Store P state data
 128  * @current_pstate:     Current requested P state
 129  * @min_pstate:         Min P state possible for this platform
 130  * @max_pstate:         Max P state possible for this platform
 131  * @max_pstate_physical:This is physical Max P state for a processor
 132  *                      This can be higher than the max_pstate which can
 133  *                      be limited by platform thermal design power limits
 134  * @scaling:            Scaling factor to  convert frequency to cpufreq
 135  *                      frequency units
 136  * @turbo_pstate:       Max Turbo P state possible for this platform
 137  * @max_freq:           @max_pstate frequency in cpufreq units
 138  * @turbo_freq:         @turbo_pstate frequency in cpufreq units
 139  *
 140  * Stores the per cpu model P state limits and current P state.
 141  */
 142 struct pstate_data {
 143         int     current_pstate;
 144         int     min_pstate;
 145         int     max_pstate;
 146         int     max_pstate_physical;
 147         int     scaling;
 148         int     turbo_pstate;
 149         unsigned int max_freq;
 150         unsigned int turbo_freq;
 151 };
 152
 153 /**
 154  * struct vid_data -    Stores voltage information data
 155  * @min:                VID data for this platform corresponding to
 156  *                      the lowest P state
 157  * @max:                VID data corresponding to the highest P State.
 158  * @turbo:              VID data for turbo P state
 159  * @ratio:              Ratio of (vid max - vid min) /
 160  *                      (max P state - Min P State)
 161  *
 162  * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling)
 163  * This data is used in Atom platforms, where in addition to target P state,
 164  * the voltage data needs to be specified to select next P State.
 165  */
 166 struct vid_data {
 167         int min;
 168         int max;
 169         int turbo;
 170         int32_t ratio;
 171 };
 172
 173 /**
 174  * struct global_params - Global parameters, mostly tunable via sysfs.
 175  * @no_turbo:           Whether or not to use turbo P-states.
 176  * @turbo_disabled:     Whethet or not turbo P-states are available at all,
 177  *                      based on the MSR_IA32_MISC_ENABLE value and whether or
 178  *                      not the maximum reported turbo P-state is different from
 179  *                      the maximum reported non-turbo one.
 180  * @min_perf_pct:       Minimum capacity limit in percent of the maximum turbo
 181  *                      P-state capacity.
 182  * @max_perf_pct:       Maximum capacity limit in percent of the maximum turbo
 183  *                      P-state capacity.
 184  */
 185 struct global_params {
 186         bool no_turbo;
 187         bool turbo_disabled;
 188         int max_perf_pct;
 189         int min_perf_pct;
 190 };
 191
 192 /**
 193  * struct cpudata -     Per CPU instance data storage
 194  * @cpu:                CPU number for this instance data
 195  * @policy:             CPUFreq policy value
 196  * @update_util:        CPUFreq utility callback information
 197  * @update_util_set:    CPUFreq utility callback is set
 198  * @iowait_boost:       iowait-related boost fraction
 199  * @last_update:        Time of the last update.
 200  * @pstate:             Stores P state limits for this CPU
 201  * @vid:                Stores VID limits for this CPU
 202  * @last_sample_time:   Last Sample time
 203  * @aperf_mperf_shift:  Number of clock cycles after aperf, merf is incremented
 204  *                      This shift is a multiplier to mperf delta to
 205  *                      calculate CPU busy.
 206  * @prev_aperf:         Last APERF value read from APERF MSR
 207  * @prev_mperf:         Last MPERF value read from MPERF MSR
 208  * @prev_tsc:           Last timestamp counter (TSC) value
 209  * @prev_cummulative_iowait: IO Wait time difference from last and
 210  *                      current sample
 211  * @sample:             Storage for storing last Sample data
 212  * @min_perf_ratio:     Minimum capacity in terms of PERF or HWP ratios
 213  * @max_perf_ratio:     Maximum capacity in terms of PERF or HWP ratios
 214  * @acpi_perf_data:     Stores ACPI perf information read from _PSS
 215  * @valid_pss_table:    Set to true for valid ACPI _PSS entries found
 216  * @epp_powersave:      Last saved HWP energy performance preference
 217  *                      (EPP) or energy performance bias (EPB),
 218  *                      when policy switched to performance
 219  * @epp_policy:         Last saved policy used to set EPP/EPB
 220  * @epp_default:        Power on default HWP energy performance
 221  *                      preference/bias
 222  * @epp_saved:          Saved EPP/EPB during system suspend or CPU offline
 223  *                      operation
 224  * @hwp_req_cached:     Cached value of the last HWP Request MSR
 225  * @hwp_cap_cached:     Cached value of the last HWP Capabilities MSR
 226  * @last_io_update:     Last time when IO wake flag was set
 227  * @sched_flags:        Store scheduler flags for possible cross CPU update
 228  * @hwp_boost_min:      Last HWP boosted min performance
 229  *
 230  * This structure stores per CPU instance data for all CPUs.
 231  */
 232 struct cpudata {
 233         int cpu;
 234
 235         unsigned int policy;
 236         struct update_util_data update_util;
 237         bool   update_util_set;
 238
 239         struct pstate_data pstate;
 240         struct vid_data vid;
 241
 242         u64     last_update;
 243         u64     last_sample_time;
 244         u64     aperf_mperf_shift;
 245         u64     prev_aperf;
 246         u64     prev_mperf;
 247         u64     prev_tsc;
 248         u64     prev_cummulative_iowait;
 249         struct sample sample;
 250         int32_t min_perf_ratio;
 251         int32_t max_perf_ratio;
 252 #ifdef CONFIG_ACPI
 253         struct acpi_processor_performance acpi_perf_data;
 254         bool valid_pss_table;
 255 #endif
 256         unsigned int iowait_boost;
 257         s16 epp_powersave;
 258         s16 epp_policy;
 259         s16 epp_default;
 260         s16 epp_saved;
 261         u64 hwp_req_cached;
 262         u64 hwp_cap_cached;
 263         u64 last_io_update;
 264         unsigned int sched_flags;
 265         u32 hwp_boost_min;
 266 };
 267
 268 static struct cpudata **all_cpu_data;
 269
 270 /**
 271  * struct pstate_funcs - Per CPU model specific callbacks
 272  * @get_max:            Callback to get maximum non turbo effective P state
 273  * @get_max_physical:   Callback to get maximum non turbo physical P state
 274  * @get_min:            Callback to get minimum P state
 275  * @get_turbo:          Callback to get turbo P state
 276  * @get_scaling:        Callback to get frequency scaling factor
 277  * @get_val:            Callback to convert P state to actual MSR write value
 278  * @get_vid:            Callback to get VID data for Atom platforms
 279  *
 280  * Core and Atom CPU models have different way to get P State limits. This
 281  * structure is used to store those callbacks.
 282  */
 283 struct pstate_funcs {
 284         int (*get_max)(void);
 285         int (*get_max_physical)(void);
 286         int (*get_min)(void);
 287         int (*get_turbo)(void);
 288         int (*get_scaling)(void);
 289         int (*get_aperf_mperf_shift)(void);
 290         u64 (*get_val)(struct cpudata*, int pstate);
 291         void (*get_vid)(struct cpudata *);
 292 };
 293
 294 static struct pstate_funcs pstate_funcs __read_mostly;
 295
 296 static int hwp_active __read_mostly;
 297 static int hwp_mode_bdw __read_mostly;
 298 static bool per_cpu_limits __read_mostly;
 299 static bool hwp_boost __read_mostly;
 300
 301 static struct cpufreq_driver *intel_pstate_driver __read_mostly;
 302
 303 #ifdef CONFIG_ACPI
 304 static bool acpi_ppc;
 305 #endif
 306
 307 static struct global_params global;
 308
 309 static DEFINE_MUTEX(intel_pstate_driver_lock);
 310 static DEFINE_MUTEX(intel_pstate_limits_lock);
 311
 312 #ifdef CONFIG_ACPI
 313
 314 static bool intel_pstate_acpi_pm_profile_server(void)
 315 {
 316         if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER ||
 317             acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER)
 318                 return true;
 319
 320         return false;
 321 }
 322
 323 static bool intel_pstate_get_ppc_enable_status(void)
 324 {
 325         if (intel_pstate_acpi_pm_profile_server())
 326                 return true;
 327
 328         return acpi_ppc;
 329 }
 330
 331 #ifdef CONFIG_ACPI_CPPC_LIB
 332
 333 /* The work item is needed to avoid CPU hotplug locking issues */
 334 static void intel_pstste_sched_itmt_work_fn(struct work_struct *work)
 335 {
 336         sched_set_itmt_support();
 337 }
 338
 339 static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn);
 340
 341 static void intel_pstate_set_itmt_prio(int cpu)
 342 {
 343         struct cppc_perf_caps cppc_perf;
 344         static u32 max_highest_perf = 0, min_highest_perf = U32_MAX;
 345         int ret;
 346
 347         ret = cppc_get_perf_caps(cpu, &cppc_perf);
 348         if (ret)
 349                 return;
 350
 351         /*
 352          * The priorities can be set regardless of whether or not
 353          * sched_set_itmt_support(true) has been called and it is valid to
 354          * update them at any time after it has been called.
 355          */
 356         sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu);
 357
 358         if (max_highest_perf <= min_highest_perf) {
 359                 if (cppc_perf.highest_perf > max_highest_perf)
 360                         max_highest_perf = cppc_perf.highest_perf;
 361
 362                 if (cppc_perf.highest_perf < min_highest_perf)
 363                         min_highest_perf = cppc_perf.highest_perf;
 364
 365                 if (max_highest_perf > min_highest_perf) {
 366                         /*
 367                          * This code can be run during CPU online under the
 368                          * CPU hotplug locks, so sched_set_itmt_support()
 369                          * cannot be called from here.  Queue up a work item
 370                          * to invoke it.
 371                          */
 372                         schedule_work(&sched_itmt_work);
 373                 }
 374         }
 375 }
 376
 377 static int intel_pstate_get_cppc_guranteed(int cpu)
 378 {
 379         struct cppc_perf_caps cppc_perf;
 380         int ret;
 381
 382         ret = cppc_get_perf_caps(cpu, &cppc_perf);
 383         if (ret)
 384                 return ret;
 385
 386         return cppc_perf.guaranteed_perf;
 387 }
 388
 389 #else /* CONFIG_ACPI_CPPC_LIB */
 390 static void intel_pstate_set_itmt_prio(int cpu)
 391 {
 392 }
 393 #endif /* CONFIG_ACPI_CPPC_LIB */
 394
 395 static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
 396 {
 397         struct cpudata *cpu;
 398         int ret;
 399         int i;
 400
 401         if (hwp_active) {
 402                 intel_pstate_set_itmt_prio(policy->cpu);
 403                 return;
 404         }
 405
 406         if (!intel_pstate_get_ppc_enable_status())
 407                 return;
 408
 409         cpu = all_cpu_data[policy->cpu];
 410
 411         ret = acpi_processor_register_performance(&cpu->acpi_perf_data,
 412                                                   policy->cpu);
 413         if (ret)
 414                 return;
 415
 416         /*
 417          * Check if the control value in _PSS is for PERF_CTL MSR, which should
 418          * guarantee that the states returned by it map to the states in our
 419          * list directly.
 420          */
 421         if (cpu->acpi_perf_data.control_register.space_id !=
 422                                                 ACPI_ADR_SPACE_FIXED_HARDWARE)
 423                 goto err;
 424
 425         /*
 426          * If there is only one entry _PSS, simply ignore _PSS and continue as
 427          * usual without taking _PSS into account
 428          */
 429         if (cpu->acpi_perf_data.state_count < 2)
 430                 goto err;
 431
 432         pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu);
 433         for (i = 0; i < cpu->acpi_perf_data.state_count; i++) {
 434                 pr_debug("     %cP%d: %u MHz, %u mW, 0x%x\n",
 435                          (i == cpu->acpi_perf_data.state ? '*' : ' '), i,
 436                          (u32) cpu->acpi_perf_data.states[i].core_frequency,
 437                          (u32) cpu->acpi_perf_data.states[i].power,
 438                          (u32) cpu->acpi_perf_data.states[i].control);
 439         }
 440
 441         /*
 442          * The _PSS table doesn't contain whole turbo frequency range.
 443          * This just contains +1 MHZ above the max non turbo frequency,
 444          * with control value corresponding to max turbo ratio. But
 445          * when cpufreq set policy is called, it will call with this
 446          * max frequency, which will cause a reduced performance as
 447          * this driver uses real max turbo frequency as the max
 448          * frequency. So correct this frequency in _PSS table to
 449          * correct max turbo frequency based on the turbo state.
 450          * Also need to convert to MHz as _PSS freq is in MHz.
 451          */
 452         if (!global.turbo_disabled)
 453                 cpu->acpi_perf_data.states[0].core_frequency =
 454                                         policy->cpuinfo.max_freq / 1000;
 455         cpu->valid_pss_table = true;
 456         pr_debug("_PPC limits will be enforced\n");
 457
 458         return;
 459
 460  err:
 461         cpu->valid_pss_table = false;
 462         acpi_processor_unregister_performance(policy->cpu);
 463 }
 464
 465 static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
 466 {
 467         struct cpudata *cpu;
 468
 469         cpu = all_cpu_data[policy->cpu];
 470         if (!cpu->valid_pss_table)
 471                 return;
 472
 473         acpi_processor_unregister_performance(policy->cpu);
 474 }
 475 #else /* CONFIG_ACPI */
 476 static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
 477 {
 478 }
 479
 480 static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
 481 {
 482 }
 483
 484 static inline bool intel_pstate_acpi_pm_profile_server(void)
 485 {
 486         return false;
 487 }
 488 #endif /* CONFIG_ACPI */
 489
 490 #ifndef CONFIG_ACPI_CPPC_LIB
 491 static int intel_pstate_get_cppc_guranteed(int cpu)
 492 {
 493         return -ENOTSUPP;
 494 }
 495 #endif /* CONFIG_ACPI_CPPC_LIB */
 496
 497 static inline void update_turbo_state(void)
 498 {
 499         u64 misc_en;
 500         struct cpudata *cpu;
 501
 502         cpu = all_cpu_data[0];
 503         rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
 504         global.turbo_disabled =
 505                 (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
 506                  cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
 507 }
 508
 509 static int min_perf_pct_min(void)
 510 {
 511         struct cpudata *cpu = all_cpu_data[0];
 512         int turbo_pstate = cpu->pstate.turbo_pstate;
 513
 514         return turbo_pstate ?
 515                 (cpu->pstate.min_pstate * 100 / turbo_pstate) : 0;
 516 }
 517
 518 static s16 intel_pstate_get_epb(struct cpudata *cpu_data)
 519 {
 520         u64 epb;
 521         int ret;
 522
 523         if (!static_cpu_has(X86_FEATURE_EPB))
 524                 return -ENXIO;
 525
 526         ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
 527         if (ret)
 528                 return (s16)ret;
 529
 530         return (s16)(epb & 0x0f);
 531 }
 532
 533 static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data)
 534 {
 535         s16 epp;
 536
 537         if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
 538                 /*
 539                  * When hwp_req_data is 0, means that caller didn't read
 540                  * MSR_HWP_REQUEST, so need to read and get EPP.
 541                  */
 542                 if (!hwp_req_data) {
 543                         epp = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST,
 544                                             &hwp_req_data);
 545                         if (epp)
 546                                 return epp;
 547                 }
 548                 epp = (hwp_req_data >> 24) & 0xff;
 549         } else {
 550                 /* When there is no EPP present, HWP uses EPB settings */
 551                 epp = intel_pstate_get_epb(cpu_data);
 552         }
 553
 554         return epp;
 555 }
 556
 557 static int intel_pstate_set_epb(int cpu, s16 pref)
 558 {
 559         u64 epb;
 560         int ret;
 561
 562         if (!static_cpu_has(X86_FEATURE_EPB))
 563                 return -ENXIO;
 564
 565         ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
 566         if (ret)
 567                 return ret;
 568
 569         epb = (epb & ~0x0f) | pref;
 570         wrmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, epb);
 571
 572         return 0;
 573 }
 574
 575 /*
 576  * EPP/EPB display strings corresponding to EPP index in the
 577  * energy_perf_strings[]
 578  *      index           String
 579  *-------------------------------------
 580  *      0               default
 581  *      1               performance
 582  *      2               balance_performance
 583  *      3               balance_power
 584  *      4               power
 585  */
 586 static const char * const energy_perf_strings[] = {
 587         "default",
 588         "performance",
 589         "balance_performance",
 590         "balance_power",
 591         "power",
 592         NULL
 593 };
 594 static const unsigned int epp_values[] = {
 595         HWP_EPP_PERFORMANCE,
 596         HWP_EPP_BALANCE_PERFORMANCE,
 597         HWP_EPP_BALANCE_POWERSAVE,
 598         HWP_EPP_POWERSAVE
 599 };
 600
 601 static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data)
 602 {
 603         s16 epp;
 604         int index = -EINVAL;
 605
 606         epp = intel_pstate_get_epp(cpu_data, 0);
 607         if (epp < 0)
 608                 return epp;
 609
 610         if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
 611                 if (epp == HWP_EPP_PERFORMANCE)
 612                         return 1;
 613                 if (epp <= HWP_EPP_BALANCE_PERFORMANCE)
 614                         return 2;
 615                 if (epp <= HWP_EPP_BALANCE_POWERSAVE)
 616                         return 3;
 617                 else
 618                         return 4;
 619         } else if (static_cpu_has(X86_FEATURE_EPB)) {
 620                 /*
 621                  * Range:
 622                  *      0x00-0x03       :       Performance
 623                  *      0x04-0x07       :       Balance performance
 624                  *      0x08-0x0B       :       Balance power
 625                  *      0x0C-0x0F       :       Power
 626                  * The EPB is a 4 bit value, but our ranges restrict the
 627                  * value which can be set. Here only using top two bits
 628                  * effectively.
 629                  */
 630                 index = (epp >> 2) + 1;
 631         }
 632
 633         return index;
 634 }
 635
 636 static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,
 637                                               int pref_index)
 638 {
 639         int epp = -EINVAL;
 640         int ret;
 641
 642         if (!pref_index)
 643                 epp = cpu_data->epp_default;
 644
 645         mutex_lock(&intel_pstate_limits_lock);
 646
 647         if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
 648                 u64 value;
 649
 650                 ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, &value);
 651                 if (ret)
 652                         goto return_pref;
 653
 654                 value &= ~GENMASK_ULL(31, 24);
 655
 656                 if (epp == -EINVAL)
 657                         epp = epp_values[pref_index - 1];
 658
 659                 value |= (u64)epp << 24;
 660                 ret = wrmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, value);
 661         } else {
 662                 if (epp == -EINVAL)
 663                         epp = (pref_index - 1) << 2;
 664                 ret = intel_pstate_set_epb(cpu_data->cpu, epp);
 665         }
 666 return_pref:
 667         mutex_unlock(&intel_pstate_limits_lock);
 668
 669         return ret;
 670 }
 671
 672 static ssize_t show_energy_performance_available_preferences(
 673                                 struct cpufreq_policy *policy, char *buf)
 674 {
 675         int i = 0;
 676         int ret = 0;
 677
 678         while (energy_perf_strings[i] != NULL)
 679                 ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]);
 680
 681         ret += sprintf(&buf[ret], "\n");
 682
 683         return ret;
 684 }
 685
 686 cpufreq_freq_attr_ro(energy_performance_available_preferences);
 687
 688 static ssize_t store_energy_performance_preference(
 689                 struct cpufreq_policy *policy, const char *buf, size_t count)
 690 {
 691         struct cpudata *cpu_data = all_cpu_data[policy->cpu];
 692         char str_preference[21];
 693         int ret;
 694
 695         ret = sscanf(buf, "%20s", str_preference);
 696         if (ret != 1)
 697                 return -EINVAL;
 698
 699         ret = match_string(energy_perf_strings, -1, str_preference);
 700         if (ret < 0)
 701                 return ret;
 702
 703         intel_pstate_set_energy_pref_index(cpu_data, ret);
 704         return count;
 705 }
 706
 707 static ssize_t show_energy_performance_preference(
 708                                 struct cpufreq_policy *policy, char *buf)
 709 {
 710         struct cpudata *cpu_data = all_cpu_data[policy->cpu];
 711         int preference;
 712
 713         preference = intel_pstate_get_energy_pref_index(cpu_data);
 714         if (preference < 0)
 715                 return preference;
 716
 717         return  sprintf(buf, "%s\n", energy_perf_strings[preference]);
 718 }
 719
 720 cpufreq_freq_attr_rw(energy_performance_preference);
 721
 722 static ssize_t show_base_frequency(struct cpufreq_policy *policy, char *buf)
 723 {
 724         struct cpudata *cpu;
 725         u64 cap;
 726         int ratio;
 727
 728         ratio = intel_pstate_get_cppc_guranteed(policy->cpu);
 729         if (ratio <= 0) {
 730                 rdmsrl_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap);
 731                 ratio = HWP_GUARANTEED_PERF(cap);
 732         }
 733
 734         cpu = all_cpu_data[policy->cpu];
 735
 736         return sprintf(buf, "%d\n", ratio * cpu->pstate.scaling);
 737 }
 738
 739 cpufreq_freq_attr_ro(base_frequency);
 740
 741 static struct freq_attr *hwp_cpufreq_attrs[] = {
 742         &energy_performance_preference,
 743         &energy_performance_available_preferences,
 744         &base_frequency,
 745         NULL,
 746 };
 747
 748 static void intel_pstate_get_hwp_max(unsigned int cpu, int *phy_max,
 749                                      int *current_max)
 750 {
 751         u64 cap;
 752
 753         rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
 754         WRITE_ONCE(all_cpu_data[cpu]->hwp_cap_cached, cap);
 755         if (global.no_turbo)
 756                 *current_max = HWP_GUARANTEED_PERF(cap);
 757         else
 758                 *current_max = HWP_HIGHEST_PERF(cap);
 759
 760         *phy_max = HWP_HIGHEST_PERF(cap);
 761 }
 762
 763 static void intel_pstate_hwp_set(unsigned int cpu)
 764 {
 765         struct cpudata *cpu_data = all_cpu_data[cpu];
 766         int max, min;
 767         u64 value;
 768         s16 epp;
 769
 770         max = cpu_data->max_perf_ratio;
 771         min = cpu_data->min_perf_ratio;
 772
 773         if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
 774                 min = max;
 775
 776         rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
 777
 778         value &= ~HWP_MIN_PERF(~0L);
 779         value |= HWP_MIN_PERF(min);
 780
 781         value &= ~HWP_MAX_PERF(~0L);
 782         value |= HWP_MAX_PERF(max);
 783
 784         if (cpu_data->epp_policy == cpu_data->policy)
 785                 goto skip_epp;
 786
 787         cpu_data->epp_policy = cpu_data->policy;
 788
 789         if (cpu_data->epp_saved >= 0) {
 790                 epp = cpu_data->epp_saved;
 791                 cpu_data->epp_saved = -EINVAL;
 792                 goto update_epp;
 793         }
 794
 795         if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
 796                 epp = intel_pstate_get_epp(cpu_data, value);
 797                 cpu_data->epp_powersave = epp;
 798                 /* If EPP read was failed, then don't try to write */
 799                 if (epp < 0)
 800                         goto skip_epp;
 801
 802                 epp = 0;
 803         } else {
 804                 /* skip setting EPP, when saved value is invalid */
 805                 if (cpu_data->epp_powersave < 0)
 806                         goto skip_epp;
 807
 808                 /*
 809                  * No need to restore EPP when it is not zero. This
 810                  * means:
 811                  *  - Policy is not changed
 812                  *  - user has manually changed
 813                  *  - Error reading EPB
 814                  */
 815                 epp = intel_pstate_get_epp(cpu_data, value);
 816                 if (epp)
 817                         goto skip_epp;
 818
 819                 epp = cpu_data->epp_powersave;
 820         }
 821 update_epp:
 822         if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
 823                 value &= ~GENMASK_ULL(31, 24);
 824                 value |= (u64)epp << 24;
 825         } else {
 826                 intel_pstate_set_epb(cpu, epp);
 827         }
 828 skip_epp:
 829         WRITE_ONCE(cpu_data->hwp_req_cached, value);
 830         wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
 831 }
 832
 833 static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
 834 {
 835         struct cpudata *cpu_data = all_cpu_data[policy->cpu];
 836
 837         if (!hwp_active)
 838                 return 0;
 839
 840         cpu_data->epp_saved = intel_pstate_get_epp(cpu_data, 0);
 841
 842         return 0;
 843 }
 844
 845 static void intel_pstate_hwp_enable(struct cpudata *cpudata);
 846
 847 static int intel_pstate_resume(struct cpufreq_policy *policy)
 848 {
 849         if (!hwp_active)
 850                 return 0;
 851
 852         mutex_lock(&intel_pstate_limits_lock);
 853
 854         if (policy->cpu == 0)
 855                 intel_pstate_hwp_enable(all_cpu_data[policy->cpu]);
 856
 857         all_cpu_data[policy->cpu]->epp_policy = 0;
 858         intel_pstate_hwp_set(policy->cpu);
 859
 860         mutex_unlock(&intel_pstate_limits_lock);
 861
 862         return 0;
 863 }
 864
 865 static void intel_pstate_update_policies(void)
 866 {
 867         int cpu;
 868
 869         for_each_possible_cpu(cpu)
 870                 cpufreq_update_policy(cpu);
 871 }
 872
 873 /************************** sysfs begin ************************/
 874 #define show_one(file_name, object)                                     \
 875         static ssize_t show_##file_name                                 \
 876         (struct kobject *kobj, struct attribute *attr, char *buf)       \
 877         {                                                               \
 878                 return sprintf(buf, "%u\n", global.object);             \
 879         }
 880
 881 static ssize_t intel_pstate_show_status(char *buf);
 882 static int intel_pstate_update_status(const char *buf, size_t size);
 883
 884 static ssize_t show_status(struct kobject *kobj,
 885                            struct attribute *attr, char *buf)
 886 {
 887         ssize_t ret;
 888
 889         mutex_lock(&intel_pstate_driver_lock);
 890         ret = intel_pstate_show_status(buf);
 891         mutex_unlock(&intel_pstate_driver_lock);
 892
 893         return ret;
 894 }
 895
 896 static ssize_t store_status(struct kobject *a, struct attribute *b,
 897                             const char *buf, size_t count)
 898 {
 899         char *p = memchr(buf, '\n', count);
 900         int ret;
 901
 902         mutex_lock(&intel_pstate_driver_lock);
 903         ret = intel_pstate_update_status(buf, p ? p - buf : count);
 904         mutex_unlock(&intel_pstate_driver_lock);
 905
 906         return ret < 0 ? ret : count;
 907 }
 908
 909 static ssize_t show_turbo_pct(struct kobject *kobj,
 910                                 struct attribute *attr, char *buf)
 911 {
 912         struct cpudata *cpu;
 913         int total, no_turbo, turbo_pct;
 914         uint32_t turbo_fp;
 915
 916         mutex_lock(&intel_pstate_driver_lock);
 917
 918         if (!intel_pstate_driver) {
 919                 mutex_unlock(&intel_pstate_driver_lock);
 920                 return -EAGAIN;
 921         }
 922
 923         cpu = all_cpu_data[0];
 924
 925         total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
 926         no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
 927         turbo_fp = div_fp(no_turbo, total);
 928         turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
 929
 930         mutex_unlock(&intel_pstate_driver_lock);
 931
 932         return sprintf(buf, "%u\n", turbo_pct);
 933 }
 934
 935 static ssize_t show_num_pstates(struct kobject *kobj,
 936                                 struct attribute *attr, char *buf)
 937 {
 938         struct cpudata *cpu;
 939         int total;
 940
 941         mutex_lock(&intel_pstate_driver_lock);
 942
 943         if (!intel_pstate_driver) {
 944                 mutex_unlock(&intel_pstate_driver_lock);
 945                 return -EAGAIN;
 946         }
 947
 948         cpu = all_cpu_data[0];
 949         total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
 950
 951         mutex_unlock(&intel_pstate_driver_lock);
 952
 953         return sprintf(buf, "%u\n", total);
 954 }
 955
 956 static ssize_t show_no_turbo(struct kobject *kobj,
 957                              struct attribute *attr, char *buf)
 958 {
 959         ssize_t ret;
 960
 961         mutex_lock(&intel_pstate_driver_lock);
 962
 963         if (!intel_pstate_driver) {
 964                 mutex_unlock(&intel_pstate_driver_lock);
 965                 return -EAGAIN;
 966         }
 967
 968         update_turbo_state();
 969         if (global.turbo_disabled)
 970                 ret = sprintf(buf, "%u\n", global.turbo_disabled);
 971         else
 972                 ret = sprintf(buf, "%u\n", global.no_turbo);
 973
 974         mutex_unlock(&intel_pstate_driver_lock);
 975
 976         return ret;
 977 }
 978
 979 static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
 980                               const char *buf, size_t count)
 981 {
 982         unsigned int input;
 983         int ret;
 984
 985         ret = sscanf(buf, "%u", &input);
 986         if (ret != 1)
 987                 return -EINVAL;
 988
 989         mutex_lock(&intel_pstate_driver_lock);
 990
 991         if (!intel_pstate_driver) {
 992                 mutex_unlock(&intel_pstate_driver_lock);
 993                 return -EAGAIN;
 994         }
 995
 996         mutex_lock(&intel_pstate_limits_lock);
 997
 998         update_turbo_state();
 999         if (global.turbo_disabled) {
1000                 pr_warn("Turbo disabled by BIOS or unavailable on processor\n");
1001                 mutex_unlock(&intel_pstate_limits_lock);
1002                 mutex_unlock(&intel_pstate_driver_lock);
1003                 return -EPERM;
1004         }
1005
1006         global.no_turbo = clamp_t(int, input, 0, 1);
1007
1008         if (global.no_turbo) {
1009                 struct cpudata *cpu = all_cpu_data[0];
1010                 int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate;
1011
1012                 /* Squash the global minimum into the permitted range. */
1013                 if (global.min_perf_pct > pct)
1014                         global.min_perf_pct = pct;
1015         }
1016
1017         mutex_unlock(&intel_pstate_limits_lock);
1018
1019         intel_pstate_update_policies();
1020
1021         mutex_unlock(&intel_pstate_driver_lock);
1022
1023         return count;
1024 }
1025
1026 static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
1027                                   const char *buf, size_t count)
1028 {
1029         unsigned int input;
1030         int ret;
1031
1032         ret = sscanf(buf, "%u", &input);
1033         if (ret != 1)
1034                 return -EINVAL;
1035
1036         mutex_lock(&intel_pstate_driver_lock);
1037
1038         if (!intel_pstate_driver) {
1039                 mutex_unlock(&intel_pstate_driver_lock);
1040                 return -EAGAIN;
1041         }
1042
1043         mutex_lock(&intel_pstate_limits_lock);
1044
1045         global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100);
1046
1047         mutex_unlock(&intel_pstate_limits_lock);
1048
1049         intel_pstate_update_policies();
1050
1051         mutex_unlock(&intel_pstate_driver_lock);
1052
1053         return count;
1054 }
1055
1056 static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
1057                                   const char *buf, size_t count)
1058 {
1059         unsigned int input;
1060         int ret;
1061
1062         ret = sscanf(buf, "%u", &input);
1063         if (ret != 1)
1064                 return -EINVAL;
1065
1066         mutex_lock(&intel_pstate_driver_lock);
1067
1068         if (!intel_pstate_driver) {
1069                 mutex_unlock(&intel_pstate_driver_lock);
1070                 return -EAGAIN;
1071         }
1072
1073         mutex_lock(&intel_pstate_limits_lock);
1074
1075         global.min_perf_pct = clamp_t(int, input,
1076                                       min_perf_pct_min(), global.max_perf_pct);
1077
1078         mutex_unlock(&intel_pstate_limits_lock);
1079
1080         intel_pstate_update_policies();
1081
1082         mutex_unlock(&intel_pstate_driver_lock);
1083
1084         return count;
1085 }
1086
1087 static ssize_t show_hwp_dynamic_boost(struct kobject *kobj,
1088                                 struct attribute *attr, char *buf)
1089 {
1090         return sprintf(buf, "%u\n", hwp_boost);
1091 }
1092
1093 static ssize_t store_hwp_dynamic_boost(struct kobject *a, struct attribute *b,
1094                                        const char *buf, size_t count)
1095 {
1096         unsigned int input;
1097         int ret;
1098
1099         ret = kstrtouint(buf, 10, &input);
1100         if (ret)
1101                 return ret;
1102
1103         mutex_lock(&intel_pstate_driver_lock);
1104         hwp_boost = !!input;
1105         intel_pstate_update_policies();
1106         mutex_unlock(&intel_pstate_driver_lock);
1107
1108         return count;
1109 }
1110
1111 show_one(max_perf_pct, max_perf_pct);
1112 show_one(min_perf_pct, min_perf_pct);
1113
1114 define_one_global_rw(status);
1115 define_one_global_rw(no_turbo);
1116 define_one_global_rw(max_perf_pct);
1117 define_one_global_rw(min_perf_pct);
1118 define_one_global_ro(turbo_pct);
1119 define_one_global_ro(num_pstates);
1120 define_one_global_rw(hwp_dynamic_boost);
1121
1122 static struct attribute *intel_pstate_attributes[] = {
1123         &status.attr,
1124         &no_turbo.attr,
1125         &turbo_pct.attr,
1126         &num_pstates.attr,
1127         NULL
1128 };
1129
1130 static const struct attribute_group intel_pstate_attr_group = {
1131         .attrs = intel_pstate_attributes,
1132 };
1133
1134 static void __init intel_pstate_sysfs_expose_params(void)
1135 {
1136         struct kobject *intel_pstate_kobject;
1137         int rc;
1138
1139         intel_pstate_kobject = kobject_create_and_add("intel_pstate",
1140                                                 &cpu_subsys.dev_root->kobj);
1141         if (WARN_ON(!intel_pstate_kobject))
1142                 return;
1143
1144         rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
1145         if (WARN_ON(rc))
1146                 return;
1147
1148         /*
1149          * If per cpu limits are enforced there are no global limits, so
1150          * return without creating max/min_perf_pct attributes
1151          */
1152         if (per_cpu_limits)
1153                 return;
1154
1155         rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr);
1156         WARN_ON(rc);
1157
1158         rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr);
1159         WARN_ON(rc);
1160
1161         if (hwp_active) {
1162                 rc = sysfs_create_file(intel_pstate_kobject,
1163                                        &hwp_dynamic_boost.attr);
1164                 WARN_ON(rc);
1165         }
1166 }
1167 /************************** sysfs end ************************/
1168
1169 static void intel_pstate_hwp_enable(struct cpudata *cpudata)
1170 {
1171         /* First disable HWP notification interrupt as we don't process them */
1172         if (static_cpu_has(X86_FEATURE_HWP_NOTIFY))
1173                 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
1174
1175         wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
1176         cpudata->epp_policy = 0;
1177         if (cpudata->epp_default == -EINVAL)
1178                 cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);
1179 }
1180
1181 #define MSR_IA32_POWER_CTL_BIT_EE       19
1182
1183 /* Disable energy efficiency optimization */
1184 static void intel_pstate_disable_ee(int cpu)
1185 {
1186         u64 power_ctl;
1187         int ret;
1188
1189         ret = rdmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, &power_ctl);
1190         if (ret)
1191                 return;
1192
1193         if (!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE))) {
1194                 pr_info("Disabling energy efficiency optimization\n");
1195                 power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE);
1196                 wrmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, power_ctl);
1197         }
1198 }
1199
1200 static int atom_get_min_pstate(void)
1201 {
1202         u64 value;
1203
1204         rdmsrl(MSR_ATOM_CORE_RATIOS, value);
1205         return (value >> 8) & 0x7F;
1206 }
1207
1208 static int atom_get_max_pstate(void)
1209 {
1210         u64 value;
1211
1212         rdmsrl(MSR_ATOM_CORE_RATIOS, value);
1213         return (value >> 16) & 0x7F;
1214 }
1215
1216 static int atom_get_turbo_pstate(void)
1217 {
1218         u64 value;
1219
1220         rdmsrl(MSR_ATOM_CORE_TURBO_RATIOS, value);
1221         return value & 0x7F;
1222 }
1223
1224 static u64 atom_get_val(struct cpudata *cpudata, int pstate)
1225 {
1226         u64 val;
1227         int32_t vid_fp;
1228         u32 vid;
1229
1230         val = (u64)pstate << 8;
1231         if (global.no_turbo && !global.turbo_disabled)
1232                 val |= (u64)1 << 32;
1233
1234         vid_fp = cpudata->vid.min + mul_fp(
1235                 int_tofp(pstate - cpudata->pstate.min_pstate),
1236                 cpudata->vid.ratio);
1237
1238         vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
1239         vid = ceiling_fp(vid_fp);
1240
1241         if (pstate > cpudata->pstate.max_pstate)
1242                 vid = cpudata->vid.turbo;
1243
1244         return val | vid;
1245 }
1246
1247 static int silvermont_get_scaling(void)
1248 {
1249         u64 value;
1250         int i;
1251         /* Defined in Table 35-6 from SDM (Sept 2015) */
1252         static int silvermont_freq_table[] = {
1253                 83300, 100000, 133300, 116700, 80000};
1254
1255         rdmsrl(MSR_FSB_FREQ, value);
1256         i = value & 0x7;
1257         WARN_ON(i > 4);
1258
1259         return silvermont_freq_table[i];
1260 }
1261
1262 static int airmont_get_scaling(void)
1263 {
1264         u64 value;
1265         int i;
1266         /* Defined in Table 35-10 from SDM (Sept 2015) */
1267         static int airmont_freq_table[] = {
1268                 83300, 100000, 133300, 116700, 80000,
1269                 93300, 90000, 88900, 87500};
1270
1271         rdmsrl(MSR_FSB_FREQ, value);
1272         i = value & 0xF;
1273         WARN_ON(i > 8);
1274
1275         return airmont_freq_table[i];
1276 }
1277
1278 static void atom_get_vid(struct cpudata *cpudata)
1279 {
1280         u64 value;
1281
1282         rdmsrl(MSR_ATOM_CORE_VIDS, value);
1283         cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
1284         cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
1285         cpudata->vid.ratio = div_fp(
1286                 cpudata->vid.max - cpudata->vid.min,
1287                 int_tofp(cpudata->pstate.max_pstate -
1288                         cpudata->pstate.min_pstate));
1289
1290         rdmsrl(MSR_ATOM_CORE_TURBO_VIDS, value);
1291         cpudata->vid.turbo = value & 0x7f;
1292 }
1293
1294 static int core_get_min_pstate(void)
1295 {
1296         u64 value;
1297
1298         rdmsrl(MSR_PLATFORM_INFO, value);
1299         return (value >> 40) & 0xFF;
1300 }
1301
1302 static int core_get_max_pstate_physical(void)
1303 {
1304         u64 value;
1305
1306         rdmsrl(MSR_PLATFORM_INFO, value);
1307         return (value >> 8) & 0xFF;
1308 }
1309
1310 static int core_get_tdp_ratio(u64 plat_info)
1311 {
1312         /* Check how many TDP levels present */
1313         if (plat_info & 0x600000000) {
1314                 u64 tdp_ctrl;
1315                 u64 tdp_ratio;
1316                 int tdp_msr;
1317                 int err;
1318
1319                 /* Get the TDP level (0, 1, 2) to get ratios */
1320                 err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
1321                 if (err)
1322                         return err;
1323
1324                 /* TDP MSR are continuous starting at 0x648 */
1325                 tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03);
1326                 err = rdmsrl_safe(tdp_msr, &tdp_ratio);
1327                 if (err)
1328                         return err;
1329
1330                 /* For level 1 and 2, bits[23:16] contain the ratio */
1331                 if (tdp_ctrl & 0x03)
1332                         tdp_ratio >>= 16;
1333
1334                 tdp_ratio &= 0xff; /* ratios are only 8 bits long */
1335                 pr_debug("tdp_ratio %x\n", (int)tdp_ratio);
1336
1337                 return (int)tdp_ratio;
1338         }
1339
1340         return -ENXIO;
1341 }
1342
1343 static int core_get_max_pstate(void)
1344 {
1345         u64 tar;
1346         u64 plat_info;
1347         int max_pstate;
1348         int tdp_ratio;
1349         int err;
1350
1351         rdmsrl(MSR_PLATFORM_INFO, plat_info);
1352         max_pstate = (plat_info >> 8) & 0xFF;
1353
1354         tdp_ratio = core_get_tdp_ratio(plat_info);
1355         if (tdp_ratio <= 0)
1356                 return max_pstate;
1357
1358         if (hwp_active) {
1359                 /* Turbo activation ratio is not used on HWP platforms */
1360                 return tdp_ratio;
1361         }
1362
1363         err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar);
1364         if (!err) {
1365                 int tar_levels;
1366
1367                 /* Do some sanity checking for safety */
1368                 tar_levels = tar & 0xff;
1369                 if (tdp_ratio - 1 == tar_levels) {
1370                         max_pstate = tar_levels;
1371                         pr_debug("max_pstate=TAC %x\n", max_pstate);
1372                 }
1373         }
1374
1375         return max_pstate;
1376 }
1377
1378 static int core_get_turbo_pstate(void)
1379 {
1380         u64 value;
1381         int nont, ret;
1382
1383         rdmsrl(MSR_TURBO_RATIO_LIMIT, value);
1384         nont = core_get_max_pstate();
1385         ret = (value) & 255;
1386         if (ret <= nont)
1387                 ret = nont;
1388         return ret;
1389 }
1390
1391 static inline int core_get_scaling(void)
1392 {
1393         return 100000;
1394 }
1395
1396 static u64 core_get_val(struct cpudata *cpudata, int pstate)
1397 {
1398         u64 val;
1399
1400         val = (u64)pstate << 8;
1401         if (global.no_turbo && !global.turbo_disabled)
1402                 val |= (u64)1 << 32;
1403
1404         return val;
1405 }
1406
1407 static int knl_get_aperf_mperf_shift(void)
1408 {
1409         return 10;
1410 }
1411
1412 static int knl_get_turbo_pstate(void)
1413 {
1414         u64 value;
1415         int nont, ret;
1416
1417         rdmsrl(MSR_TURBO_RATIO_LIMIT, value);
1418         nont = core_get_max_pstate();
1419         ret = (((value) >> 8) & 0xFF);
1420         if (ret <= nont)
1421                 ret = nont;
1422         return ret;
1423 }
1424
1425 static int intel_pstate_get_base_pstate(struct cpudata *cpu)
1426 {
1427         return global.no_turbo || global.turbo_disabled ?
1428                         cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
1429 }
1430
1431 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
1432 {
1433         trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
1434         cpu->pstate.current_pstate = pstate;
1435         /*
1436          * Generally, there is no guarantee that this code will always run on
1437          * the CPU being updated, so force the register update to run on the
1438          * right CPU.
1439          */
1440         wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
1441                       pstate_funcs.get_val(cpu, pstate));
1442 }
1443
1444 static void intel_pstate_set_min_pstate(struct cpudata *cpu)
1445 {
1446         intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
1447 }
1448
1449 static void intel_pstate_max_within_limits(struct cpudata *cpu)
1450 {
1451         int pstate;
1452
1453         update_turbo_state();
1454         pstate = intel_pstate_get_base_pstate(cpu);
1455         pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio);
1456         intel_pstate_set_pstate(cpu, pstate);
1457 }
1458
1459 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
1460 {
1461         cpu->pstate.min_pstate = pstate_funcs.get_min();
1462         cpu->pstate.max_pstate = pstate_funcs.get_max();
1463         cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical();
1464         cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
1465         cpu->pstate.scaling = pstate_funcs.get_scaling();
1466         cpu->pstate.max_freq = cpu->pstate.max_pstate * cpu->pstate.scaling;
1467
1468         if (hwp_active && !hwp_mode_bdw) {
1469                 unsigned int phy_max, current_max;
1470
1471                 intel_pstate_get_hwp_max(cpu->cpu, &phy_max, &current_max);
1472                 cpu->pstate.turbo_freq = phy_max * cpu->pstate.scaling;
1473         } else {
1474                 cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1475         }
1476
1477         if (pstate_funcs.get_aperf_mperf_shift)
1478                 cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift();
1479
1480         if (pstate_funcs.get_vid)
1481                 pstate_funcs.get_vid(cpu);
1482
1483         intel_pstate_set_min_pstate(cpu);
1484 }
1485
1486 /*
1487  * Long hold time will keep high perf limits for long time,
1488  * which negatively impacts perf/watt for some workloads,
1489  * like specpower. 3ms is based on experiements on some
1490  * workoads.
1491  */
1492 static int hwp_boost_hold_time_ns = 3 * NSEC_PER_MSEC;
1493
1494 static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu)
1495 {
1496         u64 hwp_req = READ_ONCE(cpu->hwp_req_cached);
1497         u32 max_limit = (hwp_req & 0xff00) >> 8;
1498         u32 min_limit = (hwp_req & 0xff);
1499         u32 boost_level1;
1500
1501         /*
1502          * Cases to consider (User changes via sysfs or boot time):
1503          * If, P0 (Turbo max) = P1 (Guaranteed max) = min:
1504          *      No boost, return.
1505          * If, P0 (Turbo max) > P1 (Guaranteed max) = min:
1506          *     Should result in one level boost only for P0.
1507          * If, P0 (Turbo max) = P1 (Guaranteed max) > min:
1508          *     Should result in two level boost:
1509          *         (min + p1)/2 and P1.
1510          * If, P0 (Turbo max) > P1 (Guaranteed max) > min:
1511          *     Should result in three level boost:
1512          *        (min + p1)/2, P1 and P0.
1513          */
1514
1515         /* If max and min are equal or already at max, nothing to boost */
1516         if (max_limit == min_limit || cpu->hwp_boost_min >= max_limit)
1517                 return;
1518
1519         if (!cpu->hwp_boost_min)
1520                 cpu->hwp_boost_min = min_limit;
1521
1522         /* level at half way mark between min and guranteed */
1523         boost_level1 = (HWP_GUARANTEED_PERF(cpu->hwp_cap_cached) + min_limit) >> 1;
1524
1525         if (cpu->hwp_boost_min < boost_level1)
1526                 cpu->hwp_boost_min = boost_level1;
1527         else if (cpu->hwp_boost_min < HWP_GUARANTEED_PERF(cpu->hwp_cap_cached))
1528                 cpu->hwp_boost_min = HWP_GUARANTEED_PERF(cpu->hwp_cap_cached);
1529         else if (cpu->hwp_boost_min == HWP_GUARANTEED_PERF(cpu->hwp_cap_cached) &&
1530                  max_limit != HWP_GUARANTEED_PERF(cpu->hwp_cap_cached))
1531                 cpu->hwp_boost_min = max_limit;
1532         else
1533                 return;
1534
1535         hwp_req = (hwp_req & ~GENMASK_ULL(7, 0)) | cpu->hwp_boost_min;
1536         wrmsrl(MSR_HWP_REQUEST, hwp_req);
1537         cpu->last_update = cpu->sample.time;
1538 }
1539
1540 static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu)
1541 {
1542         if (cpu->hwp_boost_min) {
1543                 bool expired;
1544
1545                 /* Check if we are idle for hold time to boost down */
1546                 expired = time_after64(cpu->sample.time, cpu->last_update +
1547                                        hwp_boost_hold_time_ns);
1548                 if (expired) {
1549                         wrmsrl(MSR_HWP_REQUEST, cpu->hwp_req_cached);
1550                         cpu->hwp_boost_min = 0;
1551                 }
1552         }
1553         cpu->last_update = cpu->sample.time;
1554 }
1555
1556 static inline void intel_pstate_update_util_hwp_local(struct cpudata *cpu,
1557                                                       u64 time)
1558 {
1559         cpu->sample.time = time;
1560
1561         if (cpu->sched_flags & SCHED_CPUFREQ_IOWAIT) {
1562                 bool do_io = false;
1563
1564                 cpu->sched_flags = 0;
1565                 /*
1566                  * Set iowait_boost flag and update time. Since IO WAIT flag
1567                  * is set all the time, we can't just conclude that there is
1568                  * some IO bound activity is scheduled on this CPU with just
1569                  * one occurrence. If we receive at least two in two
1570                  * consecutive ticks, then we treat as boost candidate.
1571                  */
1572                 if (time_before64(time, cpu->last_io_update + 2 * TICK_NSEC))
1573                         do_io = true;
1574
1575                 cpu->last_io_update = time;
1576
1577                 if (do_io)
1578                         intel_pstate_hwp_boost_up(cpu);
1579
1580         } else {
1581                 intel_pstate_hwp_boost_down(cpu);
1582         }
1583 }
1584
1585 static inline void intel_pstate_update_util_hwp(struct update_util_data *data,
1586                                                 u64 time, unsigned int flags)
1587 {
1588         struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1589
1590         cpu->sched_flags |= flags;
1591
1592         if (smp_processor_id() == cpu->cpu)
1593                 intel_pstate_update_util_hwp_local(cpu, time);
1594 }
1595
1596 static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)
1597 {
1598         struct sample *sample = &cpu->sample;
1599
1600         sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf);
1601 }
1602
1603 static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
1604 {
1605         u64 aperf, mperf;
1606         unsigned long flags;
1607         u64 tsc;
1608
1609         local_irq_save(flags);
1610         rdmsrl(MSR_IA32_APERF, aperf);
1611         rdmsrl(MSR_IA32_MPERF, mperf);
1612         tsc = rdtsc();
1613         if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) {
1614                 local_irq_restore(flags);
1615                 return false;
1616         }
1617         local_irq_restore(flags);
1618
1619         cpu->last_sample_time = cpu->sample.time;
1620         cpu->sample.time = time;
1621         cpu->sample.aperf = aperf;
1622         cpu->sample.mperf = mperf;
1623         cpu->sample.tsc =  tsc;
1624         cpu->sample.aperf -= cpu->prev_aperf;
1625         cpu->sample.mperf -= cpu->prev_mperf;
1626         cpu->sample.tsc -= cpu->prev_tsc;
1627
1628         cpu->prev_aperf = aperf;
1629         cpu->prev_mperf = mperf;
1630         cpu->prev_tsc = tsc;
1631         /*
1632          * First time this function is invoked in a given cycle, all of the
1633          * previous sample data fields are equal to zero or stale and they must
1634          * be populated with meaningful numbers for things to work, so assume
1635          * that sample.time will always be reset before setting the utilization
1636          * update hook and make the caller skip the sample then.
1637          */
1638         if (cpu->last_sample_time) {
1639                 intel_pstate_calc_avg_perf(cpu);
1640                 return true;
1641         }
1642         return false;
1643 }
1644
1645 static inline int32_t get_avg_frequency(struct cpudata *cpu)
1646 {
1647         return mul_ext_fp(cpu->sample.core_avg_perf, cpu_khz);
1648 }
1649
1650 static inline int32_t get_avg_pstate(struct cpudata *cpu)
1651 {
1652         return mul_ext_fp(cpu->pstate.max_pstate_physical,
1653                           cpu->sample.core_avg_perf);
1654 }
1655
1656 static inline int32_t get_target_pstate(struct cpudata *cpu)
1657 {
1658         struct sample *sample = &cpu->sample;
1659         int32_t busy_frac, boost;
1660         int target, avg_pstate;
1661
1662         busy_frac = div_fp(sample->mperf << cpu->aperf_mperf_shift,
1663                            sample->tsc);
1664
1665         boost = cpu->iowait_boost;
1666         cpu->iowait_boost >>= 1;
1667
1668         if (busy_frac < boost)
1669                 busy_frac = boost;
1670
1671         sample->busy_scaled = busy_frac * 100;
1672
1673         target = global.no_turbo || global.turbo_disabled ?
1674                         cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
1675         target += target >> 2;
1676         target = mul_fp(target, busy_frac);
1677         if (target < cpu->pstate.min_pstate)
1678                 target = cpu->pstate.min_pstate;
1679
1680         /*
1681          * If the average P-state during the previous cycle was higher than the
1682          * current target, add 50% of the difference to the target to reduce
1683          * possible performance oscillations and offset possible performance
1684          * loss related to moving the workload from one CPU to another within
1685          * a package/module.
1686          */
1687         avg_pstate = get_avg_pstate(cpu);
1688         if (avg_pstate > target)
1689                 target += (avg_pstate - target) >> 1;
1690
1691         return target;
1692 }
1693
1694 static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
1695 {
1696         int max_pstate = intel_pstate_get_base_pstate(cpu);
1697         int min_pstate;
1698
1699         min_pstate = max(cpu->pstate.min_pstate, cpu->min_perf_ratio);
1700         max_pstate = max(min_pstate, cpu->max_perf_ratio);
1701         return clamp_t(int, pstate, min_pstate, max_pstate);
1702 }
1703
1704 static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
1705 {
1706         if (pstate == cpu->pstate.current_pstate)
1707                 return;
1708
1709         cpu->pstate.current_pstate = pstate;
1710         wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
1711 }
1712
1713 static void intel_pstate_adjust_pstate(struct cpudata *cpu)
1714 {
1715         int from = cpu->pstate.current_pstate;
1716         struct sample *sample;
1717         int target_pstate;
1718
1719         update_turbo_state();
1720
1721         target_pstate = get_target_pstate(cpu);
1722         target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
1723         trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu);
1724         intel_pstate_update_pstate(cpu, target_pstate);
1725
1726         sample = &cpu->sample;
1727         trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf),
1728                 fp_toint(sample->busy_scaled),
1729                 from,
1730                 cpu->pstate.current_pstate,
1731                 sample->mperf,
1732                 sample->aperf,
1733                 sample->tsc,
1734                 get_avg_frequency(cpu),
1735                 fp_toint(cpu->iowait_boost * 100));
1736 }
1737
1738 static void intel_pstate_update_util(struct update_util_data *data, u64 time,
1739                                      unsigned int flags)
1740 {
1741         struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1742         u64 delta_ns;
1743
1744         /* Don't allow remote callbacks */
1745         if (smp_processor_id() != cpu->cpu)
1746                 return;
1747
1748         if (flags & SCHED_CPUFREQ_IOWAIT) {
1749                 cpu->iowait_boost = int_tofp(1);
1750                 cpu->last_update = time;
1751                 /*
1752                  * The last time the busy was 100% so P-state was max anyway
1753                  * so avoid overhead of computation.
1754                  */
1755                 if (fp_toint(cpu->sample.busy_scaled) == 100)
1756                         return;
1757
1758                 goto set_pstate;
1759         } else if (cpu->iowait_boost) {
1760                 /* Clear iowait_boost if the CPU may have been idle. */
1761                 delta_ns = time - cpu->last_update;
1762                 if (delta_ns > TICK_NSEC)
1763                         cpu->iowait_boost = 0;
1764         }
1765         cpu->last_update = time;
1766         delta_ns = time - cpu->sample.time;
1767         if ((s64)delta_ns < INTEL_PSTATE_SAMPLING_INTERVAL)
1768                 return;
1769
1770 set_pstate:
1771         if (intel_pstate_sample(cpu, time))
1772                 intel_pstate_adjust_pstate(cpu);
1773 }
1774
1775 static struct pstate_funcs core_funcs = {
1776         .get_max = core_get_max_pstate,
1777         .get_max_physical = core_get_max_pstate_physical,
1778         .get_min = core_get_min_pstate,
1779         .get_turbo = core_get_turbo_pstate,
1780         .get_scaling = core_get_scaling,
1781         .get_val = core_get_val,
1782 };
1783
1784 static const struct pstate_funcs silvermont_funcs = {
1785         .get_max = atom_get_max_pstate,
1786         .get_max_physical = atom_get_max_pstate,
1787         .get_min = atom_get_min_pstate,
1788         .get_turbo = atom_get_turbo_pstate,
1789         .get_val = atom_get_val,
1790         .get_scaling = silvermont_get_scaling,
1791         .get_vid = atom_get_vid,
1792 };
1793
1794 static const struct pstate_funcs airmont_funcs = {
1795         .get_max = atom_get_max_pstate,
1796         .get_max_physical = atom_get_max_pstate,
1797         .get_min = atom_get_min_pstate,
1798         .get_turbo = atom_get_turbo_pstate,
1799         .get_val = atom_get_val,
1800         .get_scaling = airmont_get_scaling,
1801         .get_vid = atom_get_vid,
1802 };
1803
1804 static const struct pstate_funcs knl_funcs = {
1805         .get_max = core_get_max_pstate,
1806         .get_max_physical = core_get_max_pstate_physical,
1807         .get_min = core_get_min_pstate,
1808         .get_turbo = knl_get_turbo_pstate,
1809         .get_aperf_mperf_shift = knl_get_aperf_mperf_shift,
1810         .get_scaling = core_get_scaling,
1811         .get_val = core_get_val,
1812 };
1813
1814 #define ICPU(model, policy) \
1815         { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
1816                         (unsigned long)&policy }
1817
1818 static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
1819         ICPU(INTEL_FAM6_SANDYBRIDGE,            core_funcs),
1820         ICPU(INTEL_FAM6_SANDYBRIDGE_X,          core_funcs),
1821         ICPU(INTEL_FAM6_ATOM_SILVERMONT,        silvermont_funcs),
1822         ICPU(INTEL_FAM6_IVYBRIDGE,              core_funcs),
1823         ICPU(INTEL_FAM6_HASWELL_CORE,           core_funcs),
1824         ICPU(INTEL_FAM6_BROADWELL_CORE,         core_funcs),
1825         ICPU(INTEL_FAM6_IVYBRIDGE_X,            core_funcs),
1826         ICPU(INTEL_FAM6_HASWELL_X,              core_funcs),
1827         ICPU(INTEL_FAM6_HASWELL_ULT,            core_funcs),
1828         ICPU(INTEL_FAM6_HASWELL_GT3E,           core_funcs),
1829         ICPU(INTEL_FAM6_BROADWELL_GT3E,         core_funcs),
1830         ICPU(INTEL_FAM6_ATOM_AIRMONT,           airmont_funcs),
1831         ICPU(INTEL_FAM6_SKYLAKE_MOBILE,         core_funcs),
1832         ICPU(INTEL_FAM6_BROADWELL_X,            core_funcs),
1833         ICPU(INTEL_FAM6_SKYLAKE_DESKTOP,        core_funcs),
1834         ICPU(INTEL_FAM6_BROADWELL_XEON_D,       core_funcs),
1835         ICPU(INTEL_FAM6_XEON_PHI_KNL,           knl_funcs),
1836         ICPU(INTEL_FAM6_XEON_PHI_KNM,           knl_funcs),
1837         ICPU(INTEL_FAM6_ATOM_GOLDMONT,          core_funcs),
1838         ICPU(INTEL_FAM6_ATOM_GOLDMONT_PLUS,     core_funcs),
1839         ICPU(INTEL_FAM6_SKYLAKE_X,              core_funcs),
1840         {}
1841 };
1842 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
1843
1844 static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
1845         ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
1846         ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
1847         ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
1848         {}
1849 };
1850
1851 static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
1852         ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_funcs),
1853         {}
1854 };
1855
1856 static const struct x86_cpu_id intel_pstate_hwp_boost_ids[] = {
1857         ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
1858         ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_funcs),
1859         {}
1860 };
1861
1862 static int intel_pstate_init_cpu(unsigned int cpunum)
1863 {
1864         struct cpudata *cpu;
1865
1866         cpu = all_cpu_data[cpunum];
1867
1868         if (!cpu) {
1869                 cpu = kzalloc(sizeof(*cpu), GFP_KERNEL);
1870                 if (!cpu)
1871                         return -ENOMEM;
1872
1873                 all_cpu_data[cpunum] = cpu;
1874
1875                 cpu->epp_default = -EINVAL;
1876                 cpu->epp_powersave = -EINVAL;
1877                 cpu->epp_saved = -EINVAL;
1878         }
1879
1880         cpu = all_cpu_data[cpunum];
1881
1882         cpu->cpu = cpunum;
1883
1884         if (hwp_active) {
1885                 const struct x86_cpu_id *id;
1886
1887                 id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids);
1888                 if (id)
1889                         intel_pstate_disable_ee(cpunum);
1890
1891                 intel_pstate_hwp_enable(cpu);
1892
1893                 id = x86_match_cpu(intel_pstate_hwp_boost_ids);
1894                 if (id && intel_pstate_acpi_pm_profile_server())
1895                         hwp_boost = true;
1896         }
1897
1898         intel_pstate_get_cpu_pstates(cpu);
1899
1900         pr_debug("controlling: cpu %d\n", cpunum);
1901
1902         return 0;
1903 }
1904
1905 static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
1906 {
1907         struct cpudata *cpu = all_cpu_data[cpu_num];
1908
1909         if (hwp_active && !hwp_boost)
1910                 return;
1911
1912         if (cpu->update_util_set)
1913                 return;
1914
1915         /* Prevent intel_pstate_update_util() from using stale data. */
1916         cpu->sample.time = 0;
1917         cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
1918                                      (hwp_active ?
1919                                       intel_pstate_update_util_hwp :
1920                                       intel_pstate_update_util));
1921         cpu->update_util_set = true;
1922 }
1923
1924 static void intel_pstate_clear_update_util_hook(unsigned int cpu)
1925 {
1926         struct cpudata *cpu_data = all_cpu_data[cpu];
1927
1928         if (!cpu_data->update_util_set)
1929                 return;
1930
1931         cpufreq_remove_update_util_hook(cpu);
1932         cpu_data->update_util_set = false;
1933         synchronize_sched();
1934 }
1935
1936 static int intel_pstate_get_max_freq(struct cpudata *cpu)
1937 {
1938         return global.turbo_disabled || global.no_turbo ?
1939                         cpu->pstate.max_freq : cpu->pstate.turbo_freq;
1940 }
1941
1942 static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy,
1943                                             struct cpudata *cpu)
1944 {
1945         int max_freq = intel_pstate_get_max_freq(cpu);
1946         int32_t max_policy_perf, min_policy_perf;
1947         int max_state, turbo_max;
1948
1949         /*
1950          * HWP needs some special consideration, because on BDX the
1951          * HWP_REQUEST uses abstract value to represent performance
1952          * rather than pure ratios.
1953          */
1954         if (hwp_active) {
1955                 intel_pstate_get_hwp_max(cpu->cpu, &turbo_max, &max_state);
1956         } else {
1957                 max_state = intel_pstate_get_base_pstate(cpu);
1958                 turbo_max = cpu->pstate.turbo_pstate;
1959         }
1960
1961         max_policy_perf = max_state * policy->max / max_freq;
1962         if (policy->max == policy->min) {
1963                 min_policy_perf = max_policy_perf;
1964         } else {
1965                 min_policy_perf = max_state * policy->min / max_freq;
1966                 min_policy_perf = clamp_t(int32_t, min_policy_perf,
1967                                           0, max_policy_perf);
1968         }
1969
1970         pr_debug("cpu:%d max_state %d min_policy_perf:%d max_policy_perf:%d\n",
1971                  policy->cpu, max_state,
1972                  min_policy_perf, max_policy_perf);
1973
1974         /* Normalize user input to [min_perf, max_perf] */
1975         if (per_cpu_limits) {
1976                 cpu->min_perf_ratio = min_policy_perf;
1977                 cpu->max_perf_ratio = max_policy_perf;
1978         } else {
1979                 int32_t global_min, global_max;
1980
1981                 /* Global limits are in percent of the maximum turbo P-state. */
1982                 global_max = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100);
1983                 global_min = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100);
1984                 global_min = clamp_t(int32_t, global_min, 0, global_max);
1985
1986                 pr_debug("cpu:%d global_min:%d global_max:%d\n", policy->cpu,
1987                          global_min, global_max);
1988
1989                 cpu->min_perf_ratio = max(min_policy_perf, global_min);
1990                 cpu->min_perf_ratio = min(cpu->min_perf_ratio, max_policy_perf);
1991                 cpu->max_perf_ratio = min(max_policy_perf, global_max);
1992                 cpu->max_perf_ratio = max(min_policy_perf, cpu->max_perf_ratio);
1993
1994                 /* Make sure min_perf <= max_perf */
1995                 cpu->min_perf_ratio = min(cpu->min_perf_ratio,
1996                                           cpu->max_perf_ratio);
1997
1998         }
1999         pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", policy->cpu,
2000                  cpu->max_perf_ratio,
2001                  cpu->min_perf_ratio);
2002 }
2003
2004 static int intel_pstate_set_policy(struct cpufreq_policy *policy)
2005 {
2006         struct cpudata *cpu;
2007
2008         if (!policy->cpuinfo.max_freq)
2009                 return -ENODEV;
2010
2011         pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
2012                  policy->cpuinfo.max_freq, policy->max);
2013
2014         cpu = all_cpu_data[policy->cpu];
2015         cpu->policy = policy->policy;
2016
2017         mutex_lock(&intel_pstate_limits_lock);
2018
2019         intel_pstate_update_perf_limits(policy, cpu);
2020
2021         if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
2022                 /*
2023                  * NOHZ_FULL CPUs need this as the governor callback may not
2024                  * be invoked on them.
2025                  */
2026                 intel_pstate_clear_update_util_hook(policy->cpu);
2027                 intel_pstate_max_within_limits(cpu);
2028         } else {
2029                 intel_pstate_set_update_util_hook(policy->cpu);
2030         }
2031
2032         if (hwp_active) {
2033                 /*
2034                  * When hwp_boost was active before and dynamically it
2035                  * was turned off, in that case we need to clear the
2036                  * update util hook.
2037                  */
2038                 if (!hwp_boost)
2039                         intel_pstate_clear_update_util_hook(policy->cpu);
2040                 intel_pstate_hwp_set(policy->cpu);
2041         }
2042
2043         mutex_unlock(&intel_pstate_limits_lock);
2044
2045         return 0;
2046 }
2047
2048 static void intel_pstate_adjust_policy_max(struct cpufreq_policy *policy,
2049                                          struct cpudata *cpu)
2050 {
2051         if (!hwp_active &&
2052             cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
2053             policy->max < policy->cpuinfo.max_freq &&
2054             policy->max > cpu->pstate.max_freq) {
2055                 pr_debug("policy->max > max non turbo frequency\n");
2056                 policy->max = policy->cpuinfo.max_freq;
2057         }
2058 }
2059
2060 static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
2061 {
2062         struct cpudata *cpu = all_cpu_data[policy->cpu];
2063
2064         update_turbo_state();
2065         cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
2066                                      intel_pstate_get_max_freq(cpu));
2067
2068         if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
2069             policy->policy != CPUFREQ_POLICY_PERFORMANCE)
2070                 return -EINVAL;
2071
2072         intel_pstate_adjust_policy_max(policy, cpu);
2073
2074         return 0;
2075 }
2076
2077 static void intel_cpufreq_stop_cpu(struct cpufreq_policy *policy)
2078 {
2079         intel_pstate_set_min_pstate(all_cpu_data[policy->cpu]);
2080 }
2081
2082 static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
2083 {
2084         pr_debug("CPU %d exiting\n", policy->cpu);
2085
2086         intel_pstate_clear_update_util_hook(policy->cpu);
2087         if (hwp_active)
2088                 intel_pstate_hwp_save_state(policy);
2089         else
2090                 intel_cpufreq_stop_cpu(policy);
2091 }
2092
2093 static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
2094 {
2095         intel_pstate_exit_perf_limits(policy);
2096
2097         policy->fast_switch_possible = false;
2098
2099         return 0;
2100 }
2101
2102 static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
2103 {
2104         struct cpudata *cpu;
2105         int rc;
2106
2107         rc = intel_pstate_init_cpu(policy->cpu);
2108         if (rc)
2109                 return rc;
2110
2111         cpu = all_cpu_data[policy->cpu];
2112
2113         cpu->max_perf_ratio = 0xFF;
2114         cpu->min_perf_ratio = 0;
2115
2116         policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
2117         policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
2118
2119         /* cpuinfo and default policy values */
2120         policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
2121         update_turbo_state();
2122         policy->cpuinfo.max_freq = global.turbo_disabled ?
2123                         cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
2124         policy->cpuinfo.max_freq *= cpu->pstate.scaling;
2125
2126         if (hwp_active) {
2127                 unsigned int max_freq;
2128
2129                 max_freq = global.turbo_disabled ?
2130                         cpu->pstate.max_freq : cpu->pstate.turbo_freq;
2131                 if (max_freq < policy->cpuinfo.max_freq)
2132                         policy->cpuinfo.max_freq = max_freq;
2133         }
2134
2135         intel_pstate_init_acpi_perf_limits(policy);
2136
2137         policy->fast_switch_possible = true;
2138
2139         return 0;
2140 }
2141
2142 static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
2143 {
2144         int ret = __intel_pstate_cpu_init(policy);
2145
2146         if (ret)
2147                 return ret;
2148
2149         if (IS_ENABLED(CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE))
2150                 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
2151         else
2152                 policy->policy = CPUFREQ_POLICY_POWERSAVE;
2153
2154         return 0;
2155 }
2156
2157 static struct cpufreq_driver intel_pstate = {
2158         .flags          = CPUFREQ_CONST_LOOPS,
2159         .verify         = intel_pstate_verify_policy,
2160         .setpolicy      = intel_pstate_set_policy,
2161         .suspend        = intel_pstate_hwp_save_state,
2162         .resume         = intel_pstate_resume,
2163         .init           = intel_pstate_cpu_init,
2164         .exit           = intel_pstate_cpu_exit,
2165         .stop_cpu       = intel_pstate_stop_cpu,
2166         .name           = "intel_pstate",
2167 };
2168
2169 static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy)
2170 {
2171         struct cpudata *cpu = all_cpu_data[policy->cpu];
2172
2173         update_turbo_state();
2174         cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
2175                                      intel_pstate_get_max_freq(cpu));
2176
2177         intel_pstate_adjust_policy_max(policy, cpu);
2178
2179         intel_pstate_update_perf_limits(policy, cpu);
2180
2181         return 0;
2182 }
2183
2184 /* Use of trace in passive mode:
2185  *
2186  * In passive mode the trace core_busy field (also known as the
2187  * performance field, and lablelled as such on the graphs; also known as
2188  * core_avg_perf) is not needed and so is re-assigned to indicate if the
2189  * driver call was via the normal or fast switch path. Various graphs
2190  * output from the intel_pstate_tracer.py utility that include core_busy
2191  * (or performance or core_avg_perf) have a fixed y-axis from 0 to 100%,
2192  * so we use 10 to indicate the the normal path through the driver, and
2193  * 90 to indicate the fast switch path through the driver.
2194  * The scaled_busy field is not used, and is set to 0.
2195  */
2196
2197 #define INTEL_PSTATE_TRACE_TARGET 10
2198 #define INTEL_PSTATE_TRACE_FAST_SWITCH 90
2199
2200 static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate)
2201 {
2202         struct sample *sample;
2203
2204         if (!trace_pstate_sample_enabled())
2205                 return;
2206
2207         if (!intel_pstate_sample(cpu, ktime_get()))
2208                 return;
2209
2210         sample = &cpu->sample;
2211         trace_pstate_sample(trace_type,
2212                 0,
2213                 old_pstate,
2214                 cpu->pstate.current_pstate,
2215                 sample->mperf,
2216                 sample->aperf,
2217                 sample->tsc,
2218                 get_avg_frequency(cpu),
2219                 fp_toint(cpu->iowait_boost * 100));
2220 }
2221
2222 static int intel_cpufreq_target(struct cpufreq_policy *policy,
2223                                 unsigned int target_freq,
2224                                 unsigned int relation)
2225 {
2226         struct cpudata *cpu = all_cpu_data[policy->cpu];
2227         struct cpufreq_freqs freqs;
2228         int target_pstate, old_pstate;
2229
2230         update_turbo_state();
2231
2232         freqs.old = policy->cur;
2233         freqs.new = target_freq;
2234
2235         cpufreq_freq_transition_begin(policy, &freqs);
2236         switch (relation) {
2237         case CPUFREQ_RELATION_L:
2238                 target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling);
2239                 break;
2240         case CPUFREQ_RELATION_H:
2241                 target_pstate = freqs.new / cpu->pstate.scaling;
2242                 break;
2243         default:
2244                 target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling);
2245                 break;
2246         }
2247         target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
2248         old_pstate = cpu->pstate.current_pstate;
2249         if (target_pstate != cpu->pstate.current_pstate) {
2250                 cpu->pstate.current_pstate = target_pstate;
2251                 wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL,
2252                               pstate_funcs.get_val(cpu, target_pstate));
2253         }
2254         freqs.new = target_pstate * cpu->pstate.scaling;
2255         intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_TARGET, old_pstate);
2256         cpufreq_freq_transition_end(policy, &freqs, false);
2257
2258         return 0;
2259 }
2260
2261 static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
2262                                               unsigned int target_freq)
2263 {
2264         struct cpudata *cpu = all_cpu_data[policy->cpu];
2265         int target_pstate, old_pstate;
2266
2267         update_turbo_state();
2268
2269         target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling);
2270         target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
2271         old_pstate = cpu->pstate.current_pstate;
2272         intel_pstate_update_pstate(cpu, target_pstate);
2273         intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_FAST_SWITCH, old_pstate);
2274         return target_pstate * cpu->pstate.scaling;
2275 }
2276
2277 static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
2278 {
2279         int ret = __intel_pstate_cpu_init(policy);
2280
2281         if (ret)
2282                 return ret;
2283
2284         policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
2285         policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
2286         /* This reflects the intel_pstate_get_cpu_pstates() setting. */
2287         policy->cur = policy->cpuinfo.min_freq;
2288
2289         return 0;
2290 }
2291
2292 static struct cpufreq_driver intel_cpufreq = {
2293         .flags          = CPUFREQ_CONST_LOOPS,
2294         .verify         = intel_cpufreq_verify_policy,
2295         .target         = intel_cpufreq_target,
2296         .fast_switch    = intel_cpufreq_fast_switch,
2297         .init           = intel_cpufreq_cpu_init,
2298         .exit           = intel_pstate_cpu_exit,
2299         .stop_cpu       = intel_cpufreq_stop_cpu,
2300         .name           = "intel_cpufreq",
2301 };
2302
2303 static struct cpufreq_driver *default_driver = &intel_pstate;
2304
2305 static void intel_pstate_driver_cleanup(void)
2306 {
2307         unsigned int cpu;
2308
2309         get_online_cpus();
2310         for_each_online_cpu(cpu) {
2311                 if (all_cpu_data[cpu]) {
2312                         if (intel_pstate_driver == &intel_pstate)
2313                                 intel_pstate_clear_update_util_hook(cpu);
2314
2315                         kfree(all_cpu_data[cpu]);
2316                         all_cpu_data[cpu] = NULL;
2317                 }
2318         }
2319         put_online_cpus();
2320         intel_pstate_driver = NULL;
2321 }
2322
2323 static int intel_pstate_register_driver(struct cpufreq_driver *driver)
2324 {
2325         int ret;
2326
2327         memset(&global, 0, sizeof(global));
2328         global.max_perf_pct = 100;
2329
2330         intel_pstate_driver = driver;
2331         ret = cpufreq_register_driver(intel_pstate_driver);
2332         if (ret) {
2333                 intel_pstate_driver_cleanup();
2334                 return ret;
2335         }
2336
2337         global.min_perf_pct = min_perf_pct_min();
2338
2339         return 0;
2340 }
2341
2342 static int intel_pstate_unregister_driver(void)
2343 {
2344         if (hwp_active)
2345                 return -EBUSY;
2346
2347         cpufreq_unregister_driver(intel_pstate_driver);
2348         intel_pstate_driver_cleanup();
2349
2350         return 0;
2351 }
2352
2353 static ssize_t intel_pstate_show_status(char *buf)
2354 {
2355         if (!intel_pstate_driver)
2356                 return sprintf(buf, "off\n");
2357
2358         return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ?
2359                                         "active" : "passive");
2360 }
2361
2362 static int intel_pstate_update_status(const char *buf, size_t size)
2363 {
2364         int ret;
2365
2366         if (size == 3 && !strncmp(buf, "off", size))
2367                 return intel_pstate_driver ?
2368                         intel_pstate_unregister_driver() : -EINVAL;
2369
2370         if (size == 6 && !strncmp(buf, "active", size)) {
2371                 if (intel_pstate_driver) {
2372                         if (intel_pstate_driver == &intel_pstate)
2373                                 return 0;
2374
2375                         ret = intel_pstate_unregister_driver();
2376                         if (ret)
2377                                 return ret;
2378                 }
2379
2380                 return intel_pstate_register_driver(&intel_pstate);
2381         }
2382
2383         if (size == 7 && !strncmp(buf, "passive", size)) {
2384                 if (intel_pstate_driver) {
2385                         if (intel_pstate_driver == &intel_cpufreq)
2386                                 return 0;
2387
2388                         ret = intel_pstate_unregister_driver();
2389                         if (ret)
2390                                 return ret;
2391                 }
2392
2393                 return intel_pstate_register_driver(&intel_cpufreq);
2394         }
2395
2396         return -EINVAL;
2397 }
2398
2399 static int no_load __initdata;
2400 static int no_hwp __initdata;
2401 static int hwp_only __initdata;
2402 static unsigned int force_load __initdata;
2403
2404 static int __init intel_pstate_msrs_not_valid(void)
2405 {
2406         if (!pstate_funcs.get_max() ||
2407             !pstate_funcs.get_min() ||
2408             !pstate_funcs.get_turbo())
2409                 return -ENODEV;
2410
2411         return 0;
2412 }
2413
2414 static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
2415 {
2416         pstate_funcs.get_max   = funcs->get_max;
2417         pstate_funcs.get_max_physical = funcs->get_max_physical;
2418         pstate_funcs.get_min   = funcs->get_min;
2419         pstate_funcs.get_turbo = funcs->get_turbo;
2420         pstate_funcs.get_scaling = funcs->get_scaling;
2421         pstate_funcs.get_val   = funcs->get_val;
2422         pstate_funcs.get_vid   = funcs->get_vid;
2423         pstate_funcs.get_aperf_mperf_shift = funcs->get_aperf_mperf_shift;
2424 }
2425
2426 #ifdef CONFIG_ACPI
2427
2428 static bool __init intel_pstate_no_acpi_pss(void)
2429 {
2430         int i;
2431
2432         for_each_possible_cpu(i) {
2433                 acpi_status status;
2434                 union acpi_object *pss;
2435                 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
2436                 struct acpi_processor *pr = per_cpu(processors, i);
2437
2438                 if (!pr)
2439                         continue;
2440
2441                 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
2442                 if (ACPI_FAILURE(status))
2443                         continue;
2444
2445                 pss = buffer.pointer;
2446                 if (pss && pss->type == ACPI_TYPE_PACKAGE) {
2447                         kfree(pss);
2448                         return false;
2449                 }
2450
2451                 kfree(pss);
2452         }
2453
2454         return true;
2455 }
2456
2457 static bool __init intel_pstate_no_acpi_pcch(void)
2458 {
2459         acpi_status status;
2460         acpi_handle handle;
2461
2462         status = acpi_get_handle(NULL, "\\_SB", &handle);
2463         if (ACPI_FAILURE(status))
2464                 return true;
2465
2466         return !acpi_has_method(handle, "PCCH");
2467 }
2468
2469 static bool __init intel_pstate_has_acpi_ppc(void)
2470 {
2471         int i;
2472
2473         for_each_possible_cpu(i) {
2474                 struct acpi_processor *pr = per_cpu(processors, i);
2475
2476                 if (!pr)
2477                         continue;
2478                 if (acpi_has_method(pr->handle, "_PPC"))
2479                         return true;
2480         }
2481         return false;
2482 }
2483
2484 enum {
2485         PSS,
2486         PPC,
2487 };
2488
2489 /* Hardware vendor-specific info that has its own power management modes */
2490 static struct acpi_platform_list plat_info[] __initdata = {
2491         {"HP    ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, 0, PSS},
2492         {"ORACLE", "X4-2    ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2493         {"ORACLE", "X4-2L   ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2494         {"ORACLE", "X4-2B   ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2495         {"ORACLE", "X3-2    ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2496         {"ORACLE", "X3-2L   ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2497         {"ORACLE", "X3-2B   ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2498         {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2499         {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2500         {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2501         {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2502         {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2503         {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2504         {"ORACLE", "X6-2    ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2505         {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2506         { } /* End */
2507 };
2508
2509 static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
2510 {
2511         const struct x86_cpu_id *id;
2512         u64 misc_pwr;
2513         int idx;
2514
2515         id = x86_match_cpu(intel_pstate_cpu_oob_ids);
2516         if (id) {
2517                 rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
2518                 if ( misc_pwr & (1 << 8))
2519                         return true;
2520         }
2521
2522         idx = acpi_match_platform_list(plat_info);
2523         if (idx < 0)
2524                 return false;
2525
2526         switch (plat_info[idx].data) {
2527         case PSS:
2528                 if (!intel_pstate_no_acpi_pss())
2529                         return false;
2530
2531                 return intel_pstate_no_acpi_pcch();
2532         case PPC:
2533                 return intel_pstate_has_acpi_ppc() && !force_load;
2534         }
2535
2536         return false;
2537 }
2538
2539 static void intel_pstate_request_control_from_smm(void)
2540 {
2541         /*
2542          * It may be unsafe to request P-states control from SMM if _PPC support
2543          * has not been enabled.
2544          */
2545         if (acpi_ppc)
2546                 acpi_processor_pstate_control();
2547 }
2548 #else /* CONFIG_ACPI not enabled */
2549 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
2550 static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
2551 static inline void intel_pstate_request_control_from_smm(void) {}
2552 #endif /* CONFIG_ACPI */
2553
2554 #define INTEL_PSTATE_HWP_BROADWELL      0x01
2555
2556 #define ICPU_HWP(model, hwp_mode) \
2557         { X86_VENDOR_INTEL, 6, model, X86_FEATURE_HWP, hwp_mode }
2558
2559 static const struct x86_cpu_id hwp_support_ids[] __initconst = {
2560         ICPU_HWP(INTEL_FAM6_BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL),
2561         ICPU_HWP(INTEL_FAM6_BROADWELL_XEON_D, INTEL_PSTATE_HWP_BROADWELL),
2562         ICPU_HWP(X86_MODEL_ANY, 0),
2563         {}
2564 };
2565
2566 static int __init intel_pstate_init(void)
2567 {
2568         const struct x86_cpu_id *id;
2569         int rc;
2570
2571         if (no_load)
2572                 return -ENODEV;
2573
2574         id = x86_match_cpu(hwp_support_ids);
2575         if (id) {
2576                 copy_cpu_funcs(&core_funcs);
2577                 if (!no_hwp) {
2578                         hwp_active++;
2579                         hwp_mode_bdw = id->driver_data;
2580                         intel_pstate.attr = hwp_cpufreq_attrs;
2581                         goto hwp_cpu_matched;
2582                 }
2583         } else {
2584                 id = x86_match_cpu(intel_pstate_cpu_ids);
2585                 if (!id)
2586                         return -ENODEV;
2587
2588                 copy_cpu_funcs((struct pstate_funcs *)id->driver_data);
2589         }
2590
2591         if (intel_pstate_msrs_not_valid())
2592                 return -ENODEV;
2593
2594 hwp_cpu_matched:
2595         /*
2596          * The Intel pstate driver will be ignored if the platform
2597          * firmware has its own power management modes.
2598          */
2599         if (intel_pstate_platform_pwr_mgmt_exists())
2600                 return -ENODEV;
2601
2602         if (!hwp_active && hwp_only)
2603                 return -ENOTSUPP;
2604
2605         pr_info("Intel P-state driver initializing\n");
2606
2607         all_cpu_data = vzalloc(array_size(sizeof(void *), num_possible_cpus()));
2608         if (!all_cpu_data)
2609                 return -ENOMEM;
2610
2611         intel_pstate_request_control_from_smm();
2612
2613         intel_pstate_sysfs_expose_params();
2614
2615         mutex_lock(&intel_pstate_driver_lock);
2616         rc = intel_pstate_register_driver(default_driver);
2617         mutex_unlock(&intel_pstate_driver_lock);
2618         if (rc)
2619                 return rc;
2620
2621         if (hwp_active)
2622                 pr_info("HWP enabled\n");
2623
2624         return 0;
2625 }
2626 device_initcall(intel_pstate_init);
2627
2628 static int __init intel_pstate_setup(char *str)
2629 {
2630         if (!str)
2631                 return -EINVAL;
2632
2633         if (!strcmp(str, "disable")) {
2634                 no_load = 1;
2635         } else if (!strcmp(str, "passive")) {
2636                 pr_info("Passive mode enabled\n");
2637                 default_driver = &intel_cpufreq;
2638                 no_hwp = 1;
2639         }
2640         if (!strcmp(str, "no_hwp")) {
2641                 pr_info("HWP disabled\n");
2642                 no_hwp = 1;
2643         }
2644         if (!strcmp(str, "force"))
2645                 force_load = 1;
2646         if (!strcmp(str, "hwp_only"))
2647                 hwp_only = 1;
2648         if (!strcmp(str, "per_cpu_perf_limits"))
2649                 per_cpu_limits = true;
2650
2651 #ifdef CONFIG_ACPI
2652         if (!strcmp(str, "support_acpi_ppc"))
2653                 acpi_ppc = true;
2654 #endif
2655
2656         return 0;
2657 }
2658 early_param("intel_pstate", intel_pstate_setup);
2659
2660 MODULE_AUTHOR("Dirk Brandewie <[email protected]>");
2661 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
2662 MODULE_LICENSE("GPL");