drivers/cpufreq/amd-pstate.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * amd-pstate.c - AMD Processor P-state Frequency Driver
   4  *
   5  * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
   6  *
   7  * Author: Huang Rui <[email protected]>
   8  *
   9  * AMD P-State introduces a new CPU performance scaling design for AMD
  10  * processors using the ACPI Collaborative Performance and Power Control (CPPC)
  11  * feature which works with the AMD SMU firmware providing a finer grained
  12  * frequency control range. It is to replace the legacy ACPI P-States control,
  13  * allows a flexible, low-latency interface for the Linux kernel to directly
  14  * communicate the performance hints to hardware.
  15  *
  16  * AMD P-State is supported on recent AMD Zen base CPU series include some of
  17  * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
  18  * P-State supported system. And there are two types of hardware implementations
  19  * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
  20  * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
  21  */
  22
  23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  24
  25 #include <linux/kernel.h>
  26 #include <linux/module.h>
  27 #include <linux/init.h>
  28 #include <linux/smp.h>
  29 #include <linux/sched.h>
  30 #include <linux/cpufreq.h>
  31 #include <linux/compiler.h>
  32 #include <linux/dmi.h>
  33 #include <linux/slab.h>
  34 #include <linux/acpi.h>
  35 #include <linux/io.h>
  36 #include <linux/delay.h>
  37 #include <linux/uaccess.h>
  38 #include <linux/static_call.h>
  39 #include <linux/topology.h>
  40
  41 #include <acpi/processor.h>
  42 #include <acpi/cppc_acpi.h>
  43
  44 #include <asm/msr.h>
  45 #include <asm/processor.h>
  46 #include <asm/cpufeature.h>
  47 #include <asm/cpu_device_id.h>
  48
  49 #include "amd-pstate.h"
  50 #include "amd-pstate-trace.h"
  51
  52 #define AMD_PSTATE_TRANSITION_LATENCY   20000
  53 #define AMD_PSTATE_TRANSITION_DELAY     1000
  54 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600
  55
  56 #define AMD_CPPC_EPP_PERFORMANCE                0x00
  57 #define AMD_CPPC_EPP_BALANCE_PERFORMANCE        0x80
  58 #define AMD_CPPC_EPP_BALANCE_POWERSAVE          0xBF
  59 #define AMD_CPPC_EPP_POWERSAVE                  0xFF
  60
  61 static const char * const amd_pstate_mode_string[] = {
  62         [AMD_PSTATE_UNDEFINED]   = "undefined",
  63         [AMD_PSTATE_DISABLE]     = "disable",
  64         [AMD_PSTATE_PASSIVE]     = "passive",
  65         [AMD_PSTATE_ACTIVE]      = "active",
  66         [AMD_PSTATE_GUIDED]      = "guided",
  67         NULL,
  68 };
  69
  70 const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode)
  71 {
  72         if (mode < 0 || mode >= AMD_PSTATE_MAX)
  73                 return NULL;
  74         return amd_pstate_mode_string[mode];
  75 }
  76 EXPORT_SYMBOL_GPL(amd_pstate_get_mode_string);
  77
  78 struct quirk_entry {
  79         u32 nominal_freq;
  80         u32 lowest_freq;
  81 };
  82
  83 static struct cpufreq_driver *current_pstate_driver;
  84 static struct cpufreq_driver amd_pstate_driver;
  85 static struct cpufreq_driver amd_pstate_epp_driver;
  86 static int cppc_state = AMD_PSTATE_UNDEFINED;
  87 static bool cppc_enabled;
  88 static bool amd_pstate_prefcore = true;
  89 static struct quirk_entry *quirks;
  90
  91 /*
  92  * AMD Energy Preference Performance (EPP)
  93  * The EPP is used in the CCLK DPM controller to drive
  94  * the frequency that a core is going to operate during
  95  * short periods of activity. EPP values will be utilized for
  96  * different OS profiles (balanced, performance, power savings)
  97  * display strings corresponding to EPP index in the
  98  * energy_perf_strings[]
  99  *      index           String
 100  *-------------------------------------
 101  *      0               default
 102  *      1               performance
 103  *      2               balance_performance
 104  *      3               balance_power
 105  *      4               power
 106  */
 107 enum energy_perf_value_index {
 108         EPP_INDEX_DEFAULT = 0,
 109         EPP_INDEX_PERFORMANCE,
 110         EPP_INDEX_BALANCE_PERFORMANCE,
 111         EPP_INDEX_BALANCE_POWERSAVE,
 112         EPP_INDEX_POWERSAVE,
 113 };
 114
 115 static const char * const energy_perf_strings[] = {
 116         [EPP_INDEX_DEFAULT] = "default",
 117         [EPP_INDEX_PERFORMANCE] = "performance",
 118         [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
 119         [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
 120         [EPP_INDEX_POWERSAVE] = "power",
 121         NULL
 122 };
 123
 124 static unsigned int epp_values[] = {
 125         [EPP_INDEX_DEFAULT] = 0,
 126         [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE,
 127         [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE,
 128         [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE,
 129         [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE,
 130  };
 131
 132 typedef int (*cppc_mode_transition_fn)(int);
 133
 134 static struct quirk_entry quirk_amd_7k62 = {
 135         .nominal_freq = 2600,
 136         .lowest_freq = 550,
 137 };
 138
 139 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi)
 140 {
 141         /**
 142          * match the broken bios for family 17h processor support CPPC V2
 143          * broken BIOS lack of nominal_freq and lowest_freq capabilities
 144          * definition in ACPI tables
 145          */
 146         if (cpu_feature_enabled(X86_FEATURE_ZEN2)) {
 147                 quirks = dmi->driver_data;
 148                 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident);
 149                 return 1;
 150         }
 151
 152         return 0;
 153 }
 154
 155 static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = {
 156         {
 157                 .callback = dmi_matched_7k62_bios_bug,
 158                 .ident = "AMD EPYC 7K62",
 159                 .matches = {
 160                         DMI_MATCH(DMI_BIOS_VERSION, "5.14"),
 161                         DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"),
 162                 },
 163                 .driver_data = &quirk_amd_7k62,
 164         },
 165         {}
 166 };
 167 MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table);
 168
 169 static inline int get_mode_idx_from_str(const char *str, size_t size)
 170 {
 171         int i;
 172
 173         for (i=0; i < AMD_PSTATE_MAX; i++) {
 174                 if (!strncmp(str, amd_pstate_mode_string[i], size))
 175                         return i;
 176         }
 177         return -EINVAL;
 178 }
 179
 180 static DEFINE_MUTEX(amd_pstate_limits_lock);
 181 static DEFINE_MUTEX(amd_pstate_driver_lock);
 182
 183 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached)
 184 {
 185         u64 epp;
 186         int ret;
 187
 188         if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
 189                 if (!cppc_req_cached) {
 190                         epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
 191                                         &cppc_req_cached);
 192                         if (epp)
 193                                 return epp;
 194                 }
 195                 epp = (cppc_req_cached >> 24) & 0xFF;
 196         } else {
 197                 ret = cppc_get_epp_perf(cpudata->cpu, &epp);
 198                 if (ret < 0) {
 199                         pr_debug("Could not retrieve energy perf value (%d)\n", ret);
 200                         return -EIO;
 201                 }
 202         }
 203
 204         return (s16)(epp & 0xff);
 205 }
 206
 207 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata)
 208 {
 209         s16 epp;
 210         int index = -EINVAL;
 211
 212         epp = amd_pstate_get_epp(cpudata, 0);
 213         if (epp < 0)
 214                 return epp;
 215
 216         switch (epp) {
 217         case AMD_CPPC_EPP_PERFORMANCE:
 218                 index = EPP_INDEX_PERFORMANCE;
 219                 break;
 220         case AMD_CPPC_EPP_BALANCE_PERFORMANCE:
 221                 index = EPP_INDEX_BALANCE_PERFORMANCE;
 222                 break;
 223         case AMD_CPPC_EPP_BALANCE_POWERSAVE:
 224                 index = EPP_INDEX_BALANCE_POWERSAVE;
 225                 break;
 226         case AMD_CPPC_EPP_POWERSAVE:
 227                 index = EPP_INDEX_POWERSAVE;
 228                 break;
 229         default:
 230                 break;
 231         }
 232
 233         return index;
 234 }
 235
 236 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,
 237                                u32 des_perf, u32 max_perf, bool fast_switch)
 238 {
 239         if (fast_switch)
 240                 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));
 241         else
 242                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
 243                               READ_ONCE(cpudata->cppc_req_cached));
 244 }
 245
 246 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf);
 247
 248 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,
 249                                           u32 min_perf, u32 des_perf,
 250                                           u32 max_perf, bool fast_switch)
 251 {
 252         static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,
 253                                             max_perf, fast_switch);
 254 }
 255
 256 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp)
 257 {
 258         int ret;
 259         struct cppc_perf_ctrls perf_ctrls;
 260
 261         if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
 262                 u64 value = READ_ONCE(cpudata->cppc_req_cached);
 263
 264                 value &= ~GENMASK_ULL(31, 24);
 265                 value |= (u64)epp << 24;
 266                 WRITE_ONCE(cpudata->cppc_req_cached, value);
 267
 268                 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
 269                 if (!ret)
 270                         cpudata->epp_cached = epp;
 271         } else {
 272                 amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U,
 273                                              cpudata->max_limit_perf, false);
 274
 275                 perf_ctrls.energy_perf = epp;
 276                 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1);
 277                 if (ret) {
 278                         pr_debug("failed to set energy perf value (%d)\n", ret);
 279                         return ret;
 280                 }
 281                 cpudata->epp_cached = epp;
 282         }
 283
 284         return ret;
 285 }
 286
 287 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata,
 288                 int pref_index)
 289 {
 290         int epp = -EINVAL;
 291         int ret;
 292
 293         if (!pref_index)
 294                 epp = cpudata->epp_default;
 295
 296         if (epp == -EINVAL)
 297                 epp = epp_values[pref_index];
 298
 299         if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
 300                 pr_debug("EPP cannot be set under performance policy\n");
 301                 return -EBUSY;
 302         }
 303
 304         ret = amd_pstate_set_epp(cpudata, epp);
 305
 306         return ret;
 307 }
 308
 309 static inline int pstate_enable(bool enable)
 310 {
 311         int ret, cpu;
 312         unsigned long logical_proc_id_mask = 0;
 313
 314         if (enable == cppc_enabled)
 315                 return 0;
 316
 317         for_each_present_cpu(cpu) {
 318                 unsigned long logical_id = topology_logical_package_id(cpu);
 319
 320                 if (test_bit(logical_id, &logical_proc_id_mask))
 321                         continue;
 322
 323                 set_bit(logical_id, &logical_proc_id_mask);
 324
 325                 ret = wrmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_ENABLE,
 326                                 enable);
 327                 if (ret)
 328                         return ret;
 329         }
 330
 331         cppc_enabled = enable;
 332         return 0;
 333 }
 334
 335 static int cppc_enable(bool enable)
 336 {
 337         int cpu, ret = 0;
 338         struct cppc_perf_ctrls perf_ctrls;
 339
 340         if (enable == cppc_enabled)
 341                 return 0;
 342
 343         for_each_present_cpu(cpu) {
 344                 ret = cppc_set_enable(cpu, enable);
 345                 if (ret)
 346                         return ret;
 347
 348                 /* Enable autonomous mode for EPP */
 349                 if (cppc_state == AMD_PSTATE_ACTIVE) {
 350                         /* Set desired perf as zero to allow EPP firmware control */
 351                         perf_ctrls.desired_perf = 0;
 352                         ret = cppc_set_perf(cpu, &perf_ctrls);
 353                         if (ret)
 354                                 return ret;
 355                 }
 356         }
 357
 358         cppc_enabled = enable;
 359         return ret;
 360 }
 361
 362 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable);
 363
 364 static inline int amd_pstate_enable(bool enable)
 365 {
 366         return static_call(amd_pstate_enable)(enable);
 367 }
 368
 369 static int pstate_init_perf(struct amd_cpudata *cpudata)
 370 {
 371         u64 cap1;
 372
 373         int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
 374                                      &cap1);
 375         if (ret)
 376                 return ret;
 377
 378         WRITE_ONCE(cpudata->highest_perf, AMD_CPPC_HIGHEST_PERF(cap1));
 379         WRITE_ONCE(cpudata->max_limit_perf, AMD_CPPC_HIGHEST_PERF(cap1));
 380         WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
 381         WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
 382         WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));
 383         WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1));
 384         WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1));
 385         return 0;
 386 }
 387
 388 static int cppc_init_perf(struct amd_cpudata *cpudata)
 389 {
 390         struct cppc_perf_caps cppc_perf;
 391
 392         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 393         if (ret)
 394                 return ret;
 395
 396         WRITE_ONCE(cpudata->highest_perf, cppc_perf.highest_perf);
 397         WRITE_ONCE(cpudata->max_limit_perf, cppc_perf.highest_perf);
 398         WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
 399         WRITE_ONCE(cpudata->lowest_nonlinear_perf,
 400                    cppc_perf.lowest_nonlinear_perf);
 401         WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
 402         WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf);
 403         WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf);
 404
 405         if (cppc_state == AMD_PSTATE_ACTIVE)
 406                 return 0;
 407
 408         ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf);
 409         if (ret) {
 410                 pr_warn("failed to get auto_sel, ret: %d\n", ret);
 411                 return 0;
 412         }
 413
 414         ret = cppc_set_auto_sel(cpudata->cpu,
 415                         (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
 416
 417         if (ret)
 418                 pr_warn("failed to set auto_sel, ret: %d\n", ret);
 419
 420         return ret;
 421 }
 422
 423 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
 424
 425 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
 426 {
 427         return static_call(amd_pstate_init_perf)(cpudata);
 428 }
 429
 430 static void cppc_update_perf(struct amd_cpudata *cpudata,
 431                              u32 min_perf, u32 des_perf,
 432                              u32 max_perf, bool fast_switch)
 433 {
 434         struct cppc_perf_ctrls perf_ctrls;
 435
 436         perf_ctrls.max_perf = max_perf;
 437         perf_ctrls.min_perf = min_perf;
 438         perf_ctrls.desired_perf = des_perf;
 439
 440         cppc_set_perf(cpudata->cpu, &perf_ctrls);
 441 }
 442
 443 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
 444 {
 445         u64 aperf, mperf, tsc;
 446         unsigned long flags;
 447
 448         local_irq_save(flags);
 449         rdmsrl(MSR_IA32_APERF, aperf);
 450         rdmsrl(MSR_IA32_MPERF, mperf);
 451         tsc = rdtsc();
 452
 453         if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
 454                 local_irq_restore(flags);
 455                 return false;
 456         }
 457
 458         local_irq_restore(flags);
 459
 460         cpudata->cur.aperf = aperf;
 461         cpudata->cur.mperf = mperf;
 462         cpudata->cur.tsc =  tsc;
 463         cpudata->cur.aperf -= cpudata->prev.aperf;
 464         cpudata->cur.mperf -= cpudata->prev.mperf;
 465         cpudata->cur.tsc -= cpudata->prev.tsc;
 466
 467         cpudata->prev.aperf = aperf;
 468         cpudata->prev.mperf = mperf;
 469         cpudata->prev.tsc = tsc;
 470
 471         cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
 472
 473         return true;
 474 }
 475
 476 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
 477                               u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags)
 478 {
 479         unsigned long max_freq;
 480         struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu);
 481         u64 prev = READ_ONCE(cpudata->cppc_req_cached);
 482         u32 nominal_perf = READ_ONCE(cpudata->nominal_perf);
 483         u64 value = prev;
 484
 485         min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf,
 486                         cpudata->max_limit_perf);
 487         max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf,
 488                         cpudata->max_limit_perf);
 489         des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
 490
 491         max_freq = READ_ONCE(cpudata->max_limit_freq);
 492         policy->cur = div_u64(des_perf * max_freq, max_perf);
 493
 494         if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) {
 495                 min_perf = des_perf;
 496                 des_perf = 0;
 497         }
 498
 499         value &= ~AMD_CPPC_MIN_PERF(~0L);
 500         value |= AMD_CPPC_MIN_PERF(min_perf);
 501
 502         value &= ~AMD_CPPC_DES_PERF(~0L);
 503         value |= AMD_CPPC_DES_PERF(des_perf);
 504
 505         /* limit the max perf when core performance boost feature is disabled */
 506         if (!cpudata->boost_supported)
 507                 max_perf = min_t(unsigned long, nominal_perf, max_perf);
 508
 509         value &= ~AMD_CPPC_MAX_PERF(~0L);
 510         value |= AMD_CPPC_MAX_PERF(max_perf);
 511
 512         if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
 513                 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
 514                         cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
 515                                 cpudata->cpu, (value != prev), fast_switch);
 516         }
 517
 518         if (value == prev)
 519                 goto cpufreq_policy_put;
 520
 521         WRITE_ONCE(cpudata->cppc_req_cached, value);
 522
 523         amd_pstate_update_perf(cpudata, min_perf, des_perf,
 524                                max_perf, fast_switch);
 525
 526 cpufreq_policy_put:
 527         cpufreq_cpu_put(policy);
 528 }
 529
 530 static int amd_pstate_verify(struct cpufreq_policy_data *policy)
 531 {
 532         cpufreq_verify_within_cpu_limits(policy);
 533
 534         return 0;
 535 }
 536
 537 static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy)
 538 {
 539         u32 max_limit_perf, min_limit_perf, lowest_perf;
 540         struct amd_cpudata *cpudata = policy->driver_data;
 541
 542         max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq);
 543         min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq);
 544
 545         lowest_perf = READ_ONCE(cpudata->lowest_perf);
 546         if (min_limit_perf < lowest_perf)
 547                 min_limit_perf = lowest_perf;
 548
 549         if (max_limit_perf < min_limit_perf)
 550                 max_limit_perf = min_limit_perf;
 551
 552         WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf);
 553         WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf);
 554         WRITE_ONCE(cpudata->max_limit_freq, policy->max);
 555         WRITE_ONCE(cpudata->min_limit_freq, policy->min);
 556
 557         return 0;
 558 }
 559
 560 static int amd_pstate_update_freq(struct cpufreq_policy *policy,
 561                                   unsigned int target_freq, bool fast_switch)
 562 {
 563         struct cpufreq_freqs freqs;
 564         struct amd_cpudata *cpudata = policy->driver_data;
 565         unsigned long max_perf, min_perf, des_perf, cap_perf;
 566
 567         if (!cpudata->max_freq)
 568                 return -ENODEV;
 569
 570         if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)
 571                 amd_pstate_update_min_max_limit(policy);
 572
 573         cap_perf = READ_ONCE(cpudata->highest_perf);
 574         min_perf = READ_ONCE(cpudata->lowest_perf);
 575         max_perf = cap_perf;
 576
 577         freqs.old = policy->cur;
 578         freqs.new = target_freq;
 579
 580         des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf,
 581                                      cpudata->max_freq);
 582
 583         WARN_ON(fast_switch && !policy->fast_switch_enabled);
 584         /*
 585          * If fast_switch is desired, then there aren't any registered
 586          * transition notifiers. See comment for
 587          * cpufreq_enable_fast_switch().
 588          */
 589         if (!fast_switch)
 590                 cpufreq_freq_transition_begin(policy, &freqs);
 591
 592         amd_pstate_update(cpudata, min_perf, des_perf,
 593                         max_perf, fast_switch, policy->governor->flags);
 594
 595         if (!fast_switch)
 596                 cpufreq_freq_transition_end(policy, &freqs, false);
 597
 598         return 0;
 599 }
 600
 601 static int amd_pstate_target(struct cpufreq_policy *policy,
 602                              unsigned int target_freq,
 603                              unsigned int relation)
 604 {
 605         return amd_pstate_update_freq(policy, target_freq, false);
 606 }
 607
 608 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy,
 609                                   unsigned int target_freq)
 610 {
 611         if (!amd_pstate_update_freq(policy, target_freq, true))
 612                 return target_freq;
 613         return policy->cur;
 614 }
 615
 616 static void amd_pstate_adjust_perf(unsigned int cpu,
 617                                    unsigned long _min_perf,
 618                                    unsigned long target_perf,
 619                                    unsigned long capacity)
 620 {
 621         unsigned long max_perf, min_perf, des_perf,
 622                       cap_perf, lowest_nonlinear_perf;
 623         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
 624         struct amd_cpudata *cpudata;
 625
 626         if (!policy)
 627                 return;
 628
 629         cpudata = policy->driver_data;
 630
 631         if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)
 632                 amd_pstate_update_min_max_limit(policy);
 633
 634
 635         cap_perf = READ_ONCE(cpudata->highest_perf);
 636         lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
 637
 638         des_perf = cap_perf;
 639         if (target_perf < capacity)
 640                 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
 641
 642         min_perf = READ_ONCE(cpudata->lowest_perf);
 643         if (_min_perf < capacity)
 644                 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
 645
 646         if (min_perf < lowest_nonlinear_perf)
 647                 min_perf = lowest_nonlinear_perf;
 648
 649         max_perf = cap_perf;
 650         if (max_perf < min_perf)
 651                 max_perf = min_perf;
 652
 653         des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
 654
 655         amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true,
 656                         policy->governor->flags);
 657         cpufreq_cpu_put(policy);
 658 }
 659
 660 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on)
 661 {
 662         struct amd_cpudata *cpudata = policy->driver_data;
 663         struct cppc_perf_ctrls perf_ctrls;
 664         u32 highest_perf, nominal_perf, nominal_freq, max_freq;
 665         int ret = 0;
 666
 667         highest_perf = READ_ONCE(cpudata->highest_perf);
 668         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 669         nominal_freq = READ_ONCE(cpudata->nominal_freq);
 670         max_freq = READ_ONCE(cpudata->max_freq);
 671
 672         if (boot_cpu_has(X86_FEATURE_CPPC)) {
 673                 u64 value = READ_ONCE(cpudata->cppc_req_cached);
 674
 675                 value &= ~GENMASK_ULL(7, 0);
 676                 value |= on ? highest_perf : nominal_perf;
 677                 WRITE_ONCE(cpudata->cppc_req_cached, value);
 678
 679                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
 680         } else {
 681                 perf_ctrls.max_perf = on ? highest_perf : nominal_perf;
 682                 ret = cppc_set_perf(cpudata->cpu, &perf_ctrls);
 683                 if (ret) {
 684                         cpufreq_cpu_release(policy);
 685                         pr_debug("Failed to set max perf on CPU:%d. ret:%d\n",
 686                                 cpudata->cpu, ret);
 687                         return ret;
 688                 }
 689         }
 690
 691         if (on)
 692                 policy->cpuinfo.max_freq = max_freq;
 693         else if (policy->cpuinfo.max_freq > nominal_freq * 1000)
 694                 policy->cpuinfo.max_freq = nominal_freq * 1000;
 695
 696         policy->max = policy->cpuinfo.max_freq;
 697
 698         if (cppc_state == AMD_PSTATE_PASSIVE) {
 699                 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq);
 700                 if (ret < 0)
 701                         pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu);
 702         }
 703
 704         return ret < 0 ? ret : 0;
 705 }
 706
 707 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
 708 {
 709         struct amd_cpudata *cpudata = policy->driver_data;
 710         int ret;
 711
 712         if (!cpudata->boost_supported) {
 713                 pr_err("Boost mode is not supported by this processor or SBIOS\n");
 714                 return -EOPNOTSUPP;
 715         }
 716         mutex_lock(&amd_pstate_driver_lock);
 717         ret = amd_pstate_cpu_boost_update(policy, state);
 718         WRITE_ONCE(cpudata->boost_state, !ret ? state : false);
 719         policy->boost_enabled = !ret ? state : false;
 720         refresh_frequency_limits(policy);
 721         mutex_unlock(&amd_pstate_driver_lock);
 722
 723         return ret;
 724 }
 725
 726 static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata)
 727 {
 728         u64 boost_val;
 729         int ret = -1;
 730
 731         /*
 732          * If platform has no CPB support or disable it, initialize current driver
 733          * boost_enabled state to be false, it is not an error for cpufreq core to handle.
 734          */
 735         if (!cpu_feature_enabled(X86_FEATURE_CPB)) {
 736                 pr_debug_once("Boost CPB capabilities not present in the processor\n");
 737                 ret = 0;
 738                 goto exit_err;
 739         }
 740
 741         /* at least one CPU supports CPB, even if others fail later on to set up */
 742         current_pstate_driver->boost_enabled = true;
 743
 744         ret = rdmsrl_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val);
 745         if (ret) {
 746                 pr_err_once("failed to read initial CPU boost state!\n");
 747                 ret = -EIO;
 748                 goto exit_err;
 749         }
 750
 751         if (!(boost_val & MSR_K7_HWCR_CPB_DIS))
 752                 cpudata->boost_supported = true;
 753
 754         return 0;
 755
 756 exit_err:
 757         cpudata->boost_supported = false;
 758         return ret;
 759 }
 760
 761 static void amd_perf_ctl_reset(unsigned int cpu)
 762 {
 763         wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0);
 764 }
 765
 766 /*
 767  * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks
 768  * due to locking, so queue the work for later.
 769  */
 770 static void amd_pstste_sched_prefcore_workfn(struct work_struct *work)
 771 {
 772         sched_set_itmt_support();
 773 }
 774 static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn);
 775
 776 #define CPPC_MAX_PERF   U8_MAX
 777
 778 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata)
 779 {
 780         /* user disabled or not detected */
 781         if (!amd_pstate_prefcore)
 782                 return;
 783
 784         cpudata->hw_prefcore = true;
 785
 786         /*
 787          * The priorities can be set regardless of whether or not
 788          * sched_set_itmt_support(true) has been called and it is valid to
 789          * update them at any time after it has been called.
 790          */
 791         sched_set_itmt_core_prio((int)READ_ONCE(cpudata->highest_perf), cpudata->cpu);
 792
 793         schedule_work(&sched_prefcore_work);
 794 }
 795
 796 static void amd_pstate_update_limits(unsigned int cpu)
 797 {
 798         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
 799         struct amd_cpudata *cpudata;
 800         u32 prev_high = 0, cur_high = 0;
 801         int ret;
 802         bool highest_perf_changed = false;
 803
 804         if (!policy)
 805                 return;
 806
 807         cpudata = policy->driver_data;
 808
 809         if (!amd_pstate_prefcore)
 810                 return;
 811
 812         mutex_lock(&amd_pstate_driver_lock);
 813         ret = amd_get_highest_perf(cpu, &cur_high);
 814         if (ret)
 815                 goto free_cpufreq_put;
 816
 817         prev_high = READ_ONCE(cpudata->prefcore_ranking);
 818         highest_perf_changed = (prev_high != cur_high);
 819         if (highest_perf_changed) {
 820                 WRITE_ONCE(cpudata->prefcore_ranking, cur_high);
 821
 822                 if (cur_high < CPPC_MAX_PERF)
 823                         sched_set_itmt_core_prio((int)cur_high, cpu);
 824         }
 825
 826 free_cpufreq_put:
 827         cpufreq_cpu_put(policy);
 828
 829         if (!highest_perf_changed)
 830                 cpufreq_update_policy(cpu);
 831
 832         mutex_unlock(&amd_pstate_driver_lock);
 833 }
 834
 835 /*
 836  * Get pstate transition delay time from ACPI tables that firmware set
 837  * instead of using hardcode value directly.
 838  */
 839 static u32 amd_pstate_get_transition_delay_us(unsigned int cpu)
 840 {
 841         u32 transition_delay_ns;
 842
 843         transition_delay_ns = cppc_get_transition_latency(cpu);
 844         if (transition_delay_ns == CPUFREQ_ETERNAL) {
 845                 if (cpu_feature_enabled(X86_FEATURE_FAST_CPPC))
 846                         return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY;
 847                 else
 848                         return AMD_PSTATE_TRANSITION_DELAY;
 849         }
 850
 851         return transition_delay_ns / NSEC_PER_USEC;
 852 }
 853
 854 /*
 855  * Get pstate transition latency value from ACPI tables that firmware
 856  * set instead of using hardcode value directly.
 857  */
 858 static u32 amd_pstate_get_transition_latency(unsigned int cpu)
 859 {
 860         u32 transition_latency;
 861
 862         transition_latency = cppc_get_transition_latency(cpu);
 863         if (transition_latency  == CPUFREQ_ETERNAL)
 864                 return AMD_PSTATE_TRANSITION_LATENCY;
 865
 866         return transition_latency;
 867 }
 868
 869 /*
 870  * amd_pstate_init_freq: Initialize the max_freq, min_freq,
 871  *                       nominal_freq and lowest_nonlinear_freq for
 872  *                       the @cpudata object.
 873  *
 874  *  Requires: highest_perf, lowest_perf, nominal_perf and
 875  *            lowest_nonlinear_perf members of @cpudata to be
 876  *            initialized.
 877  *
 878  *  Returns 0 on success, non-zero value on failure.
 879  */
 880 static int amd_pstate_init_freq(struct amd_cpudata *cpudata)
 881 {
 882         int ret;
 883         u32 min_freq, max_freq;
 884         u64 numerator;
 885         u32 nominal_perf, nominal_freq;
 886         u32 lowest_nonlinear_perf, lowest_nonlinear_freq;
 887         u32 boost_ratio, lowest_nonlinear_ratio;
 888         struct cppc_perf_caps cppc_perf;
 889
 890         ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 891         if (ret)
 892                 return ret;
 893
 894         if (quirks && quirks->lowest_freq)
 895                 min_freq = quirks->lowest_freq * 1000;
 896         else
 897                 min_freq = cppc_perf.lowest_freq * 1000;
 898
 899         if (quirks && quirks->nominal_freq)
 900                 nominal_freq = quirks->nominal_freq ;
 901         else
 902                 nominal_freq = cppc_perf.nominal_freq;
 903
 904         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 905
 906         ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator);
 907         if (ret)
 908                 return ret;
 909         boost_ratio = div_u64(numerator << SCHED_CAPACITY_SHIFT, nominal_perf);
 910         max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT) * 1000;
 911
 912         lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
 913         lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT,
 914                                          nominal_perf);
 915         lowest_nonlinear_freq = (nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT) * 1000;
 916
 917         WRITE_ONCE(cpudata->min_freq, min_freq);
 918         WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq);
 919         WRITE_ONCE(cpudata->nominal_freq, nominal_freq);
 920         WRITE_ONCE(cpudata->max_freq, max_freq);
 921
 922         /**
 923          * Below values need to be initialized correctly, otherwise driver will fail to load
 924          * max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf
 925          * lowest_nonlinear_freq is a value between [min_freq, nominal_freq]
 926          * Check _CPC in ACPI table objects if any values are incorrect
 927          */
 928         if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) {
 929                 pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n",
 930                         min_freq, max_freq, nominal_freq * 1000);
 931                 return -EINVAL;
 932         }
 933
 934         if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq * 1000) {
 935                 pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n",
 936                         lowest_nonlinear_freq, min_freq, nominal_freq * 1000);
 937                 return -EINVAL;
 938         }
 939
 940         return 0;
 941 }
 942
 943 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
 944 {
 945         int min_freq, max_freq, ret;
 946         struct device *dev;
 947         struct amd_cpudata *cpudata;
 948
 949         /*
 950          * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
 951          * which is ideal for initialization process.
 952          */
 953         amd_perf_ctl_reset(policy->cpu);
 954         dev = get_cpu_device(policy->cpu);
 955         if (!dev)
 956                 return -ENODEV;
 957
 958         cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
 959         if (!cpudata)
 960                 return -ENOMEM;
 961
 962         cpudata->cpu = policy->cpu;
 963
 964         ret = amd_pstate_init_perf(cpudata);
 965         if (ret)
 966                 goto free_cpudata1;
 967
 968         amd_pstate_init_prefcore(cpudata);
 969
 970         ret = amd_pstate_init_freq(cpudata);
 971         if (ret)
 972                 goto free_cpudata1;
 973
 974         ret = amd_pstate_init_boost_support(cpudata);
 975         if (ret)
 976                 goto free_cpudata1;
 977
 978         min_freq = READ_ONCE(cpudata->min_freq);
 979         max_freq = READ_ONCE(cpudata->max_freq);
 980
 981         policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu);
 982         policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu);
 983
 984         policy->min = min_freq;
 985         policy->max = max_freq;
 986
 987         policy->cpuinfo.min_freq = min_freq;
 988         policy->cpuinfo.max_freq = max_freq;
 989
 990         policy->boost_enabled = READ_ONCE(cpudata->boost_supported);
 991
 992         /* It will be updated by governor */
 993         policy->cur = policy->cpuinfo.min_freq;
 994
 995         if (cpu_feature_enabled(X86_FEATURE_CPPC))
 996                 policy->fast_switch_possible = true;
 997
 998         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
 999                                    FREQ_QOS_MIN, policy->cpuinfo.min_freq);
1000         if (ret < 0) {
1001                 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
1002                 goto free_cpudata1;
1003         }
1004
1005         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
1006                                    FREQ_QOS_MAX, policy->cpuinfo.max_freq);
1007         if (ret < 0) {
1008                 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
1009                 goto free_cpudata2;
1010         }
1011
1012         cpudata->max_limit_freq = max_freq;
1013         cpudata->min_limit_freq = min_freq;
1014
1015         policy->driver_data = cpudata;
1016
1017         if (!current_pstate_driver->adjust_perf)
1018                 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
1019
1020         return 0;
1021
1022 free_cpudata2:
1023         freq_qos_remove_request(&cpudata->req[0]);
1024 free_cpudata1:
1025         kfree(cpudata);
1026         return ret;
1027 }
1028
1029 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy)
1030 {
1031         struct amd_cpudata *cpudata = policy->driver_data;
1032
1033         freq_qos_remove_request(&cpudata->req[1]);
1034         freq_qos_remove_request(&cpudata->req[0]);
1035         policy->fast_switch_possible = false;
1036         kfree(cpudata);
1037 }
1038
1039 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)
1040 {
1041         int ret;
1042
1043         ret = amd_pstate_enable(true);
1044         if (ret)
1045                 pr_err("failed to enable amd-pstate during resume, return %d\n", ret);
1046
1047         return ret;
1048 }
1049
1050 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)
1051 {
1052         int ret;
1053
1054         ret = amd_pstate_enable(false);
1055         if (ret)
1056                 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret);
1057
1058         return ret;
1059 }
1060
1061 /* Sysfs attributes */
1062
1063 /*
1064  * This frequency is to indicate the maximum hardware frequency.
1065  * If boost is not active but supported, the frequency will be larger than the
1066  * one in cpuinfo.
1067  */
1068 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
1069                                         char *buf)
1070 {
1071         int max_freq;
1072         struct amd_cpudata *cpudata = policy->driver_data;
1073
1074         max_freq = READ_ONCE(cpudata->max_freq);
1075         if (max_freq < 0)
1076                 return max_freq;
1077
1078         return sysfs_emit(buf, "%u\n", max_freq);
1079 }
1080
1081 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
1082                                                      char *buf)
1083 {
1084         int freq;
1085         struct amd_cpudata *cpudata = policy->driver_data;
1086
1087         freq = READ_ONCE(cpudata->lowest_nonlinear_freq);
1088         if (freq < 0)
1089                 return freq;
1090
1091         return sysfs_emit(buf, "%u\n", freq);
1092 }
1093
1094 /*
1095  * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
1096  * need to expose it to sysfs.
1097  */
1098 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
1099                                             char *buf)
1100 {
1101         u32 perf;
1102         struct amd_cpudata *cpudata = policy->driver_data;
1103
1104         perf = READ_ONCE(cpudata->highest_perf);
1105
1106         return sysfs_emit(buf, "%u\n", perf);
1107 }
1108
1109 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy,
1110                                                 char *buf)
1111 {
1112         u32 perf;
1113         struct amd_cpudata *cpudata = policy->driver_data;
1114
1115         perf = READ_ONCE(cpudata->prefcore_ranking);
1116
1117         return sysfs_emit(buf, "%u\n", perf);
1118 }
1119
1120 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy,
1121                                            char *buf)
1122 {
1123         bool hw_prefcore;
1124         struct amd_cpudata *cpudata = policy->driver_data;
1125
1126         hw_prefcore = READ_ONCE(cpudata->hw_prefcore);
1127
1128         return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore));
1129 }
1130
1131 static ssize_t show_energy_performance_available_preferences(
1132                                 struct cpufreq_policy *policy, char *buf)
1133 {
1134         int i = 0;
1135         int offset = 0;
1136         struct amd_cpudata *cpudata = policy->driver_data;
1137
1138         if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1139                 return sysfs_emit_at(buf, offset, "%s\n",
1140                                 energy_perf_strings[EPP_INDEX_PERFORMANCE]);
1141
1142         while (energy_perf_strings[i] != NULL)
1143                 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]);
1144
1145         offset += sysfs_emit_at(buf, offset, "\n");
1146
1147         return offset;
1148 }
1149
1150 static ssize_t store_energy_performance_preference(
1151                 struct cpufreq_policy *policy, const char *buf, size_t count)
1152 {
1153         struct amd_cpudata *cpudata = policy->driver_data;
1154         char str_preference[21];
1155         ssize_t ret;
1156
1157         ret = sscanf(buf, "%20s", str_preference);
1158         if (ret != 1)
1159                 return -EINVAL;
1160
1161         ret = match_string(energy_perf_strings, -1, str_preference);
1162         if (ret < 0)
1163                 return -EINVAL;
1164
1165         mutex_lock(&amd_pstate_limits_lock);
1166         ret = amd_pstate_set_energy_pref_index(cpudata, ret);
1167         mutex_unlock(&amd_pstate_limits_lock);
1168
1169         return ret ?: count;
1170 }
1171
1172 static ssize_t show_energy_performance_preference(
1173                                 struct cpufreq_policy *policy, char *buf)
1174 {
1175         struct amd_cpudata *cpudata = policy->driver_data;
1176         int preference;
1177
1178         preference = amd_pstate_get_energy_pref_index(cpudata);
1179         if (preference < 0)
1180                 return preference;
1181
1182         return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]);
1183 }
1184
1185 static void amd_pstate_driver_cleanup(void)
1186 {
1187         amd_pstate_enable(false);
1188         cppc_state = AMD_PSTATE_DISABLE;
1189         current_pstate_driver = NULL;
1190 }
1191
1192 static int amd_pstate_register_driver(int mode)
1193 {
1194         int ret;
1195
1196         if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED)
1197                 current_pstate_driver = &amd_pstate_driver;
1198         else if (mode == AMD_PSTATE_ACTIVE)
1199                 current_pstate_driver = &amd_pstate_epp_driver;
1200         else
1201                 return -EINVAL;
1202
1203         cppc_state = mode;
1204         ret = cpufreq_register_driver(current_pstate_driver);
1205         if (ret) {
1206                 amd_pstate_driver_cleanup();
1207                 return ret;
1208         }
1209         return 0;
1210 }
1211
1212 static int amd_pstate_unregister_driver(int dummy)
1213 {
1214         cpufreq_unregister_driver(current_pstate_driver);
1215         amd_pstate_driver_cleanup();
1216         return 0;
1217 }
1218
1219 static int amd_pstate_change_mode_without_dvr_change(int mode)
1220 {
1221         int cpu = 0;
1222
1223         cppc_state = mode;
1224
1225         if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE)
1226                 return 0;
1227
1228         for_each_present_cpu(cpu) {
1229                 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
1230         }
1231
1232         return 0;
1233 }
1234
1235 static int amd_pstate_change_driver_mode(int mode)
1236 {
1237         int ret;
1238
1239         ret = amd_pstate_unregister_driver(0);
1240         if (ret)
1241                 return ret;
1242
1243         ret = amd_pstate_register_driver(mode);
1244         if (ret)
1245                 return ret;
1246
1247         return 0;
1248 }
1249
1250 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = {
1251         [AMD_PSTATE_DISABLE]         = {
1252                 [AMD_PSTATE_DISABLE]     = NULL,
1253                 [AMD_PSTATE_PASSIVE]     = amd_pstate_register_driver,
1254                 [AMD_PSTATE_ACTIVE]      = amd_pstate_register_driver,
1255                 [AMD_PSTATE_GUIDED]      = amd_pstate_register_driver,
1256         },
1257         [AMD_PSTATE_PASSIVE]         = {
1258                 [AMD_PSTATE_DISABLE]     = amd_pstate_unregister_driver,
1259                 [AMD_PSTATE_PASSIVE]     = NULL,
1260                 [AMD_PSTATE_ACTIVE]      = amd_pstate_change_driver_mode,
1261                 [AMD_PSTATE_GUIDED]      = amd_pstate_change_mode_without_dvr_change,
1262         },
1263         [AMD_PSTATE_ACTIVE]          = {
1264                 [AMD_PSTATE_DISABLE]     = amd_pstate_unregister_driver,
1265                 [AMD_PSTATE_PASSIVE]     = amd_pstate_change_driver_mode,
1266                 [AMD_PSTATE_ACTIVE]      = NULL,
1267                 [AMD_PSTATE_GUIDED]      = amd_pstate_change_driver_mode,
1268         },
1269         [AMD_PSTATE_GUIDED]          = {
1270                 [AMD_PSTATE_DISABLE]     = amd_pstate_unregister_driver,
1271                 [AMD_PSTATE_PASSIVE]     = amd_pstate_change_mode_without_dvr_change,
1272                 [AMD_PSTATE_ACTIVE]      = amd_pstate_change_driver_mode,
1273                 [AMD_PSTATE_GUIDED]      = NULL,
1274         },
1275 };
1276
1277 static ssize_t amd_pstate_show_status(char *buf)
1278 {
1279         if (!current_pstate_driver)
1280                 return sysfs_emit(buf, "disable\n");
1281
1282         return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]);
1283 }
1284
1285 int amd_pstate_update_status(const char *buf, size_t size)
1286 {
1287         int mode_idx;
1288
1289         if (size > strlen("passive") || size < strlen("active"))
1290                 return -EINVAL;
1291
1292         mode_idx = get_mode_idx_from_str(buf, size);
1293
1294         if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX)
1295                 return -EINVAL;
1296
1297         if (mode_state_machine[cppc_state][mode_idx])
1298                 return mode_state_machine[cppc_state][mode_idx](mode_idx);
1299
1300         return 0;
1301 }
1302 EXPORT_SYMBOL_GPL(amd_pstate_update_status);
1303
1304 static ssize_t status_show(struct device *dev,
1305                            struct device_attribute *attr, char *buf)
1306 {
1307         ssize_t ret;
1308
1309         mutex_lock(&amd_pstate_driver_lock);
1310         ret = amd_pstate_show_status(buf);
1311         mutex_unlock(&amd_pstate_driver_lock);
1312
1313         return ret;
1314 }
1315
1316 static ssize_t status_store(struct device *a, struct device_attribute *b,
1317                             const char *buf, size_t count)
1318 {
1319         char *p = memchr(buf, '\n', count);
1320         int ret;
1321
1322         mutex_lock(&amd_pstate_driver_lock);
1323         ret = amd_pstate_update_status(buf, p ? p - buf : count);
1324         mutex_unlock(&amd_pstate_driver_lock);
1325
1326         return ret < 0 ? ret : count;
1327 }
1328
1329 static ssize_t prefcore_show(struct device *dev,
1330                              struct device_attribute *attr, char *buf)
1331 {
1332         return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore));
1333 }
1334
1335 cpufreq_freq_attr_ro(amd_pstate_max_freq);
1336 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
1337
1338 cpufreq_freq_attr_ro(amd_pstate_highest_perf);
1339 cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking);
1340 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore);
1341 cpufreq_freq_attr_rw(energy_performance_preference);
1342 cpufreq_freq_attr_ro(energy_performance_available_preferences);
1343 static DEVICE_ATTR_RW(status);
1344 static DEVICE_ATTR_RO(prefcore);
1345
1346 static struct freq_attr *amd_pstate_attr[] = {
1347         &amd_pstate_max_freq,
1348         &amd_pstate_lowest_nonlinear_freq,
1349         &amd_pstate_highest_perf,
1350         &amd_pstate_prefcore_ranking,
1351         &amd_pstate_hw_prefcore,
1352         NULL,
1353 };
1354
1355 static struct freq_attr *amd_pstate_epp_attr[] = {
1356         &amd_pstate_max_freq,
1357         &amd_pstate_lowest_nonlinear_freq,
1358         &amd_pstate_highest_perf,
1359         &amd_pstate_prefcore_ranking,
1360         &amd_pstate_hw_prefcore,
1361         &energy_performance_preference,
1362         &energy_performance_available_preferences,
1363         NULL,
1364 };
1365
1366 static struct attribute *pstate_global_attributes[] = {
1367         &dev_attr_status.attr,
1368         &dev_attr_prefcore.attr,
1369         NULL
1370 };
1371
1372 static const struct attribute_group amd_pstate_global_attr_group = {
1373         .name = "amd_pstate",
1374         .attrs = pstate_global_attributes,
1375 };
1376
1377 static bool amd_pstate_acpi_pm_profile_server(void)
1378 {
1379         switch (acpi_gbl_FADT.preferred_profile) {
1380         case PM_ENTERPRISE_SERVER:
1381         case PM_SOHO_SERVER:
1382         case PM_PERFORMANCE_SERVER:
1383                 return true;
1384         }
1385         return false;
1386 }
1387
1388 static bool amd_pstate_acpi_pm_profile_undefined(void)
1389 {
1390         if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED)
1391                 return true;
1392         if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES)
1393                 return true;
1394         return false;
1395 }
1396
1397 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
1398 {
1399         int min_freq, max_freq, ret;
1400         struct amd_cpudata *cpudata;
1401         struct device *dev;
1402         u64 value;
1403
1404         /*
1405          * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
1406          * which is ideal for initialization process.
1407          */
1408         amd_perf_ctl_reset(policy->cpu);
1409         dev = get_cpu_device(policy->cpu);
1410         if (!dev)
1411                 return -ENODEV;
1412
1413         cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
1414         if (!cpudata)
1415                 return -ENOMEM;
1416
1417         cpudata->cpu = policy->cpu;
1418         cpudata->epp_policy = 0;
1419
1420         ret = amd_pstate_init_perf(cpudata);
1421         if (ret)
1422                 goto free_cpudata1;
1423
1424         amd_pstate_init_prefcore(cpudata);
1425
1426         ret = amd_pstate_init_freq(cpudata);
1427         if (ret)
1428                 goto free_cpudata1;
1429
1430         ret = amd_pstate_init_boost_support(cpudata);
1431         if (ret)
1432                 goto free_cpudata1;
1433
1434         min_freq = READ_ONCE(cpudata->min_freq);
1435         max_freq = READ_ONCE(cpudata->max_freq);
1436
1437         policy->cpuinfo.min_freq = min_freq;
1438         policy->cpuinfo.max_freq = max_freq;
1439         /* It will be updated by governor */
1440         policy->cur = policy->cpuinfo.min_freq;
1441
1442         policy->driver_data = cpudata;
1443
1444         cpudata->epp_cached = cpudata->epp_default = amd_pstate_get_epp(cpudata, 0);
1445
1446         policy->min = policy->cpuinfo.min_freq;
1447         policy->max = policy->cpuinfo.max_freq;
1448
1449         policy->boost_enabled = READ_ONCE(cpudata->boost_supported);
1450
1451         /*
1452          * Set the policy to provide a valid fallback value in case
1453          * the default cpufreq governor is neither powersave nor performance.
1454          */
1455         if (amd_pstate_acpi_pm_profile_server() ||
1456             amd_pstate_acpi_pm_profile_undefined())
1457                 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
1458         else
1459                 policy->policy = CPUFREQ_POLICY_POWERSAVE;
1460
1461         if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
1462                 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value);
1463                 if (ret)
1464                         return ret;
1465                 WRITE_ONCE(cpudata->cppc_req_cached, value);
1466
1467                 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value);
1468                 if (ret)
1469                         return ret;
1470                 WRITE_ONCE(cpudata->cppc_cap1_cached, value);
1471         }
1472
1473         return 0;
1474
1475 free_cpudata1:
1476         kfree(cpudata);
1477         return ret;
1478 }
1479
1480 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)
1481 {
1482         struct amd_cpudata *cpudata = policy->driver_data;
1483
1484         if (cpudata) {
1485                 kfree(cpudata);
1486                 policy->driver_data = NULL;
1487         }
1488
1489         pr_debug("CPU %d exiting\n", policy->cpu);
1490 }
1491
1492 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy)
1493 {
1494         struct amd_cpudata *cpudata = policy->driver_data;
1495         u32 max_perf, min_perf, min_limit_perf, max_limit_perf;
1496         u64 value;
1497         s16 epp;
1498
1499         max_perf = READ_ONCE(cpudata->highest_perf);
1500         min_perf = READ_ONCE(cpudata->lowest_perf);
1501         max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq);
1502         min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq);
1503
1504         if (min_limit_perf < min_perf)
1505                 min_limit_perf = min_perf;
1506
1507         if (max_limit_perf < min_limit_perf)
1508                 max_limit_perf = min_limit_perf;
1509
1510         WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf);
1511         WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf);
1512
1513         max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf,
1514                         cpudata->max_limit_perf);
1515         min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf,
1516                         cpudata->max_limit_perf);
1517         value = READ_ONCE(cpudata->cppc_req_cached);
1518
1519         if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1520                 min_perf = max_perf;
1521
1522         /* Initial min/max values for CPPC Performance Controls Register */
1523         value &= ~AMD_CPPC_MIN_PERF(~0L);
1524         value |= AMD_CPPC_MIN_PERF(min_perf);
1525
1526         value &= ~AMD_CPPC_MAX_PERF(~0L);
1527         value |= AMD_CPPC_MAX_PERF(max_perf);
1528
1529         /* CPPC EPP feature require to set zero to the desire perf bit */
1530         value &= ~AMD_CPPC_DES_PERF(~0L);
1531         value |= AMD_CPPC_DES_PERF(0);
1532
1533         cpudata->epp_policy = cpudata->policy;
1534
1535         /* Get BIOS pre-defined epp value */
1536         epp = amd_pstate_get_epp(cpudata, value);
1537         if (epp < 0) {
1538                 /**
1539                  * This return value can only be negative for shared_memory
1540                  * systems where EPP register read/write not supported.
1541                  */
1542                 return epp;
1543         }
1544
1545         if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1546                 epp = 0;
1547
1548         /* Set initial EPP value */
1549         if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
1550                 value &= ~GENMASK_ULL(31, 24);
1551                 value |= (u64)epp << 24;
1552         }
1553
1554         WRITE_ONCE(cpudata->cppc_req_cached, value);
1555         return amd_pstate_set_epp(cpudata, epp);
1556 }
1557
1558 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy)
1559 {
1560         struct amd_cpudata *cpudata = policy->driver_data;
1561         int ret;
1562
1563         if (!policy->cpuinfo.max_freq)
1564                 return -ENODEV;
1565
1566         pr_debug("set_policy: cpuinfo.max %u policy->max %u\n",
1567                                 policy->cpuinfo.max_freq, policy->max);
1568
1569         cpudata->policy = policy->policy;
1570
1571         ret = amd_pstate_epp_update_limit(policy);
1572         if (ret)
1573                 return ret;
1574
1575         /*
1576          * policy->cur is never updated with the amd_pstate_epp driver, but it
1577          * is used as a stale frequency value. So, keep it within limits.
1578          */
1579         policy->cur = policy->min;
1580
1581         return 0;
1582 }
1583
1584 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata)
1585 {
1586         struct cppc_perf_ctrls perf_ctrls;
1587         u64 value, max_perf;
1588         int ret;
1589
1590         ret = amd_pstate_enable(true);
1591         if (ret)
1592                 pr_err("failed to enable amd pstate during resume, return %d\n", ret);
1593
1594         value = READ_ONCE(cpudata->cppc_req_cached);
1595         max_perf = READ_ONCE(cpudata->highest_perf);
1596
1597         if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
1598                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1599         } else {
1600                 perf_ctrls.max_perf = max_perf;
1601                 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached);
1602                 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1603         }
1604 }
1605
1606 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy)
1607 {
1608         struct amd_cpudata *cpudata = policy->driver_data;
1609
1610         pr_debug("AMD CPU Core %d going online\n", cpudata->cpu);
1611
1612         if (cppc_state == AMD_PSTATE_ACTIVE) {
1613                 amd_pstate_epp_reenable(cpudata);
1614                 cpudata->suspended = false;
1615         }
1616
1617         return 0;
1618 }
1619
1620 static void amd_pstate_epp_offline(struct cpufreq_policy *policy)
1621 {
1622         struct amd_cpudata *cpudata = policy->driver_data;
1623         struct cppc_perf_ctrls perf_ctrls;
1624         int min_perf;
1625         u64 value;
1626
1627         min_perf = READ_ONCE(cpudata->lowest_perf);
1628         value = READ_ONCE(cpudata->cppc_req_cached);
1629
1630         mutex_lock(&amd_pstate_limits_lock);
1631         if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
1632                 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN;
1633
1634                 /* Set max perf same as min perf */
1635                 value &= ~AMD_CPPC_MAX_PERF(~0L);
1636                 value |= AMD_CPPC_MAX_PERF(min_perf);
1637                 value &= ~AMD_CPPC_MIN_PERF(~0L);
1638                 value |= AMD_CPPC_MIN_PERF(min_perf);
1639                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1640         } else {
1641                 perf_ctrls.desired_perf = 0;
1642                 perf_ctrls.max_perf = min_perf;
1643                 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE);
1644                 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1645         }
1646         mutex_unlock(&amd_pstate_limits_lock);
1647 }
1648
1649 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy)
1650 {
1651         struct amd_cpudata *cpudata = policy->driver_data;
1652
1653         pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu);
1654
1655         if (cpudata->suspended)
1656                 return 0;
1657
1658         if (cppc_state == AMD_PSTATE_ACTIVE)
1659                 amd_pstate_epp_offline(policy);
1660
1661         return 0;
1662 }
1663
1664 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy)
1665 {
1666         cpufreq_verify_within_cpu_limits(policy);
1667         pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min);
1668         return 0;
1669 }
1670
1671 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy)
1672 {
1673         struct amd_cpudata *cpudata = policy->driver_data;
1674         int ret;
1675
1676         /* avoid suspending when EPP is not enabled */
1677         if (cppc_state != AMD_PSTATE_ACTIVE)
1678                 return 0;
1679
1680         /* set this flag to avoid setting core offline*/
1681         cpudata->suspended = true;
1682
1683         /* disable CPPC in lowlevel firmware */
1684         ret = amd_pstate_enable(false);
1685         if (ret)
1686                 pr_err("failed to suspend, return %d\n", ret);
1687
1688         return 0;
1689 }
1690
1691 static int amd_pstate_epp_resume(struct cpufreq_policy *policy)
1692 {
1693         struct amd_cpudata *cpudata = policy->driver_data;
1694
1695         if (cpudata->suspended) {
1696                 mutex_lock(&amd_pstate_limits_lock);
1697
1698                 /* enable amd pstate from suspend state*/
1699                 amd_pstate_epp_reenable(cpudata);
1700
1701                 mutex_unlock(&amd_pstate_limits_lock);
1702
1703                 cpudata->suspended = false;
1704         }
1705
1706         return 0;
1707 }
1708
1709 static struct cpufreq_driver amd_pstate_driver = {
1710         .flags          = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
1711         .verify         = amd_pstate_verify,
1712         .target         = amd_pstate_target,
1713         .fast_switch    = amd_pstate_fast_switch,
1714         .init           = amd_pstate_cpu_init,
1715         .exit           = amd_pstate_cpu_exit,
1716         .suspend        = amd_pstate_cpu_suspend,
1717         .resume         = amd_pstate_cpu_resume,
1718         .set_boost      = amd_pstate_set_boost,
1719         .update_limits  = amd_pstate_update_limits,
1720         .name           = "amd-pstate",
1721         .attr           = amd_pstate_attr,
1722 };
1723
1724 static struct cpufreq_driver amd_pstate_epp_driver = {
1725         .flags          = CPUFREQ_CONST_LOOPS,
1726         .verify         = amd_pstate_epp_verify_policy,
1727         .setpolicy      = amd_pstate_epp_set_policy,
1728         .init           = amd_pstate_epp_cpu_init,
1729         .exit           = amd_pstate_epp_cpu_exit,
1730         .offline        = amd_pstate_epp_cpu_offline,
1731         .online         = amd_pstate_epp_cpu_online,
1732         .suspend        = amd_pstate_epp_suspend,
1733         .resume         = amd_pstate_epp_resume,
1734         .update_limits  = amd_pstate_update_limits,
1735         .set_boost      = amd_pstate_set_boost,
1736         .name           = "amd-pstate-epp",
1737         .attr           = amd_pstate_epp_attr,
1738 };
1739
1740 static int __init amd_pstate_set_driver(int mode_idx)
1741 {
1742         if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) {
1743                 cppc_state = mode_idx;
1744                 if (cppc_state == AMD_PSTATE_DISABLE)
1745                         pr_info("driver is explicitly disabled\n");
1746
1747                 if (cppc_state == AMD_PSTATE_ACTIVE)
1748                         current_pstate_driver = &amd_pstate_epp_driver;
1749
1750                 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED)
1751                         current_pstate_driver = &amd_pstate_driver;
1752
1753                 return 0;
1754         }
1755
1756         return -EINVAL;
1757 }
1758
1759 /**
1760  * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F.
1761  * show the debug message that helps to check if the CPU has CPPC support for loading issue.
1762  */
1763 static bool amd_cppc_supported(void)
1764 {
1765         struct cpuinfo_x86 *c = &cpu_data(0);
1766         bool warn = false;
1767
1768         if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) {
1769                 pr_debug_once("CPPC feature is not supported by the processor\n");
1770                 return false;
1771         }
1772
1773         /*
1774          * If the CPPC feature is disabled in the BIOS for processors
1775          * that support MSR-based CPPC, the AMD Pstate driver may not
1776          * function correctly.
1777          *
1778          * For such processors, check the CPPC flag and display a
1779          * warning message if the platform supports CPPC.
1780          *
1781          * Note: The code check below will not abort the driver
1782          * registration process because of the code is added for
1783          * debugging purposes. Besides, it may still be possible for
1784          * the driver to work using the shared-memory mechanism.
1785          */
1786         if (!cpu_feature_enabled(X86_FEATURE_CPPC)) {
1787                 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) {
1788                         switch (c->x86_model) {
1789                         case 0x60 ... 0x6F:
1790                         case 0x80 ... 0xAF:
1791                                 warn = true;
1792                                 break;
1793                         }
1794                 } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) ||
1795                            cpu_feature_enabled(X86_FEATURE_ZEN4)) {
1796                         switch (c->x86_model) {
1797                         case 0x10 ... 0x1F:
1798                         case 0x40 ... 0xAF:
1799                                 warn = true;
1800                                 break;
1801                         }
1802                 } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) {
1803                         warn = true;
1804                 }
1805         }
1806
1807         if (warn)
1808                 pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n"
1809                                         "Please enable it if your BIOS has the CPPC option.\n");
1810         return true;
1811 }
1812
1813 static int __init amd_pstate_init(void)
1814 {
1815         struct device *dev_root;
1816         int ret;
1817
1818         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
1819                 return -ENODEV;
1820
1821         /* show debug message only if CPPC is not supported */
1822         if (!amd_cppc_supported())
1823                 return -EOPNOTSUPP;
1824
1825         /* show warning message when BIOS broken or ACPI disabled */
1826         if (!acpi_cpc_valid()) {
1827                 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
1828                 return -ENODEV;
1829         }
1830
1831         /* don't keep reloading if cpufreq_driver exists */
1832         if (cpufreq_get_current_driver())
1833                 return -EEXIST;
1834
1835         quirks = NULL;
1836
1837         /* check if this machine need CPPC quirks */
1838         dmi_check_system(amd_pstate_quirks_table);
1839
1840         /*
1841         * determine the driver mode from the command line or kernel config.
1842         * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED.
1843         * command line options will override the kernel config settings.
1844         */
1845
1846         if (cppc_state == AMD_PSTATE_UNDEFINED) {
1847                 /* Disable on the following configs by default:
1848                  * 1. Undefined platforms
1849                  * 2. Server platforms
1850                  */
1851                 if (amd_pstate_acpi_pm_profile_undefined() ||
1852                     amd_pstate_acpi_pm_profile_server()) {
1853                         pr_info("driver load is disabled, boot with specific mode to enable this\n");
1854                         return -ENODEV;
1855                 }
1856                 /* get driver mode from kernel config option [1:4] */
1857                 cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE;
1858         }
1859
1860         switch (cppc_state) {
1861         case AMD_PSTATE_DISABLE:
1862                 pr_info("driver load is disabled, boot with specific mode to enable this\n");
1863                 return -ENODEV;
1864         case AMD_PSTATE_PASSIVE:
1865         case AMD_PSTATE_ACTIVE:
1866         case AMD_PSTATE_GUIDED:
1867                 ret = amd_pstate_set_driver(cppc_state);
1868                 if (ret)
1869                         return ret;
1870                 break;
1871         default:
1872                 return -EINVAL;
1873         }
1874
1875         /* capability check */
1876         if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
1877                 pr_debug("AMD CPPC MSR based functionality is supported\n");
1878                 if (cppc_state != AMD_PSTATE_ACTIVE)
1879                         current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
1880         } else {
1881                 pr_debug("AMD CPPC shared memory based functionality is supported\n");
1882                 static_call_update(amd_pstate_enable, cppc_enable);
1883                 static_call_update(amd_pstate_init_perf, cppc_init_perf);
1884                 static_call_update(amd_pstate_update_perf, cppc_update_perf);
1885         }
1886
1887         if (amd_pstate_prefcore) {
1888                 ret = amd_detect_prefcore(&amd_pstate_prefcore);
1889                 if (ret)
1890                         return ret;
1891         }
1892
1893         /* enable amd pstate feature */
1894         ret = amd_pstate_enable(true);
1895         if (ret) {
1896                 pr_err("failed to enable driver mode(%d)\n", cppc_state);
1897                 return ret;
1898         }
1899
1900         ret = cpufreq_register_driver(current_pstate_driver);
1901         if (ret) {
1902                 pr_err("failed to register with return %d\n", ret);
1903                 goto disable_driver;
1904         }
1905
1906         dev_root = bus_get_dev_root(&cpu_subsys);
1907         if (dev_root) {
1908                 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group);
1909                 put_device(dev_root);
1910                 if (ret) {
1911                         pr_err("sysfs attribute export failed with error %d.\n", ret);
1912                         goto global_attr_free;
1913                 }
1914         }
1915
1916         return ret;
1917
1918 global_attr_free:
1919         cpufreq_unregister_driver(current_pstate_driver);
1920 disable_driver:
1921         amd_pstate_enable(false);
1922         return ret;
1923 }
1924 device_initcall(amd_pstate_init);
1925
1926 static int __init amd_pstate_param(char *str)
1927 {
1928         size_t size;
1929         int mode_idx;
1930
1931         if (!str)
1932                 return -EINVAL;
1933
1934         size = strlen(str);
1935         mode_idx = get_mode_idx_from_str(str, size);
1936
1937         return amd_pstate_set_driver(mode_idx);
1938 }
1939
1940 static int __init amd_prefcore_param(char *str)
1941 {
1942         if (!strcmp(str, "disable"))
1943                 amd_pstate_prefcore = false;
1944
1945         return 0;
1946 }
1947
1948 early_param("amd_pstate", amd_pstate_param);
1949 early_param("amd_prefcore", amd_prefcore_param);
1950
1951 MODULE_AUTHOR("Huang Rui <[email protected]>");
1952 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");