drivers/cpufreq/amd-pstate.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * amd-pstate.c - AMD Processor P-state Frequency Driver
   4  *
   5  * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
   6  *
   7  * Author: Huang Rui <ray.huang@amd.com>
   8  *
   9  * AMD P-State introduces a new CPU performance scaling design for AMD
  10  * processors using the ACPI Collaborative Performance and Power Control (CPPC)
  11  * feature which works with the AMD SMU firmware providing a finer grained
  12  * frequency control range. It is to replace the legacy ACPI P-States control,
  13  * allows a flexible, low-latency interface for the Linux kernel to directly
  14  * communicate the performance hints to hardware.
  15  *
  16  * AMD P-State is supported on recent AMD Zen base CPU series include some of
  17  * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
  18  * P-State supported system. And there are two types of hardware implementations
  19  * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
  20  * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
  21  */
  22
  23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  24
  25 #include <linux/kernel.h>
  26 #include <linux/module.h>
  27 #include <linux/init.h>
  28 #include <linux/smp.h>
  29 #include <linux/sched.h>
  30 #include <linux/cpufreq.h>
  31 #include <linux/compiler.h>
  32 #include <linux/dmi.h>
  33 #include <linux/slab.h>
  34 #include <linux/acpi.h>
  35 #include <linux/io.h>
  36 #include <linux/delay.h>
  37 #include <linux/uaccess.h>
  38 #include <linux/static_call.h>
  39 #include <linux/amd-pstate.h>
  40
  41 #include <acpi/processor.h>
  42 #include <acpi/cppc_acpi.h>
  43
  44 #include <asm/msr.h>
  45 #include <asm/processor.h>
  46 #include <asm/cpufeature.h>
  47 #include <asm/cpu_device_id.h>
  48 #include "amd-pstate-trace.h"
  49
  50 #define AMD_PSTATE_TRANSITION_LATENCY   20000
  51 #define AMD_PSTATE_TRANSITION_DELAY     1000
  52
  53 /*
  54  * TODO: We need more time to fine tune processors with shared memory solution
  55  * with community together.
  56  *
  57  * There are some performance drops on the CPU benchmarks which reports from
  58  * Suse. We are co-working with them to fine tune the shared memory solution. So
  59  * we disable it by default to go acpi-cpufreq on these processors and add a
  60  * module parameter to be able to enable it manually for debugging.
  61  */
  62 static struct cpufreq_driver *current_pstate_driver;
  63 static struct cpufreq_driver amd_pstate_driver;
  64 static struct cpufreq_driver amd_pstate_epp_driver;
  65 static int cppc_state = AMD_PSTATE_DISABLE;
  66
  67 /*
  68  * AMD Energy Preference Performance (EPP)
  69  * The EPP is used in the CCLK DPM controller to drive
  70  * the frequency that a core is going to operate during
  71  * short periods of activity. EPP values will be utilized for
  72  * different OS profiles (balanced, performance, power savings)
  73  * display strings corresponding to EPP index in the
  74  * energy_perf_strings[]
  75  *      index           String
  76  *-------------------------------------
  77  *      0               default
  78  *      1               performance
  79  *      2               balance_performance
  80  *      3               balance_power
  81  *      4               power
  82  */
  83 enum energy_perf_value_index {
  84         EPP_INDEX_DEFAULT = 0,
  85         EPP_INDEX_PERFORMANCE,
  86         EPP_INDEX_BALANCE_PERFORMANCE,
  87         EPP_INDEX_BALANCE_POWERSAVE,
  88         EPP_INDEX_POWERSAVE,
  89 };
  90
  91 static const char * const energy_perf_strings[] = {
  92         [EPP_INDEX_DEFAULT] = "default",
  93         [EPP_INDEX_PERFORMANCE] = "performance",
  94         [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
  95         [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
  96         [EPP_INDEX_POWERSAVE] = "power",
  97         NULL
  98 };
  99
 100 static unsigned int epp_values[] = {
 101         [EPP_INDEX_DEFAULT] = 0,
 102         [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE,
 103         [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE,
 104         [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE,
 105         [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE,
 106  };
 107
 108 typedef int (*cppc_mode_transition_fn)(int);
 109
 110 static inline int get_mode_idx_from_str(const char *str, size_t size)
 111 {
 112         int i;
 113
 114         for (i=0; i < AMD_PSTATE_MAX; i++) {
 115                 if (!strncmp(str, amd_pstate_mode_string[i], size))
 116                         return i;
 117         }
 118         return -EINVAL;
 119 }
 120
 121 static DEFINE_MUTEX(amd_pstate_limits_lock);
 122 static DEFINE_MUTEX(amd_pstate_driver_lock);
 123
 124 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached)
 125 {
 126         u64 epp;
 127         int ret;
 128
 129         if (boot_cpu_has(X86_FEATURE_CPPC)) {
 130                 if (!cppc_req_cached) {
 131                         epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
 132                                         &cppc_req_cached);
 133                         if (epp)
 134                                 return epp;
 135                 }
 136                 epp = (cppc_req_cached >> 24) & 0xFF;
 137         } else {
 138                 ret = cppc_get_epp_perf(cpudata->cpu, &epp);
 139                 if (ret < 0) {
 140                         pr_debug("Could not retrieve energy perf value (%d)\n", ret);
 141                         return -EIO;
 142                 }
 143         }
 144
 145         return (s16)(epp & 0xff);
 146 }
 147
 148 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata)
 149 {
 150         s16 epp;
 151         int index = -EINVAL;
 152
 153         epp = amd_pstate_get_epp(cpudata, 0);
 154         if (epp < 0)
 155                 return epp;
 156
 157         switch (epp) {
 158         case AMD_CPPC_EPP_PERFORMANCE:
 159                 index = EPP_INDEX_PERFORMANCE;
 160                 break;
 161         case AMD_CPPC_EPP_BALANCE_PERFORMANCE:
 162                 index = EPP_INDEX_BALANCE_PERFORMANCE;
 163                 break;
 164         case AMD_CPPC_EPP_BALANCE_POWERSAVE:
 165                 index = EPP_INDEX_BALANCE_POWERSAVE;
 166                 break;
 167         case AMD_CPPC_EPP_POWERSAVE:
 168                 index = EPP_INDEX_POWERSAVE;
 169                 break;
 170         default:
 171                 break;
 172         }
 173
 174         return index;
 175 }
 176
 177 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp)
 178 {
 179         int ret;
 180         struct cppc_perf_ctrls perf_ctrls;
 181
 182         if (boot_cpu_has(X86_FEATURE_CPPC)) {
 183                 u64 value = READ_ONCE(cpudata->cppc_req_cached);
 184
 185                 value &= ~GENMASK_ULL(31, 24);
 186                 value |= (u64)epp << 24;
 187                 WRITE_ONCE(cpudata->cppc_req_cached, value);
 188
 189                 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
 190                 if (!ret)
 191                         cpudata->epp_cached = epp;
 192         } else {
 193                 perf_ctrls.energy_perf = epp;
 194                 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1);
 195                 if (ret) {
 196                         pr_debug("failed to set energy perf value (%d)\n", ret);
 197                         return ret;
 198                 }
 199                 cpudata->epp_cached = epp;
 200         }
 201
 202         return ret;
 203 }
 204
 205 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata,
 206                 int pref_index)
 207 {
 208         int epp = -EINVAL;
 209         int ret;
 210
 211         if (!pref_index) {
 212                 pr_debug("EPP pref_index is invalid\n");
 213                 return -EINVAL;
 214         }
 215
 216         if (epp == -EINVAL)
 217                 epp = epp_values[pref_index];
 218
 219         if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
 220                 pr_debug("EPP cannot be set under performance policy\n");
 221                 return -EBUSY;
 222         }
 223
 224         ret = amd_pstate_set_epp(cpudata, epp);
 225
 226         return ret;
 227 }
 228
 229 static inline int pstate_enable(bool enable)
 230 {
 231         return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable);
 232 }
 233
 234 static int cppc_enable(bool enable)
 235 {
 236         int cpu, ret = 0;
 237         struct cppc_perf_ctrls perf_ctrls;
 238
 239         for_each_present_cpu(cpu) {
 240                 ret = cppc_set_enable(cpu, enable);
 241                 if (ret)
 242                         return ret;
 243
 244                 /* Enable autonomous mode for EPP */
 245                 if (cppc_state == AMD_PSTATE_ACTIVE) {
 246                         /* Set desired perf as zero to allow EPP firmware control */
 247                         perf_ctrls.desired_perf = 0;
 248                         ret = cppc_set_perf(cpu, &perf_ctrls);
 249                         if (ret)
 250                                 return ret;
 251                 }
 252         }
 253
 254         return ret;
 255 }
 256
 257 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable);
 258
 259 static inline int amd_pstate_enable(bool enable)
 260 {
 261         return static_call(amd_pstate_enable)(enable);
 262 }
 263
 264 static int pstate_init_perf(struct amd_cpudata *cpudata)
 265 {
 266         u64 cap1;
 267         u32 highest_perf;
 268
 269         int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
 270                                      &cap1);
 271         if (ret)
 272                 return ret;
 273
 274         /*
 275          * TODO: Introduce AMD specific power feature.
 276          *
 277          * CPPC entry doesn't indicate the highest performance in some ASICs.
 278          */
 279         highest_perf = amd_get_highest_perf();
 280         if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1))
 281                 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
 282
 283         WRITE_ONCE(cpudata->highest_perf, highest_perf);
 284
 285         WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
 286         WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
 287         WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));
 288
 289         return 0;
 290 }
 291
 292 static int cppc_init_perf(struct amd_cpudata *cpudata)
 293 {
 294         struct cppc_perf_caps cppc_perf;
 295         u32 highest_perf;
 296
 297         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 298         if (ret)
 299                 return ret;
 300
 301         highest_perf = amd_get_highest_perf();
 302         if (highest_perf > cppc_perf.highest_perf)
 303                 highest_perf = cppc_perf.highest_perf;
 304
 305         WRITE_ONCE(cpudata->highest_perf, highest_perf);
 306
 307         WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
 308         WRITE_ONCE(cpudata->lowest_nonlinear_perf,
 309                    cppc_perf.lowest_nonlinear_perf);
 310         WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
 311
 312         if (cppc_state == AMD_PSTATE_ACTIVE)
 313                 return 0;
 314
 315         ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf);
 316         if (ret) {
 317                 pr_warn("failed to get auto_sel, ret: %d\n", ret);
 318                 return 0;
 319         }
 320
 321         ret = cppc_set_auto_sel(cpudata->cpu,
 322                         (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
 323
 324         if (ret)
 325                 pr_warn("failed to set auto_sel, ret: %d\n", ret);
 326
 327         return ret;
 328 }
 329
 330 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
 331
 332 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
 333 {
 334         return static_call(amd_pstate_init_perf)(cpudata);
 335 }
 336
 337 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,
 338                                u32 des_perf, u32 max_perf, bool fast_switch)
 339 {
 340         if (fast_switch)
 341                 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));
 342         else
 343                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
 344                               READ_ONCE(cpudata->cppc_req_cached));
 345 }
 346
 347 static void cppc_update_perf(struct amd_cpudata *cpudata,
 348                              u32 min_perf, u32 des_perf,
 349                              u32 max_perf, bool fast_switch)
 350 {
 351         struct cppc_perf_ctrls perf_ctrls;
 352
 353         perf_ctrls.max_perf = max_perf;
 354         perf_ctrls.min_perf = min_perf;
 355         perf_ctrls.desired_perf = des_perf;
 356
 357         cppc_set_perf(cpudata->cpu, &perf_ctrls);
 358 }
 359
 360 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf);
 361
 362 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,
 363                                           u32 min_perf, u32 des_perf,
 364                                           u32 max_perf, bool fast_switch)
 365 {
 366         static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,
 367                                             max_perf, fast_switch);
 368 }
 369
 370 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
 371 {
 372         u64 aperf, mperf, tsc;
 373         unsigned long flags;
 374
 375         local_irq_save(flags);
 376         rdmsrl(MSR_IA32_APERF, aperf);
 377         rdmsrl(MSR_IA32_MPERF, mperf);
 378         tsc = rdtsc();
 379
 380         if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
 381                 local_irq_restore(flags);
 382                 return false;
 383         }
 384
 385         local_irq_restore(flags);
 386
 387         cpudata->cur.aperf = aperf;
 388         cpudata->cur.mperf = mperf;
 389         cpudata->cur.tsc =  tsc;
 390         cpudata->cur.aperf -= cpudata->prev.aperf;
 391         cpudata->cur.mperf -= cpudata->prev.mperf;
 392         cpudata->cur.tsc -= cpudata->prev.tsc;
 393
 394         cpudata->prev.aperf = aperf;
 395         cpudata->prev.mperf = mperf;
 396         cpudata->prev.tsc = tsc;
 397
 398         cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
 399
 400         return true;
 401 }
 402
 403 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
 404                               u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags)
 405 {
 406         u64 prev = READ_ONCE(cpudata->cppc_req_cached);
 407         u64 value = prev;
 408
 409         des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
 410
 411         if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) {
 412                 min_perf = des_perf;
 413                 des_perf = 0;
 414         }
 415
 416         value &= ~AMD_CPPC_MIN_PERF(~0L);
 417         value |= AMD_CPPC_MIN_PERF(min_perf);
 418
 419         value &= ~AMD_CPPC_DES_PERF(~0L);
 420         value |= AMD_CPPC_DES_PERF(des_perf);
 421
 422         value &= ~AMD_CPPC_MAX_PERF(~0L);
 423         value |= AMD_CPPC_MAX_PERF(max_perf);
 424
 425         if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
 426                 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
 427                         cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
 428                                 cpudata->cpu, (value != prev), fast_switch);
 429         }
 430
 431         if (value == prev)
 432                 return;
 433
 434         WRITE_ONCE(cpudata->cppc_req_cached, value);
 435
 436         amd_pstate_update_perf(cpudata, min_perf, des_perf,
 437                                max_perf, fast_switch);
 438 }
 439
 440 static int amd_pstate_verify(struct cpufreq_policy_data *policy)
 441 {
 442         cpufreq_verify_within_cpu_limits(policy);
 443
 444         return 0;
 445 }
 446
 447 static int amd_pstate_update_freq(struct cpufreq_policy *policy,
 448                                   unsigned int target_freq, bool fast_switch)
 449 {
 450         struct cpufreq_freqs freqs;
 451         struct amd_cpudata *cpudata = policy->driver_data;
 452         unsigned long max_perf, min_perf, des_perf, cap_perf;
 453
 454         if (!cpudata->max_freq)
 455                 return -ENODEV;
 456
 457         cap_perf = READ_ONCE(cpudata->highest_perf);
 458         min_perf = READ_ONCE(cpudata->lowest_perf);
 459         max_perf = cap_perf;
 460
 461         freqs.old = policy->cur;
 462         freqs.new = target_freq;
 463
 464         des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf,
 465                                      cpudata->max_freq);
 466
 467         WARN_ON(fast_switch && !policy->fast_switch_enabled);
 468         /*
 469          * If fast_switch is desired, then there aren't any registered
 470          * transition notifiers. See comment for
 471          * cpufreq_enable_fast_switch().
 472          */
 473         if (!fast_switch)
 474                 cpufreq_freq_transition_begin(policy, &freqs);
 475
 476         amd_pstate_update(cpudata, min_perf, des_perf,
 477                         max_perf, fast_switch, policy->governor->flags);
 478
 479         if (!fast_switch)
 480                 cpufreq_freq_transition_end(policy, &freqs, false);
 481
 482         return 0;
 483 }
 484
 485 static int amd_pstate_target(struct cpufreq_policy *policy,
 486                              unsigned int target_freq,
 487                              unsigned int relation)
 488 {
 489         return amd_pstate_update_freq(policy, target_freq, false);
 490 }
 491
 492 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy,
 493                                   unsigned int target_freq)
 494 {
 495         return amd_pstate_update_freq(policy, target_freq, true);
 496 }
 497
 498 static void amd_pstate_adjust_perf(unsigned int cpu,
 499                                    unsigned long _min_perf,
 500                                    unsigned long target_perf,
 501                                    unsigned long capacity)
 502 {
 503         unsigned long max_perf, min_perf, des_perf,
 504                       cap_perf, lowest_nonlinear_perf, max_freq;
 505         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
 506         struct amd_cpudata *cpudata = policy->driver_data;
 507         unsigned int target_freq;
 508
 509         cap_perf = READ_ONCE(cpudata->highest_perf);
 510         lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
 511         max_freq = READ_ONCE(cpudata->max_freq);
 512
 513         des_perf = cap_perf;
 514         if (target_perf < capacity)
 515                 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
 516
 517         min_perf = READ_ONCE(cpudata->highest_perf);
 518         if (_min_perf < capacity)
 519                 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
 520
 521         if (min_perf < lowest_nonlinear_perf)
 522                 min_perf = lowest_nonlinear_perf;
 523
 524         max_perf = cap_perf;
 525         if (max_perf < min_perf)
 526                 max_perf = min_perf;
 527
 528         des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
 529         target_freq = div_u64(des_perf * max_freq, max_perf);
 530         policy->cur = target_freq;
 531
 532         amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true,
 533                         policy->governor->flags);
 534         cpufreq_cpu_put(policy);
 535 }
 536
 537 static int amd_get_min_freq(struct amd_cpudata *cpudata)
 538 {
 539         struct cppc_perf_caps cppc_perf;
 540
 541         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 542         if (ret)
 543                 return ret;
 544
 545         /* Switch to khz */
 546         return cppc_perf.lowest_freq * 1000;
 547 }
 548
 549 static int amd_get_max_freq(struct amd_cpudata *cpudata)
 550 {
 551         struct cppc_perf_caps cppc_perf;
 552         u32 max_perf, max_freq, nominal_freq, nominal_perf;
 553         u64 boost_ratio;
 554
 555         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 556         if (ret)
 557                 return ret;
 558
 559         nominal_freq = cppc_perf.nominal_freq;
 560         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 561         max_perf = READ_ONCE(cpudata->highest_perf);
 562
 563         boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT,
 564                               nominal_perf);
 565
 566         max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT;
 567
 568         /* Switch to khz */
 569         return max_freq * 1000;
 570 }
 571
 572 static int amd_get_nominal_freq(struct amd_cpudata *cpudata)
 573 {
 574         struct cppc_perf_caps cppc_perf;
 575
 576         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 577         if (ret)
 578                 return ret;
 579
 580         /* Switch to khz */
 581         return cppc_perf.nominal_freq * 1000;
 582 }
 583
 584 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata)
 585 {
 586         struct cppc_perf_caps cppc_perf;
 587         u32 lowest_nonlinear_freq, lowest_nonlinear_perf,
 588             nominal_freq, nominal_perf;
 589         u64 lowest_nonlinear_ratio;
 590
 591         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 592         if (ret)
 593                 return ret;
 594
 595         nominal_freq = cppc_perf.nominal_freq;
 596         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 597
 598         lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
 599
 600         lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT,
 601                                          nominal_perf);
 602
 603         lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT;
 604
 605         /* Switch to khz */
 606         return lowest_nonlinear_freq * 1000;
 607 }
 608
 609 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
 610 {
 611         struct amd_cpudata *cpudata = policy->driver_data;
 612         int ret;
 613
 614         if (!cpudata->boost_supported) {
 615                 pr_err("Boost mode is not supported by this processor or SBIOS\n");
 616                 return -EINVAL;
 617         }
 618
 619         if (state)
 620                 policy->cpuinfo.max_freq = cpudata->max_freq;
 621         else
 622                 policy->cpuinfo.max_freq = cpudata->nominal_freq;
 623
 624         policy->max = policy->cpuinfo.max_freq;
 625
 626         ret = freq_qos_update_request(&cpudata->req[1],
 627                                       policy->cpuinfo.max_freq);
 628         if (ret < 0)
 629                 return ret;
 630
 631         return 0;
 632 }
 633
 634 static void amd_pstate_boost_init(struct amd_cpudata *cpudata)
 635 {
 636         u32 highest_perf, nominal_perf;
 637
 638         highest_perf = READ_ONCE(cpudata->highest_perf);
 639         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 640
 641         if (highest_perf <= nominal_perf)
 642                 return;
 643
 644         cpudata->boost_supported = true;
 645         current_pstate_driver->boost_enabled = true;
 646 }
 647
 648 static void amd_perf_ctl_reset(unsigned int cpu)
 649 {
 650         wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0);
 651 }
 652
 653 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
 654 {
 655         int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
 656         struct device *dev;
 657         struct amd_cpudata *cpudata;
 658
 659         /*
 660          * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
 661          * which is ideal for initialization process.
 662          */
 663         amd_perf_ctl_reset(policy->cpu);
 664         dev = get_cpu_device(policy->cpu);
 665         if (!dev)
 666                 return -ENODEV;
 667
 668         cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
 669         if (!cpudata)
 670                 return -ENOMEM;
 671
 672         cpudata->cpu = policy->cpu;
 673
 674         ret = amd_pstate_init_perf(cpudata);
 675         if (ret)
 676                 goto free_cpudata1;
 677
 678         min_freq = amd_get_min_freq(cpudata);
 679         max_freq = amd_get_max_freq(cpudata);
 680         nominal_freq = amd_get_nominal_freq(cpudata);
 681         lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
 682
 683         if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
 684                 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
 685                         min_freq, max_freq);
 686                 ret = -EINVAL;
 687                 goto free_cpudata1;
 688         }
 689
 690         policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY;
 691         policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY;
 692
 693         policy->min = min_freq;
 694         policy->max = max_freq;
 695
 696         policy->cpuinfo.min_freq = min_freq;
 697         policy->cpuinfo.max_freq = max_freq;
 698
 699         /* It will be updated by governor */
 700         policy->cur = policy->cpuinfo.min_freq;
 701
 702         if (boot_cpu_has(X86_FEATURE_CPPC))
 703                 policy->fast_switch_possible = true;
 704
 705         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
 706                                    FREQ_QOS_MIN, policy->cpuinfo.min_freq);
 707         if (ret < 0) {
 708                 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
 709                 goto free_cpudata1;
 710         }
 711
 712         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
 713                                    FREQ_QOS_MAX, policy->cpuinfo.max_freq);
 714         if (ret < 0) {
 715                 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
 716                 goto free_cpudata2;
 717         }
 718
 719         /* Initial processor data capability frequencies */
 720         cpudata->max_freq = max_freq;
 721         cpudata->min_freq = min_freq;
 722         cpudata->nominal_freq = nominal_freq;
 723         cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
 724
 725         policy->driver_data = cpudata;
 726
 727         amd_pstate_boost_init(cpudata);
 728         if (!current_pstate_driver->adjust_perf)
 729                 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
 730
 731         return 0;
 732
 733 free_cpudata2:
 734         freq_qos_remove_request(&cpudata->req[0]);
 735 free_cpudata1:
 736         kfree(cpudata);
 737         return ret;
 738 }
 739
 740 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy)
 741 {
 742         struct amd_cpudata *cpudata = policy->driver_data;
 743
 744         freq_qos_remove_request(&cpudata->req[1]);
 745         freq_qos_remove_request(&cpudata->req[0]);
 746         policy->fast_switch_possible = false;
 747         kfree(cpudata);
 748
 749         return 0;
 750 }
 751
 752 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)
 753 {
 754         int ret;
 755
 756         ret = amd_pstate_enable(true);
 757         if (ret)
 758                 pr_err("failed to enable amd-pstate during resume, return %d\n", ret);
 759
 760         return ret;
 761 }
 762
 763 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)
 764 {
 765         int ret;
 766
 767         ret = amd_pstate_enable(false);
 768         if (ret)
 769                 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret);
 770
 771         return ret;
 772 }
 773
 774 /* Sysfs attributes */
 775
 776 /*
 777  * This frequency is to indicate the maximum hardware frequency.
 778  * If boost is not active but supported, the frequency will be larger than the
 779  * one in cpuinfo.
 780  */
 781 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
 782                                         char *buf)
 783 {
 784         int max_freq;
 785         struct amd_cpudata *cpudata = policy->driver_data;
 786
 787         max_freq = amd_get_max_freq(cpudata);
 788         if (max_freq < 0)
 789                 return max_freq;
 790
 791         return sysfs_emit(buf, "%u\n", max_freq);
 792 }
 793
 794 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
 795                                                      char *buf)
 796 {
 797         int freq;
 798         struct amd_cpudata *cpudata = policy->driver_data;
 799
 800         freq = amd_get_lowest_nonlinear_freq(cpudata);
 801         if (freq < 0)
 802                 return freq;
 803
 804         return sysfs_emit(buf, "%u\n", freq);
 805 }
 806
 807 /*
 808  * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
 809  * need to expose it to sysfs.
 810  */
 811 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
 812                                             char *buf)
 813 {
 814         u32 perf;
 815         struct amd_cpudata *cpudata = policy->driver_data;
 816
 817         perf = READ_ONCE(cpudata->highest_perf);
 818
 819         return sysfs_emit(buf, "%u\n", perf);
 820 }
 821
 822 static ssize_t show_energy_performance_available_preferences(
 823                                 struct cpufreq_policy *policy, char *buf)
 824 {
 825         int i = 0;
 826         int offset = 0;
 827
 828         while (energy_perf_strings[i] != NULL)
 829                 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]);
 830
 831         sysfs_emit_at(buf, offset, "\n");
 832
 833         return offset;
 834 }
 835
 836 static ssize_t store_energy_performance_preference(
 837                 struct cpufreq_policy *policy, const char *buf, size_t count)
 838 {
 839         struct amd_cpudata *cpudata = policy->driver_data;
 840         char str_preference[21];
 841         ssize_t ret;
 842
 843         ret = sscanf(buf, "%20s", str_preference);
 844         if (ret != 1)
 845                 return -EINVAL;
 846
 847         ret = match_string(energy_perf_strings, -1, str_preference);
 848         if (ret < 0)
 849                 return -EINVAL;
 850
 851         mutex_lock(&amd_pstate_limits_lock);
 852         ret = amd_pstate_set_energy_pref_index(cpudata, ret);
 853         mutex_unlock(&amd_pstate_limits_lock);
 854
 855         return ret ?: count;
 856 }
 857
 858 static ssize_t show_energy_performance_preference(
 859                                 struct cpufreq_policy *policy, char *buf)
 860 {
 861         struct amd_cpudata *cpudata = policy->driver_data;
 862         int preference;
 863
 864         preference = amd_pstate_get_energy_pref_index(cpudata);
 865         if (preference < 0)
 866                 return preference;
 867
 868         return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]);
 869 }
 870
 871 static void amd_pstate_driver_cleanup(void)
 872 {
 873         amd_pstate_enable(false);
 874         cppc_state = AMD_PSTATE_DISABLE;
 875         current_pstate_driver = NULL;
 876 }
 877
 878 static int amd_pstate_register_driver(int mode)
 879 {
 880         int ret;
 881
 882         if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED)
 883                 current_pstate_driver = &amd_pstate_driver;
 884         else if (mode == AMD_PSTATE_ACTIVE)
 885                 current_pstate_driver = &amd_pstate_epp_driver;
 886         else
 887                 return -EINVAL;
 888
 889         cppc_state = mode;
 890         ret = cpufreq_register_driver(current_pstate_driver);
 891         if (ret) {
 892                 amd_pstate_driver_cleanup();
 893                 return ret;
 894         }
 895         return 0;
 896 }
 897
 898 static int amd_pstate_unregister_driver(int dummy)
 899 {
 900         cpufreq_unregister_driver(current_pstate_driver);
 901         amd_pstate_driver_cleanup();
 902         return 0;
 903 }
 904
 905 static int amd_pstate_change_mode_without_dvr_change(int mode)
 906 {
 907         int cpu = 0;
 908
 909         cppc_state = mode;
 910
 911         if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE)
 912                 return 0;
 913
 914         for_each_present_cpu(cpu) {
 915                 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
 916         }
 917
 918         return 0;
 919 }
 920
 921 static int amd_pstate_change_driver_mode(int mode)
 922 {
 923         int ret;
 924
 925         ret = amd_pstate_unregister_driver(0);
 926         if (ret)
 927                 return ret;
 928
 929         ret = amd_pstate_register_driver(mode);
 930         if (ret)
 931                 return ret;
 932
 933         return 0;
 934 }
 935
 936 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = {
 937         [AMD_PSTATE_DISABLE]         = {
 938                 [AMD_PSTATE_DISABLE]     = NULL,
 939                 [AMD_PSTATE_PASSIVE]     = amd_pstate_register_driver,
 940                 [AMD_PSTATE_ACTIVE]      = amd_pstate_register_driver,
 941                 [AMD_PSTATE_GUIDED]      = amd_pstate_register_driver,
 942         },
 943         [AMD_PSTATE_PASSIVE]         = {
 944                 [AMD_PSTATE_DISABLE]     = amd_pstate_unregister_driver,
 945                 [AMD_PSTATE_PASSIVE]     = NULL,
 946                 [AMD_PSTATE_ACTIVE]      = amd_pstate_change_driver_mode,
 947                 [AMD_PSTATE_GUIDED]      = amd_pstate_change_mode_without_dvr_change,
 948         },
 949         [AMD_PSTATE_ACTIVE]          = {
 950                 [AMD_PSTATE_DISABLE]     = amd_pstate_unregister_driver,
 951                 [AMD_PSTATE_PASSIVE]     = amd_pstate_change_driver_mode,
 952                 [AMD_PSTATE_ACTIVE]      = NULL,
 953                 [AMD_PSTATE_GUIDED]      = amd_pstate_change_driver_mode,
 954         },
 955         [AMD_PSTATE_GUIDED]          = {
 956                 [AMD_PSTATE_DISABLE]     = amd_pstate_unregister_driver,
 957                 [AMD_PSTATE_PASSIVE]     = amd_pstate_change_mode_without_dvr_change,
 958                 [AMD_PSTATE_ACTIVE]      = amd_pstate_change_driver_mode,
 959                 [AMD_PSTATE_GUIDED]      = NULL,
 960         },
 961 };
 962
 963 static ssize_t amd_pstate_show_status(char *buf)
 964 {
 965         if (!current_pstate_driver)
 966                 return sysfs_emit(buf, "disable\n");
 967
 968         return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]);
 969 }
 970
 971 static int amd_pstate_update_status(const char *buf, size_t size)
 972 {
 973         int mode_idx;
 974
 975         if (size > strlen("passive") || size < strlen("active"))
 976                 return -EINVAL;
 977
 978         mode_idx = get_mode_idx_from_str(buf, size);
 979
 980         if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX)
 981                 return -EINVAL;
 982
 983         if (mode_state_machine[cppc_state][mode_idx])
 984                 return mode_state_machine[cppc_state][mode_idx](mode_idx);
 985
 986         return 0;
 987 }
 988
 989 static ssize_t show_status(struct kobject *kobj,
 990                            struct kobj_attribute *attr, char *buf)
 991 {
 992         ssize_t ret;
 993
 994         mutex_lock(&amd_pstate_driver_lock);
 995         ret = amd_pstate_show_status(buf);
 996         mutex_unlock(&amd_pstate_driver_lock);
 997
 998         return ret;
 999 }
1000
1001 static ssize_t store_status(struct kobject *a, struct kobj_attribute *b,
1002                             const char *buf, size_t count)
1003 {
1004         char *p = memchr(buf, '\n', count);
1005         int ret;
1006
1007         mutex_lock(&amd_pstate_driver_lock);
1008         ret = amd_pstate_update_status(buf, p ? p - buf : count);
1009         mutex_unlock(&amd_pstate_driver_lock);
1010
1011         return ret < 0 ? ret : count;
1012 }
1013
1014 cpufreq_freq_attr_ro(amd_pstate_max_freq);
1015 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
1016
1017 cpufreq_freq_attr_ro(amd_pstate_highest_perf);
1018 cpufreq_freq_attr_rw(energy_performance_preference);
1019 cpufreq_freq_attr_ro(energy_performance_available_preferences);
1020 define_one_global_rw(status);
1021
1022 static struct freq_attr *amd_pstate_attr[] = {
1023         &amd_pstate_max_freq,
1024         &amd_pstate_lowest_nonlinear_freq,
1025         &amd_pstate_highest_perf,
1026         NULL,
1027 };
1028
1029 static struct freq_attr *amd_pstate_epp_attr[] = {
1030         &amd_pstate_max_freq,
1031         &amd_pstate_lowest_nonlinear_freq,
1032         &amd_pstate_highest_perf,
1033         &energy_performance_preference,
1034         &energy_performance_available_preferences,
1035         NULL,
1036 };
1037
1038 static struct attribute *pstate_global_attributes[] = {
1039         &status.attr,
1040         NULL
1041 };
1042
1043 static const struct attribute_group amd_pstate_global_attr_group = {
1044         .name = "amd_pstate",
1045         .attrs = pstate_global_attributes,
1046 };
1047
1048 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
1049 {
1050         int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
1051         struct amd_cpudata *cpudata;
1052         struct device *dev;
1053         u64 value;
1054
1055         /*
1056          * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
1057          * which is ideal for initialization process.
1058          */
1059         amd_perf_ctl_reset(policy->cpu);
1060         dev = get_cpu_device(policy->cpu);
1061         if (!dev)
1062                 return -ENODEV;
1063
1064         cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
1065         if (!cpudata)
1066                 return -ENOMEM;
1067
1068         cpudata->cpu = policy->cpu;
1069         cpudata->epp_policy = 0;
1070
1071         ret = amd_pstate_init_perf(cpudata);
1072         if (ret)
1073                 goto free_cpudata1;
1074
1075         min_freq = amd_get_min_freq(cpudata);
1076         max_freq = amd_get_max_freq(cpudata);
1077         nominal_freq = amd_get_nominal_freq(cpudata);
1078         lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
1079         if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
1080                 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
1081                                 min_freq, max_freq);
1082                 ret = -EINVAL;
1083                 goto free_cpudata1;
1084         }
1085
1086         policy->cpuinfo.min_freq = min_freq;
1087         policy->cpuinfo.max_freq = max_freq;
1088         /* It will be updated by governor */
1089         policy->cur = policy->cpuinfo.min_freq;
1090
1091         /* Initial processor data capability frequencies */
1092         cpudata->max_freq = max_freq;
1093         cpudata->min_freq = min_freq;
1094         cpudata->nominal_freq = nominal_freq;
1095         cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
1096
1097         policy->driver_data = cpudata;
1098
1099         cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0);
1100
1101         policy->min = policy->cpuinfo.min_freq;
1102         policy->max = policy->cpuinfo.max_freq;
1103
1104         /*
1105          * Set the policy to powersave to provide a valid fallback value in case
1106          * the default cpufreq governor is neither powersave nor performance.
1107          */
1108         policy->policy = CPUFREQ_POLICY_POWERSAVE;
1109
1110         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1111                 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value);
1112                 if (ret)
1113                         return ret;
1114                 WRITE_ONCE(cpudata->cppc_req_cached, value);
1115
1116                 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value);
1117                 if (ret)
1118                         return ret;
1119                 WRITE_ONCE(cpudata->cppc_cap1_cached, value);
1120         }
1121         amd_pstate_boost_init(cpudata);
1122
1123         return 0;
1124
1125 free_cpudata1:
1126         kfree(cpudata);
1127         return ret;
1128 }
1129
1130 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)
1131 {
1132         pr_debug("CPU %d exiting\n", policy->cpu);
1133         return 0;
1134 }
1135
1136 static void amd_pstate_epp_init(unsigned int cpu)
1137 {
1138         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1139         struct amd_cpudata *cpudata = policy->driver_data;
1140         u32 max_perf, min_perf;
1141         u64 value;
1142         s16 epp;
1143
1144         max_perf = READ_ONCE(cpudata->highest_perf);
1145         min_perf = READ_ONCE(cpudata->lowest_perf);
1146
1147         value = READ_ONCE(cpudata->cppc_req_cached);
1148
1149         if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1150                 min_perf = max_perf;
1151
1152         /* Initial min/max values for CPPC Performance Controls Register */
1153         value &= ~AMD_CPPC_MIN_PERF(~0L);
1154         value |= AMD_CPPC_MIN_PERF(min_perf);
1155
1156         value &= ~AMD_CPPC_MAX_PERF(~0L);
1157         value |= AMD_CPPC_MAX_PERF(max_perf);
1158
1159         /* CPPC EPP feature require to set zero to the desire perf bit */
1160         value &= ~AMD_CPPC_DES_PERF(~0L);
1161         value |= AMD_CPPC_DES_PERF(0);
1162
1163         if (cpudata->epp_policy == cpudata->policy)
1164                 goto skip_epp;
1165
1166         cpudata->epp_policy = cpudata->policy;
1167
1168         /* Get BIOS pre-defined epp value */
1169         epp = amd_pstate_get_epp(cpudata, value);
1170         if (epp < 0) {
1171                 /**
1172                  * This return value can only be negative for shared_memory
1173                  * systems where EPP register read/write not supported.
1174                  */
1175                 goto skip_epp;
1176         }
1177
1178         if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1179                 epp = 0;
1180
1181         /* Set initial EPP value */
1182         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1183                 value &= ~GENMASK_ULL(31, 24);
1184                 value |= (u64)epp << 24;
1185         }
1186
1187         WRITE_ONCE(cpudata->cppc_req_cached, value);
1188         amd_pstate_set_epp(cpudata, epp);
1189 skip_epp:
1190         cpufreq_cpu_put(policy);
1191 }
1192
1193 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy)
1194 {
1195         struct amd_cpudata *cpudata = policy->driver_data;
1196
1197         if (!policy->cpuinfo.max_freq)
1198                 return -ENODEV;
1199
1200         pr_debug("set_policy: cpuinfo.max %u policy->max %u\n",
1201                                 policy->cpuinfo.max_freq, policy->max);
1202
1203         cpudata->policy = policy->policy;
1204
1205         amd_pstate_epp_init(policy->cpu);
1206
1207         return 0;
1208 }
1209
1210 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata)
1211 {
1212         struct cppc_perf_ctrls perf_ctrls;
1213         u64 value, max_perf;
1214         int ret;
1215
1216         ret = amd_pstate_enable(true);
1217         if (ret)
1218                 pr_err("failed to enable amd pstate during resume, return %d\n", ret);
1219
1220         value = READ_ONCE(cpudata->cppc_req_cached);
1221         max_perf = READ_ONCE(cpudata->highest_perf);
1222
1223         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1224                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1225         } else {
1226                 perf_ctrls.max_perf = max_perf;
1227                 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached);
1228                 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1229         }
1230 }
1231
1232 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy)
1233 {
1234         struct amd_cpudata *cpudata = policy->driver_data;
1235
1236         pr_debug("AMD CPU Core %d going online\n", cpudata->cpu);
1237
1238         if (cppc_state == AMD_PSTATE_ACTIVE) {
1239                 amd_pstate_epp_reenable(cpudata);
1240                 cpudata->suspended = false;
1241         }
1242
1243         return 0;
1244 }
1245
1246 static void amd_pstate_epp_offline(struct cpufreq_policy *policy)
1247 {
1248         struct amd_cpudata *cpudata = policy->driver_data;
1249         struct cppc_perf_ctrls perf_ctrls;
1250         int min_perf;
1251         u64 value;
1252
1253         min_perf = READ_ONCE(cpudata->lowest_perf);
1254         value = READ_ONCE(cpudata->cppc_req_cached);
1255
1256         mutex_lock(&amd_pstate_limits_lock);
1257         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1258                 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN;
1259
1260                 /* Set max perf same as min perf */
1261                 value &= ~AMD_CPPC_MAX_PERF(~0L);
1262                 value |= AMD_CPPC_MAX_PERF(min_perf);
1263                 value &= ~AMD_CPPC_MIN_PERF(~0L);
1264                 value |= AMD_CPPC_MIN_PERF(min_perf);
1265                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1266         } else {
1267                 perf_ctrls.desired_perf = 0;
1268                 perf_ctrls.max_perf = min_perf;
1269                 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE);
1270                 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1271         }
1272         mutex_unlock(&amd_pstate_limits_lock);
1273 }
1274
1275 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy)
1276 {
1277         struct amd_cpudata *cpudata = policy->driver_data;
1278
1279         pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu);
1280
1281         if (cpudata->suspended)
1282                 return 0;
1283
1284         if (cppc_state == AMD_PSTATE_ACTIVE)
1285                 amd_pstate_epp_offline(policy);
1286
1287         return 0;
1288 }
1289
1290 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy)
1291 {
1292         cpufreq_verify_within_cpu_limits(policy);
1293         pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min);
1294         return 0;
1295 }
1296
1297 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy)
1298 {
1299         struct amd_cpudata *cpudata = policy->driver_data;
1300         int ret;
1301
1302         /* avoid suspending when EPP is not enabled */
1303         if (cppc_state != AMD_PSTATE_ACTIVE)
1304                 return 0;
1305
1306         /* set this flag to avoid setting core offline*/
1307         cpudata->suspended = true;
1308
1309         /* disable CPPC in lowlevel firmware */
1310         ret = amd_pstate_enable(false);
1311         if (ret)
1312                 pr_err("failed to suspend, return %d\n", ret);
1313
1314         return 0;
1315 }
1316
1317 static int amd_pstate_epp_resume(struct cpufreq_policy *policy)
1318 {
1319         struct amd_cpudata *cpudata = policy->driver_data;
1320
1321         if (cpudata->suspended) {
1322                 mutex_lock(&amd_pstate_limits_lock);
1323
1324                 /* enable amd pstate from suspend state*/
1325                 amd_pstate_epp_reenable(cpudata);
1326
1327                 mutex_unlock(&amd_pstate_limits_lock);
1328
1329                 cpudata->suspended = false;
1330         }
1331
1332         return 0;
1333 }
1334
1335 static struct cpufreq_driver amd_pstate_driver = {
1336         .flags          = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
1337         .verify         = amd_pstate_verify,
1338         .target         = amd_pstate_target,
1339         .fast_switch    = amd_pstate_fast_switch,
1340         .init           = amd_pstate_cpu_init,
1341         .exit           = amd_pstate_cpu_exit,
1342         .suspend        = amd_pstate_cpu_suspend,
1343         .resume         = amd_pstate_cpu_resume,
1344         .set_boost      = amd_pstate_set_boost,
1345         .name           = "amd-pstate",
1346         .attr           = amd_pstate_attr,
1347 };
1348
1349 static struct cpufreq_driver amd_pstate_epp_driver = {
1350         .flags          = CPUFREQ_CONST_LOOPS,
1351         .verify         = amd_pstate_epp_verify_policy,
1352         .setpolicy      = amd_pstate_epp_set_policy,
1353         .init           = amd_pstate_epp_cpu_init,
1354         .exit           = amd_pstate_epp_cpu_exit,
1355         .offline        = amd_pstate_epp_cpu_offline,
1356         .online         = amd_pstate_epp_cpu_online,
1357         .suspend        = amd_pstate_epp_suspend,
1358         .resume         = amd_pstate_epp_resume,
1359         .name           = "amd_pstate_epp",
1360         .attr           = amd_pstate_epp_attr,
1361 };
1362
1363 static int __init amd_pstate_init(void)
1364 {
1365         struct device *dev_root;
1366         int ret;
1367
1368         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
1369                 return -ENODEV;
1370         /*
1371          * by default the pstate driver is disabled to load
1372          * enable the amd_pstate passive mode driver explicitly
1373          * with amd_pstate=passive or other modes in kernel command line
1374          */
1375         if (cppc_state == AMD_PSTATE_DISABLE) {
1376                 pr_info("driver load is disabled, boot with specific mode to enable this\n");
1377                 return -ENODEV;
1378         }
1379
1380         if (!acpi_cpc_valid()) {
1381                 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
1382                 return -ENODEV;
1383         }
1384
1385         /* don't keep reloading if cpufreq_driver exists */
1386         if (cpufreq_get_current_driver())
1387                 return -EEXIST;
1388
1389         /* capability check */
1390         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1391                 pr_debug("AMD CPPC MSR based functionality is supported\n");
1392                 if (cppc_state != AMD_PSTATE_ACTIVE)
1393                         current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
1394         } else {
1395                 pr_debug("AMD CPPC shared memory based functionality is supported\n");
1396                 static_call_update(amd_pstate_enable, cppc_enable);
1397                 static_call_update(amd_pstate_init_perf, cppc_init_perf);
1398                 static_call_update(amd_pstate_update_perf, cppc_update_perf);
1399         }
1400
1401         /* enable amd pstate feature */
1402         ret = amd_pstate_enable(true);
1403         if (ret) {
1404                 pr_err("failed to enable with return %d\n", ret);
1405                 return ret;
1406         }
1407
1408         ret = cpufreq_register_driver(current_pstate_driver);
1409         if (ret)
1410                 pr_err("failed to register with return %d\n", ret);
1411
1412         dev_root = bus_get_dev_root(&cpu_subsys);
1413         if (dev_root) {
1414                 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group);
1415                 put_device(dev_root);
1416                 if (ret) {
1417                         pr_err("sysfs attribute export failed with error %d.\n", ret);
1418                         goto global_attr_free;
1419                 }
1420         }
1421
1422         return ret;
1423
1424 global_attr_free:
1425         cpufreq_unregister_driver(current_pstate_driver);
1426         return ret;
1427 }
1428 device_initcall(amd_pstate_init);
1429
1430 static int __init amd_pstate_param(char *str)
1431 {
1432         size_t size;
1433         int mode_idx;
1434
1435         if (!str)
1436                 return -EINVAL;
1437
1438         size = strlen(str);
1439         mode_idx = get_mode_idx_from_str(str, size);
1440
1441         if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) {
1442                 cppc_state = mode_idx;
1443                 if (cppc_state == AMD_PSTATE_DISABLE)
1444                         pr_info("driver is explicitly disabled\n");
1445
1446                 if (cppc_state == AMD_PSTATE_ACTIVE)
1447                         current_pstate_driver = &amd_pstate_epp_driver;
1448
1449                 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED)
1450                         current_pstate_driver = &amd_pstate_driver;
1451
1452                 return 0;
1453         }
1454
1455         return -EINVAL;
1456 }
1457 early_param("amd_pstate", amd_pstate_param);
1458
1459 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
1460 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");