drivers/cpufreq/amd-pstate.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * amd-pstate.c - AMD Processor P-state Frequency Driver
   4  *
   5  * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
   6  *
   7  * Author: Huang Rui <ray.huang@amd.com>
   8  *
   9  * AMD P-State introduces a new CPU performance scaling design for AMD
  10  * processors using the ACPI Collaborative Performance and Power Control (CPPC)
  11  * feature which works with the AMD SMU firmware providing a finer grained
  12  * frequency control range. It is to replace the legacy ACPI P-States control,
  13  * allows a flexible, low-latency interface for the Linux kernel to directly
  14  * communicate the performance hints to hardware.
  15  *
  16  * AMD P-State is supported on recent AMD Zen base CPU series include some of
  17  * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
  18  * P-State supported system. And there are two types of hardware implementations
  19  * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
  20  * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
  21  */
  22
  23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  24
  25 #include <linux/kernel.h>
  26 #include <linux/module.h>
  27 #include <linux/init.h>
  28 #include <linux/smp.h>
  29 #include <linux/sched.h>
  30 #include <linux/cpufreq.h>
  31 #include <linux/compiler.h>
  32 #include <linux/dmi.h>
  33 #include <linux/slab.h>
  34 #include <linux/acpi.h>
  35 #include <linux/io.h>
  36 #include <linux/delay.h>
  37 #include <linux/uaccess.h>
  38 #include <linux/static_call.h>
  39 #include <linux/amd-pstate.h>
  40
  41 #include <acpi/processor.h>
  42 #include <acpi/cppc_acpi.h>
  43
  44 #include <asm/msr.h>
  45 #include <asm/processor.h>
  46 #include <asm/cpufeature.h>
  47 #include <asm/cpu_device_id.h>
  48 #include "amd-pstate-trace.h"
  49
  50 #define AMD_PSTATE_TRANSITION_LATENCY   20000
  51 #define AMD_PSTATE_TRANSITION_DELAY     1000
  52
  53 /*
  54  * TODO: We need more time to fine tune processors with shared memory solution
  55  * with community together.
  56  *
  57  * There are some performance drops on the CPU benchmarks which reports from
  58  * Suse. We are co-working with them to fine tune the shared memory solution. So
  59  * we disable it by default to go acpi-cpufreq on these processors and add a
  60  * module parameter to be able to enable it manually for debugging.
  61  */
  62 static bool shared_mem = false;
  63 module_param(shared_mem, bool, 0444);
  64 MODULE_PARM_DESC(shared_mem,
  65                  "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)");
  66
  67 static struct cpufreq_driver amd_pstate_driver;
  68
  69 static inline int pstate_enable(bool enable)
  70 {
  71         return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable);
  72 }
  73
  74 static int cppc_enable(bool enable)
  75 {
  76         int cpu, ret = 0;
  77
  78         for_each_present_cpu(cpu) {
  79                 ret = cppc_set_enable(cpu, enable);
  80                 if (ret)
  81                         return ret;
  82         }
  83
  84         return ret;
  85 }
  86
  87 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable);
  88
  89 static inline int amd_pstate_enable(bool enable)
  90 {
  91         return static_call(amd_pstate_enable)(enable);
  92 }
  93
  94 static int pstate_init_perf(struct amd_cpudata *cpudata)
  95 {
  96         u64 cap1;
  97         u32 highest_perf;
  98
  99         int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
 100                                      &cap1);
 101         if (ret)
 102                 return ret;
 103
 104         /*
 105          * TODO: Introduce AMD specific power feature.
 106          *
 107          * CPPC entry doesn't indicate the highest performance in some ASICs.
 108          */
 109         highest_perf = amd_get_highest_perf();
 110         if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1))
 111                 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
 112
 113         WRITE_ONCE(cpudata->highest_perf, highest_perf);
 114
 115         WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
 116         WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
 117         WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));
 118
 119         return 0;
 120 }
 121
 122 static int cppc_init_perf(struct amd_cpudata *cpudata)
 123 {
 124         struct cppc_perf_caps cppc_perf;
 125         u32 highest_perf;
 126
 127         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 128         if (ret)
 129                 return ret;
 130
 131         highest_perf = amd_get_highest_perf();
 132         if (highest_perf > cppc_perf.highest_perf)
 133                 highest_perf = cppc_perf.highest_perf;
 134
 135         WRITE_ONCE(cpudata->highest_perf, highest_perf);
 136
 137         WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
 138         WRITE_ONCE(cpudata->lowest_nonlinear_perf,
 139                    cppc_perf.lowest_nonlinear_perf);
 140         WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
 141
 142         return 0;
 143 }
 144
 145 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
 146
 147 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
 148 {
 149         return static_call(amd_pstate_init_perf)(cpudata);
 150 }
 151
 152 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,
 153                                u32 des_perf, u32 max_perf, bool fast_switch)
 154 {
 155         if (fast_switch)
 156                 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));
 157         else
 158                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
 159                               READ_ONCE(cpudata->cppc_req_cached));
 160 }
 161
 162 static void cppc_update_perf(struct amd_cpudata *cpudata,
 163                              u32 min_perf, u32 des_perf,
 164                              u32 max_perf, bool fast_switch)
 165 {
 166         struct cppc_perf_ctrls perf_ctrls;
 167
 168         perf_ctrls.max_perf = max_perf;
 169         perf_ctrls.min_perf = min_perf;
 170         perf_ctrls.desired_perf = des_perf;
 171
 172         cppc_set_perf(cpudata->cpu, &perf_ctrls);
 173 }
 174
 175 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf);
 176
 177 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,
 178                                           u32 min_perf, u32 des_perf,
 179                                           u32 max_perf, bool fast_switch)
 180 {
 181         static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,
 182                                             max_perf, fast_switch);
 183 }
 184
 185 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
 186 {
 187         u64 aperf, mperf, tsc;
 188         unsigned long flags;
 189
 190         local_irq_save(flags);
 191         rdmsrl(MSR_IA32_APERF, aperf);
 192         rdmsrl(MSR_IA32_MPERF, mperf);
 193         tsc = rdtsc();
 194
 195         if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
 196                 local_irq_restore(flags);
 197                 return false;
 198         }
 199
 200         local_irq_restore(flags);
 201
 202         cpudata->cur.aperf = aperf;
 203         cpudata->cur.mperf = mperf;
 204         cpudata->cur.tsc =  tsc;
 205         cpudata->cur.aperf -= cpudata->prev.aperf;
 206         cpudata->cur.mperf -= cpudata->prev.mperf;
 207         cpudata->cur.tsc -= cpudata->prev.tsc;
 208
 209         cpudata->prev.aperf = aperf;
 210         cpudata->prev.mperf = mperf;
 211         cpudata->prev.tsc = tsc;
 212
 213         cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
 214
 215         return true;
 216 }
 217
 218 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
 219                               u32 des_perf, u32 max_perf, bool fast_switch)
 220 {
 221         u64 prev = READ_ONCE(cpudata->cppc_req_cached);
 222         u64 value = prev;
 223
 224         des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
 225         value &= ~AMD_CPPC_MIN_PERF(~0L);
 226         value |= AMD_CPPC_MIN_PERF(min_perf);
 227
 228         value &= ~AMD_CPPC_DES_PERF(~0L);
 229         value |= AMD_CPPC_DES_PERF(des_perf);
 230
 231         value &= ~AMD_CPPC_MAX_PERF(~0L);
 232         value |= AMD_CPPC_MAX_PERF(max_perf);
 233
 234         if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
 235                 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
 236                         cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
 237                                 cpudata->cpu, (value != prev), fast_switch);
 238         }
 239
 240         if (value == prev)
 241                 return;
 242
 243         WRITE_ONCE(cpudata->cppc_req_cached, value);
 244
 245         amd_pstate_update_perf(cpudata, min_perf, des_perf,
 246                                max_perf, fast_switch);
 247 }
 248
 249 static int amd_pstate_verify(struct cpufreq_policy_data *policy)
 250 {
 251         cpufreq_verify_within_cpu_limits(policy);
 252
 253         return 0;
 254 }
 255
 256 static int amd_pstate_target(struct cpufreq_policy *policy,
 257                              unsigned int target_freq,
 258                              unsigned int relation)
 259 {
 260         struct cpufreq_freqs freqs;
 261         struct amd_cpudata *cpudata = policy->driver_data;
 262         unsigned long max_perf, min_perf, des_perf, cap_perf;
 263
 264         if (!cpudata->max_freq)
 265                 return -ENODEV;
 266
 267         cap_perf = READ_ONCE(cpudata->highest_perf);
 268         min_perf = READ_ONCE(cpudata->lowest_perf);
 269         max_perf = cap_perf;
 270
 271         freqs.old = policy->cur;
 272         freqs.new = target_freq;
 273
 274         des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf,
 275                                      cpudata->max_freq);
 276
 277         cpufreq_freq_transition_begin(policy, &freqs);
 278         amd_pstate_update(cpudata, min_perf, des_perf,
 279                           max_perf, false);
 280         cpufreq_freq_transition_end(policy, &freqs, false);
 281
 282         return 0;
 283 }
 284
 285 static void amd_pstate_adjust_perf(unsigned int cpu,
 286                                    unsigned long _min_perf,
 287                                    unsigned long target_perf,
 288                                    unsigned long capacity)
 289 {
 290         unsigned long max_perf, min_perf, des_perf,
 291                       cap_perf, lowest_nonlinear_perf;
 292         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
 293         struct amd_cpudata *cpudata = policy->driver_data;
 294
 295         cap_perf = READ_ONCE(cpudata->highest_perf);
 296         lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
 297
 298         des_perf = cap_perf;
 299         if (target_perf < capacity)
 300                 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
 301
 302         min_perf = READ_ONCE(cpudata->highest_perf);
 303         if (_min_perf < capacity)
 304                 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
 305
 306         if (min_perf < lowest_nonlinear_perf)
 307                 min_perf = lowest_nonlinear_perf;
 308
 309         max_perf = cap_perf;
 310         if (max_perf < min_perf)
 311                 max_perf = min_perf;
 312
 313         amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true);
 314 }
 315
 316 static int amd_get_min_freq(struct amd_cpudata *cpudata)
 317 {
 318         struct cppc_perf_caps cppc_perf;
 319
 320         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 321         if (ret)
 322                 return ret;
 323
 324         /* Switch to khz */
 325         return cppc_perf.lowest_freq * 1000;
 326 }
 327
 328 static int amd_get_max_freq(struct amd_cpudata *cpudata)
 329 {
 330         struct cppc_perf_caps cppc_perf;
 331         u32 max_perf, max_freq, nominal_freq, nominal_perf;
 332         u64 boost_ratio;
 333
 334         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 335         if (ret)
 336                 return ret;
 337
 338         nominal_freq = cppc_perf.nominal_freq;
 339         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 340         max_perf = READ_ONCE(cpudata->highest_perf);
 341
 342         boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT,
 343                               nominal_perf);
 344
 345         max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT;
 346
 347         /* Switch to khz */
 348         return max_freq * 1000;
 349 }
 350
 351 static int amd_get_nominal_freq(struct amd_cpudata *cpudata)
 352 {
 353         struct cppc_perf_caps cppc_perf;
 354
 355         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 356         if (ret)
 357                 return ret;
 358
 359         /* Switch to khz */
 360         return cppc_perf.nominal_freq * 1000;
 361 }
 362
 363 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata)
 364 {
 365         struct cppc_perf_caps cppc_perf;
 366         u32 lowest_nonlinear_freq, lowest_nonlinear_perf,
 367             nominal_freq, nominal_perf;
 368         u64 lowest_nonlinear_ratio;
 369
 370         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 371         if (ret)
 372                 return ret;
 373
 374         nominal_freq = cppc_perf.nominal_freq;
 375         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 376
 377         lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
 378
 379         lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT,
 380                                          nominal_perf);
 381
 382         lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT;
 383
 384         /* Switch to khz */
 385         return lowest_nonlinear_freq * 1000;
 386 }
 387
 388 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
 389 {
 390         struct amd_cpudata *cpudata = policy->driver_data;
 391         int ret;
 392
 393         if (!cpudata->boost_supported) {
 394                 pr_err("Boost mode is not supported by this processor or SBIOS\n");
 395                 return -EINVAL;
 396         }
 397
 398         if (state)
 399                 policy->cpuinfo.max_freq = cpudata->max_freq;
 400         else
 401                 policy->cpuinfo.max_freq = cpudata->nominal_freq;
 402
 403         policy->max = policy->cpuinfo.max_freq;
 404
 405         ret = freq_qos_update_request(&cpudata->req[1],
 406                                       policy->cpuinfo.max_freq);
 407         if (ret < 0)
 408                 return ret;
 409
 410         return 0;
 411 }
 412
 413 static void amd_pstate_boost_init(struct amd_cpudata *cpudata)
 414 {
 415         u32 highest_perf, nominal_perf;
 416
 417         highest_perf = READ_ONCE(cpudata->highest_perf);
 418         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 419
 420         if (highest_perf <= nominal_perf)
 421                 return;
 422
 423         cpudata->boost_supported = true;
 424         amd_pstate_driver.boost_enabled = true;
 425 }
 426
 427 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
 428 {
 429         int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
 430         struct device *dev;
 431         struct amd_cpudata *cpudata;
 432
 433         dev = get_cpu_device(policy->cpu);
 434         if (!dev)
 435                 return -ENODEV;
 436
 437         cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
 438         if (!cpudata)
 439                 return -ENOMEM;
 440
 441         cpudata->cpu = policy->cpu;
 442
 443         ret = amd_pstate_init_perf(cpudata);
 444         if (ret)
 445                 goto free_cpudata1;
 446
 447         min_freq = amd_get_min_freq(cpudata);
 448         max_freq = amd_get_max_freq(cpudata);
 449         nominal_freq = amd_get_nominal_freq(cpudata);
 450         lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
 451
 452         if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
 453                 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
 454                         min_freq, max_freq);
 455                 ret = -EINVAL;
 456                 goto free_cpudata1;
 457         }
 458
 459         policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY;
 460         policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY;
 461
 462         policy->min = min_freq;
 463         policy->max = max_freq;
 464
 465         policy->cpuinfo.min_freq = min_freq;
 466         policy->cpuinfo.max_freq = max_freq;
 467
 468         /* It will be updated by governor */
 469         policy->cur = policy->cpuinfo.min_freq;
 470
 471         if (boot_cpu_has(X86_FEATURE_CPPC))
 472                 policy->fast_switch_possible = true;
 473
 474         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
 475                                    FREQ_QOS_MIN, policy->cpuinfo.min_freq);
 476         if (ret < 0) {
 477                 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
 478                 goto free_cpudata1;
 479         }
 480
 481         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
 482                                    FREQ_QOS_MAX, policy->cpuinfo.max_freq);
 483         if (ret < 0) {
 484                 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
 485                 goto free_cpudata2;
 486         }
 487
 488         /* Initial processor data capability frequencies */
 489         cpudata->max_freq = max_freq;
 490         cpudata->min_freq = min_freq;
 491         cpudata->nominal_freq = nominal_freq;
 492         cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
 493
 494         policy->driver_data = cpudata;
 495
 496         amd_pstate_boost_init(cpudata);
 497
 498         return 0;
 499
 500 free_cpudata2:
 501         freq_qos_remove_request(&cpudata->req[0]);
 502 free_cpudata1:
 503         kfree(cpudata);
 504         return ret;
 505 }
 506
 507 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy)
 508 {
 509         struct amd_cpudata *cpudata = policy->driver_data;
 510
 511         freq_qos_remove_request(&cpudata->req[1]);
 512         freq_qos_remove_request(&cpudata->req[0]);
 513         kfree(cpudata);
 514
 515         return 0;
 516 }
 517
 518 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)
 519 {
 520         int ret;
 521
 522         ret = amd_pstate_enable(true);
 523         if (ret)
 524                 pr_err("failed to enable amd-pstate during resume, return %d\n", ret);
 525
 526         return ret;
 527 }
 528
 529 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)
 530 {
 531         int ret;
 532
 533         ret = amd_pstate_enable(false);
 534         if (ret)
 535                 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret);
 536
 537         return ret;
 538 }
 539
 540 /* Sysfs attributes */
 541
 542 /*
 543  * This frequency is to indicate the maximum hardware frequency.
 544  * If boost is not active but supported, the frequency will be larger than the
 545  * one in cpuinfo.
 546  */
 547 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
 548                                         char *buf)
 549 {
 550         int max_freq;
 551         struct amd_cpudata *cpudata = policy->driver_data;
 552
 553         max_freq = amd_get_max_freq(cpudata);
 554         if (max_freq < 0)
 555                 return max_freq;
 556
 557         return sprintf(&buf[0], "%u\n", max_freq);
 558 }
 559
 560 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
 561                                                      char *buf)
 562 {
 563         int freq;
 564         struct amd_cpudata *cpudata = policy->driver_data;
 565
 566         freq = amd_get_lowest_nonlinear_freq(cpudata);
 567         if (freq < 0)
 568                 return freq;
 569
 570         return sprintf(&buf[0], "%u\n", freq);
 571 }
 572
 573 /*
 574  * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
 575  * need to expose it to sysfs.
 576  */
 577 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
 578                                             char *buf)
 579 {
 580         u32 perf;
 581         struct amd_cpudata *cpudata = policy->driver_data;
 582
 583         perf = READ_ONCE(cpudata->highest_perf);
 584
 585         return sprintf(&buf[0], "%u\n", perf);
 586 }
 587
 588 cpufreq_freq_attr_ro(amd_pstate_max_freq);
 589 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
 590
 591 cpufreq_freq_attr_ro(amd_pstate_highest_perf);
 592
 593 static struct freq_attr *amd_pstate_attr[] = {
 594         &amd_pstate_max_freq,
 595         &amd_pstate_lowest_nonlinear_freq,
 596         &amd_pstate_highest_perf,
 597         NULL,
 598 };
 599
 600 static struct cpufreq_driver amd_pstate_driver = {
 601         .flags          = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
 602         .verify         = amd_pstate_verify,
 603         .target         = amd_pstate_target,
 604         .init           = amd_pstate_cpu_init,
 605         .exit           = amd_pstate_cpu_exit,
 606         .suspend        = amd_pstate_cpu_suspend,
 607         .resume         = amd_pstate_cpu_resume,
 608         .set_boost      = amd_pstate_set_boost,
 609         .name           = "amd-pstate",
 610         .attr           = amd_pstate_attr,
 611 };
 612
 613 static int __init amd_pstate_init(void)
 614 {
 615         int ret;
 616
 617         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
 618                 return -ENODEV;
 619
 620         if (!acpi_cpc_valid()) {
 621                 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
 622                 return -ENODEV;
 623         }
 624
 625         /* don't keep reloading if cpufreq_driver exists */
 626         if (cpufreq_get_current_driver())
 627                 return -EEXIST;
 628
 629         /* capability check */
 630         if (boot_cpu_has(X86_FEATURE_CPPC)) {
 631                 pr_debug("AMD CPPC MSR based functionality is supported\n");
 632                 amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf;
 633         } else if (shared_mem) {
 634                 static_call_update(amd_pstate_enable, cppc_enable);
 635                 static_call_update(amd_pstate_init_perf, cppc_init_perf);
 636                 static_call_update(amd_pstate_update_perf, cppc_update_perf);
 637         } else {
 638                 pr_info("This processor supports shared memory solution, you can enable it with amd_pstate.shared_mem=1\n");
 639                 return -ENODEV;
 640         }
 641
 642         /* enable amd pstate feature */
 643         ret = amd_pstate_enable(true);
 644         if (ret) {
 645                 pr_err("failed to enable amd-pstate with return %d\n", ret);
 646                 return ret;
 647         }
 648
 649         ret = cpufreq_register_driver(&amd_pstate_driver);
 650         if (ret)
 651                 pr_err("failed to register amd_pstate_driver with return %d\n",
 652                        ret);
 653
 654         return ret;
 655 }
 656
 657 static void __exit amd_pstate_exit(void)
 658 {
 659         cpufreq_unregister_driver(&amd_pstate_driver);
 660
 661         amd_pstate_enable(false);
 662 }
 663
 664 module_init(amd_pstate_init);
 665 module_exit(amd_pstate_exit);
 666
 667 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
 668 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");
 669 MODULE_LICENSE("GPL");