drivers/cpufreq/amd-pstate.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * amd-pstate.c - AMD Processor P-state Frequency Driver
   4  *
   5  * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
   6  *
   7  * Author: Huang Rui <ray.huang@amd.com>
   8  *
   9  * AMD P-State introduces a new CPU performance scaling design for AMD
  10  * processors using the ACPI Collaborative Performance and Power Control (CPPC)
  11  * feature which works with the AMD SMU firmware providing a finer grained
  12  * frequency control range. It is to replace the legacy ACPI P-States control,
  13  * allows a flexible, low-latency interface for the Linux kernel to directly
  14  * communicate the performance hints to hardware.
  15  *
  16  * AMD P-State is supported on recent AMD Zen base CPU series include some of
  17  * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
  18  * P-State supported system. And there are two types of hardware implementations
  19  * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
  20  * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
  21  */
  22
  23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  24
  25 #include <linux/kernel.h>
  26 #include <linux/module.h>
  27 #include <linux/init.h>
  28 #include <linux/smp.h>
  29 #include <linux/sched.h>
  30 #include <linux/cpufreq.h>
  31 #include <linux/compiler.h>
  32 #include <linux/dmi.h>
  33 #include <linux/slab.h>
  34 #include <linux/acpi.h>
  35 #include <linux/io.h>
  36 #include <linux/delay.h>
  37 #include <linux/uaccess.h>
  38 #include <linux/static_call.h>
  39 #include <linux/amd-pstate.h>
  40
  41 #include <acpi/processor.h>
  42 #include <acpi/cppc_acpi.h>
  43
  44 #include <asm/msr.h>
  45 #include <asm/processor.h>
  46 #include <asm/cpufeature.h>
  47 #include <asm/cpu_device_id.h>
  48 #include "amd-pstate-trace.h"
  49
  50 #define AMD_PSTATE_TRANSITION_LATENCY   20000
  51 #define AMD_PSTATE_TRANSITION_DELAY     1000
  52
  53 /*
  54  * TODO: We need more time to fine tune processors with shared memory solution
  55  * with community together.
  56  *
  57  * There are some performance drops on the CPU benchmarks which reports from
  58  * Suse. We are co-working with them to fine tune the shared memory solution. So
  59  * we disable it by default to go acpi-cpufreq on these processors and add a
  60  * module parameter to be able to enable it manually for debugging.
  61  */
  62 static bool shared_mem = false;
  63 module_param(shared_mem, bool, 0444);
  64 MODULE_PARM_DESC(shared_mem,
  65                  "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)");
  66
  67 static struct cpufreq_driver amd_pstate_driver;
  68
  69 static inline int pstate_enable(bool enable)
  70 {
  71         return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable);
  72 }
  73
  74 static int cppc_enable(bool enable)
  75 {
  76         int cpu, ret = 0;
  77
  78         for_each_present_cpu(cpu) {
  79                 ret = cppc_set_enable(cpu, enable);
  80                 if (ret)
  81                         return ret;
  82         }
  83
  84         return ret;
  85 }
  86
  87 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable);
  88
  89 static inline int amd_pstate_enable(bool enable)
  90 {
  91         return static_call(amd_pstate_enable)(enable);
  92 }
  93
  94 static int pstate_init_perf(struct amd_cpudata *cpudata)
  95 {
  96         u64 cap1;
  97         u32 highest_perf;
  98
  99         int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
 100                                      &cap1);
 101         if (ret)
 102                 return ret;
 103
 104         /*
 105          * TODO: Introduce AMD specific power feature.
 106          *
 107          * CPPC entry doesn't indicate the highest performance in some ASICs.
 108          */
 109         highest_perf = amd_get_highest_perf();
 110         if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1))
 111                 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
 112
 113         WRITE_ONCE(cpudata->highest_perf, highest_perf);
 114
 115         WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
 116         WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
 117         WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));
 118
 119         return 0;
 120 }
 121
 122 static int cppc_init_perf(struct amd_cpudata *cpudata)
 123 {
 124         struct cppc_perf_caps cppc_perf;
 125         u32 highest_perf;
 126
 127         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 128         if (ret)
 129                 return ret;
 130
 131         highest_perf = amd_get_highest_perf();
 132         if (highest_perf > cppc_perf.highest_perf)
 133                 highest_perf = cppc_perf.highest_perf;
 134
 135         WRITE_ONCE(cpudata->highest_perf, highest_perf);
 136
 137         WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
 138         WRITE_ONCE(cpudata->lowest_nonlinear_perf,
 139                    cppc_perf.lowest_nonlinear_perf);
 140         WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
 141
 142         return 0;
 143 }
 144
 145 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
 146
 147 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
 148 {
 149         return static_call(amd_pstate_init_perf)(cpudata);
 150 }
 151
 152 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,
 153                                u32 des_perf, u32 max_perf, bool fast_switch)
 154 {
 155         if (fast_switch)
 156                 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));
 157         else
 158                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
 159                               READ_ONCE(cpudata->cppc_req_cached));
 160 }
 161
 162 static void cppc_update_perf(struct amd_cpudata *cpudata,
 163                              u32 min_perf, u32 des_perf,
 164                              u32 max_perf, bool fast_switch)
 165 {
 166         struct cppc_perf_ctrls perf_ctrls;
 167
 168         perf_ctrls.max_perf = max_perf;
 169         perf_ctrls.min_perf = min_perf;
 170         perf_ctrls.desired_perf = des_perf;
 171
 172         cppc_set_perf(cpudata->cpu, &perf_ctrls);
 173 }
 174
 175 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf);
 176
 177 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,
 178                                           u32 min_perf, u32 des_perf,
 179                                           u32 max_perf, bool fast_switch)
 180 {
 181         static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,
 182                                             max_perf, fast_switch);
 183 }
 184
 185 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
 186 {
 187         u64 aperf, mperf, tsc;
 188         unsigned long flags;
 189
 190         local_irq_save(flags);
 191         rdmsrl(MSR_IA32_APERF, aperf);
 192         rdmsrl(MSR_IA32_MPERF, mperf);
 193         tsc = rdtsc();
 194
 195         if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
 196                 local_irq_restore(flags);
 197                 return false;
 198         }
 199
 200         local_irq_restore(flags);
 201
 202         cpudata->cur.aperf = aperf;
 203         cpudata->cur.mperf = mperf;
 204         cpudata->cur.tsc =  tsc;
 205         cpudata->cur.aperf -= cpudata->prev.aperf;
 206         cpudata->cur.mperf -= cpudata->prev.mperf;
 207         cpudata->cur.tsc -= cpudata->prev.tsc;
 208
 209         cpudata->prev.aperf = aperf;
 210         cpudata->prev.mperf = mperf;
 211         cpudata->prev.tsc = tsc;
 212
 213         cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
 214
 215         return true;
 216 }
 217
 218 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
 219                               u32 des_perf, u32 max_perf, bool fast_switch)
 220 {
 221         u64 prev = READ_ONCE(cpudata->cppc_req_cached);
 222         u64 value = prev;
 223
 224         des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
 225         value &= ~AMD_CPPC_MIN_PERF(~0L);
 226         value |= AMD_CPPC_MIN_PERF(min_perf);
 227
 228         value &= ~AMD_CPPC_DES_PERF(~0L);
 229         value |= AMD_CPPC_DES_PERF(des_perf);
 230
 231         value &= ~AMD_CPPC_MAX_PERF(~0L);
 232         value |= AMD_CPPC_MAX_PERF(max_perf);
 233
 234         if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
 235                 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
 236                         cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
 237                                 cpudata->cpu, (value != prev), fast_switch);
 238         }
 239
 240         if (value == prev)
 241                 return;
 242
 243         WRITE_ONCE(cpudata->cppc_req_cached, value);
 244
 245         amd_pstate_update_perf(cpudata, min_perf, des_perf,
 246                                max_perf, fast_switch);
 247 }
 248
 249 static int amd_pstate_verify(struct cpufreq_policy_data *policy)
 250 {
 251         cpufreq_verify_within_cpu_limits(policy);
 252
 253         return 0;
 254 }
 255
 256 static int amd_pstate_target(struct cpufreq_policy *policy,
 257                              unsigned int target_freq,
 258                              unsigned int relation)
 259 {
 260         struct cpufreq_freqs freqs;
 261         struct amd_cpudata *cpudata = policy->driver_data;
 262         unsigned long max_perf, min_perf, des_perf, cap_perf;
 263
 264         if (!cpudata->max_freq)
 265                 return -ENODEV;
 266
 267         cap_perf = READ_ONCE(cpudata->highest_perf);
 268         min_perf = READ_ONCE(cpudata->lowest_perf);
 269         max_perf = cap_perf;
 270
 271         freqs.old = policy->cur;
 272         freqs.new = target_freq;
 273
 274         des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf,
 275                                      cpudata->max_freq);
 276
 277         cpufreq_freq_transition_begin(policy, &freqs);
 278         amd_pstate_update(cpudata, min_perf, des_perf,
 279                           max_perf, false);
 280         cpufreq_freq_transition_end(policy, &freqs, false);
 281
 282         return 0;
 283 }
 284
 285 static void amd_pstate_adjust_perf(unsigned int cpu,
 286                                    unsigned long _min_perf,
 287                                    unsigned long target_perf,
 288                                    unsigned long capacity)
 289 {
 290         unsigned long max_perf, min_perf, des_perf,
 291                       cap_perf, lowest_nonlinear_perf;
 292         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
 293         struct amd_cpudata *cpudata = policy->driver_data;
 294
 295         cap_perf = READ_ONCE(cpudata->highest_perf);
 296         lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
 297
 298         des_perf = cap_perf;
 299         if (target_perf < capacity)
 300                 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
 301
 302         min_perf = READ_ONCE(cpudata->highest_perf);
 303         if (_min_perf < capacity)
 304                 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
 305
 306         if (min_perf < lowest_nonlinear_perf)
 307                 min_perf = lowest_nonlinear_perf;
 308
 309         max_perf = cap_perf;
 310         if (max_perf < min_perf)
 311                 max_perf = min_perf;
 312
 313         amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true);
 314 }
 315
 316 static int amd_get_min_freq(struct amd_cpudata *cpudata)
 317 {
 318         struct cppc_perf_caps cppc_perf;
 319
 320         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 321         if (ret)
 322                 return ret;
 323
 324         /* Switch to khz */
 325         return cppc_perf.lowest_freq * 1000;
 326 }
 327
 328 static int amd_get_max_freq(struct amd_cpudata *cpudata)
 329 {
 330         struct cppc_perf_caps cppc_perf;
 331         u32 max_perf, max_freq, nominal_freq, nominal_perf;
 332         u64 boost_ratio;
 333
 334         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 335         if (ret)
 336                 return ret;
 337
 338         nominal_freq = cppc_perf.nominal_freq;
 339         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 340         max_perf = READ_ONCE(cpudata->highest_perf);
 341
 342         boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT,
 343                               nominal_perf);
 344
 345         max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT;
 346
 347         /* Switch to khz */
 348         return max_freq * 1000;
 349 }
 350
 351 static int amd_get_nominal_freq(struct amd_cpudata *cpudata)
 352 {
 353         struct cppc_perf_caps cppc_perf;
 354
 355         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 356         if (ret)
 357                 return ret;
 358
 359         /* Switch to khz */
 360         return cppc_perf.nominal_freq * 1000;
 361 }
 362
 363 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata)
 364 {
 365         struct cppc_perf_caps cppc_perf;
 366         u32 lowest_nonlinear_freq, lowest_nonlinear_perf,
 367             nominal_freq, nominal_perf;
 368         u64 lowest_nonlinear_ratio;
 369
 370         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 371         if (ret)
 372                 return ret;
 373
 374         nominal_freq = cppc_perf.nominal_freq;
 375         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 376
 377         lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
 378
 379         lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT,
 380                                          nominal_perf);
 381
 382         lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT;
 383
 384         /* Switch to khz */
 385         return lowest_nonlinear_freq * 1000;
 386 }
 387
 388 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
 389 {
 390         struct amd_cpudata *cpudata = policy->driver_data;
 391         int ret;
 392
 393         if (!cpudata->boost_supported) {
 394                 pr_err("Boost mode is not supported by this processor or SBIOS\n");
 395                 return -EINVAL;
 396         }
 397
 398         if (state)
 399                 policy->cpuinfo.max_freq = cpudata->max_freq;
 400         else
 401                 policy->cpuinfo.max_freq = cpudata->nominal_freq;
 402
 403         policy->max = policy->cpuinfo.max_freq;
 404
 405         ret = freq_qos_update_request(&cpudata->req[1],
 406                                       policy->cpuinfo.max_freq);
 407         if (ret < 0)
 408                 return ret;
 409
 410         return 0;
 411 }
 412
 413 static void amd_pstate_boost_init(struct amd_cpudata *cpudata)
 414 {
 415         u32 highest_perf, nominal_perf;
 416
 417         highest_perf = READ_ONCE(cpudata->highest_perf);
 418         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 419
 420         if (highest_perf <= nominal_perf)
 421                 return;
 422
 423         cpudata->boost_supported = true;
 424         amd_pstate_driver.boost_enabled = true;
 425 }
 426
 427 static void amd_perf_ctl_reset(unsigned int cpu)
 428 {
 429         wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0);
 430 }
 431
 432 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
 433 {
 434         int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
 435         struct device *dev;
 436         struct amd_cpudata *cpudata;
 437
 438         /*
 439          * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
 440          * which is ideal for initialization process.
 441          */
 442         amd_perf_ctl_reset(policy->cpu);
 443         dev = get_cpu_device(policy->cpu);
 444         if (!dev)
 445                 return -ENODEV;
 446
 447         cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
 448         if (!cpudata)
 449                 return -ENOMEM;
 450
 451         cpudata->cpu = policy->cpu;
 452
 453         ret = amd_pstate_init_perf(cpudata);
 454         if (ret)
 455                 goto free_cpudata1;
 456
 457         min_freq = amd_get_min_freq(cpudata);
 458         max_freq = amd_get_max_freq(cpudata);
 459         nominal_freq = amd_get_nominal_freq(cpudata);
 460         lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
 461
 462         if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
 463                 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
 464                         min_freq, max_freq);
 465                 ret = -EINVAL;
 466                 goto free_cpudata1;
 467         }
 468
 469         policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY;
 470         policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY;
 471
 472         policy->min = min_freq;
 473         policy->max = max_freq;
 474
 475         policy->cpuinfo.min_freq = min_freq;
 476         policy->cpuinfo.max_freq = max_freq;
 477
 478         /* It will be updated by governor */
 479         policy->cur = policy->cpuinfo.min_freq;
 480
 481         if (boot_cpu_has(X86_FEATURE_CPPC))
 482                 policy->fast_switch_possible = true;
 483
 484         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
 485                                    FREQ_QOS_MIN, policy->cpuinfo.min_freq);
 486         if (ret < 0) {
 487                 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
 488                 goto free_cpudata1;
 489         }
 490
 491         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
 492                                    FREQ_QOS_MAX, policy->cpuinfo.max_freq);
 493         if (ret < 0) {
 494                 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
 495                 goto free_cpudata2;
 496         }
 497
 498         /* Initial processor data capability frequencies */
 499         cpudata->max_freq = max_freq;
 500         cpudata->min_freq = min_freq;
 501         cpudata->nominal_freq = nominal_freq;
 502         cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
 503
 504         policy->driver_data = cpudata;
 505
 506         amd_pstate_boost_init(cpudata);
 507
 508         return 0;
 509
 510 free_cpudata2:
 511         freq_qos_remove_request(&cpudata->req[0]);
 512 free_cpudata1:
 513         kfree(cpudata);
 514         return ret;
 515 }
 516
 517 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy)
 518 {
 519         struct amd_cpudata *cpudata = policy->driver_data;
 520
 521         freq_qos_remove_request(&cpudata->req[1]);
 522         freq_qos_remove_request(&cpudata->req[0]);
 523         kfree(cpudata);
 524
 525         return 0;
 526 }
 527
 528 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)
 529 {
 530         int ret;
 531
 532         ret = amd_pstate_enable(true);
 533         if (ret)
 534                 pr_err("failed to enable amd-pstate during resume, return %d\n", ret);
 535
 536         return ret;
 537 }
 538
 539 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)
 540 {
 541         int ret;
 542
 543         ret = amd_pstate_enable(false);
 544         if (ret)
 545                 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret);
 546
 547         return ret;
 548 }
 549
 550 /* Sysfs attributes */
 551
 552 /*
 553  * This frequency is to indicate the maximum hardware frequency.
 554  * If boost is not active but supported, the frequency will be larger than the
 555  * one in cpuinfo.
 556  */
 557 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
 558                                         char *buf)
 559 {
 560         int max_freq;
 561         struct amd_cpudata *cpudata = policy->driver_data;
 562
 563         max_freq = amd_get_max_freq(cpudata);
 564         if (max_freq < 0)
 565                 return max_freq;
 566
 567         return sprintf(&buf[0], "%u\n", max_freq);
 568 }
 569
 570 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
 571                                                      char *buf)
 572 {
 573         int freq;
 574         struct amd_cpudata *cpudata = policy->driver_data;
 575
 576         freq = amd_get_lowest_nonlinear_freq(cpudata);
 577         if (freq < 0)
 578                 return freq;
 579
 580         return sprintf(&buf[0], "%u\n", freq);
 581 }
 582
 583 /*
 584  * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
 585  * need to expose it to sysfs.
 586  */
 587 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
 588                                             char *buf)
 589 {
 590         u32 perf;
 591         struct amd_cpudata *cpudata = policy->driver_data;
 592
 593         perf = READ_ONCE(cpudata->highest_perf);
 594
 595         return sprintf(&buf[0], "%u\n", perf);
 596 }
 597
 598 cpufreq_freq_attr_ro(amd_pstate_max_freq);
 599 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
 600
 601 cpufreq_freq_attr_ro(amd_pstate_highest_perf);
 602
 603 static struct freq_attr *amd_pstate_attr[] = {
 604         &amd_pstate_max_freq,
 605         &amd_pstate_lowest_nonlinear_freq,
 606         &amd_pstate_highest_perf,
 607         NULL,
 608 };
 609
 610 static struct cpufreq_driver amd_pstate_driver = {
 611         .flags          = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
 612         .verify         = amd_pstate_verify,
 613         .target         = amd_pstate_target,
 614         .init           = amd_pstate_cpu_init,
 615         .exit           = amd_pstate_cpu_exit,
 616         .suspend        = amd_pstate_cpu_suspend,
 617         .resume         = amd_pstate_cpu_resume,
 618         .set_boost      = amd_pstate_set_boost,
 619         .name           = "amd-pstate",
 620         .attr           = amd_pstate_attr,
 621 };
 622
 623 static int __init amd_pstate_init(void)
 624 {
 625         int ret;
 626
 627         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
 628                 return -ENODEV;
 629
 630         if (!acpi_cpc_valid()) {
 631                 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
 632                 return -ENODEV;
 633         }
 634
 635         /* don't keep reloading if cpufreq_driver exists */
 636         if (cpufreq_get_current_driver())
 637                 return -EEXIST;
 638
 639         /* capability check */
 640         if (boot_cpu_has(X86_FEATURE_CPPC)) {
 641                 pr_debug("AMD CPPC MSR based functionality is supported\n");
 642                 amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf;
 643         } else if (shared_mem) {
 644                 static_call_update(amd_pstate_enable, cppc_enable);
 645                 static_call_update(amd_pstate_init_perf, cppc_init_perf);
 646                 static_call_update(amd_pstate_update_perf, cppc_update_perf);
 647         } else {
 648                 pr_info("This processor supports shared memory solution, you can enable it with amd_pstate.shared_mem=1\n");
 649                 return -ENODEV;
 650         }
 651
 652         /* enable amd pstate feature */
 653         ret = amd_pstate_enable(true);
 654         if (ret) {
 655                 pr_err("failed to enable amd-pstate with return %d\n", ret);
 656                 return ret;
 657         }
 658
 659         ret = cpufreq_register_driver(&amd_pstate_driver);
 660         if (ret)
 661                 pr_err("failed to register amd_pstate_driver with return %d\n",
 662                        ret);
 663
 664         return ret;
 665 }
 666 device_initcall(amd_pstate_init);
 667
 668 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
 669 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");
 670 MODULE_LICENSE("GPL");