// SPDX-License-Identifier: GPL-2.0-only
/*
- * Intel Running Average Power Limit (RAPL) Driver
- * Copyright (c) 2013, Intel Corporation.
+ * Common code for Intel Running Average Power Limit (RAPL) support.
+ * Copyright (c) 2019, Intel Corporation.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/cpu.h>
#include <linux/powercap.h>
#include <linux/suspend.h>
+#include <asm/iosf_mbi.h>
#include <linux/intel_rapl.h>
-#include <asm/iosf_mbi.h>
-#include <asm/processor.h>
+#include <linux/processor.h>
#include <asm/cpu_device_id.h>
#include <asm/intel-family.h>
#define PP_POLICY_MASK 0x1F
/* Non HW constants */
-#define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */
+#define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */
#define RAPL_PRIMITIVE_DUMMY BIT(2)
#define TIME_WINDOW_MAX_MSEC 40000
#define TIME_WINDOW_MIN_MSEC 250
-#define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */
+#define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */
enum unit_type {
- ARBITRARY_UNIT, /* no translation */
+ ARBITRARY_UNIT, /* no translation */
POWER_UNIT,
ENERGY_UNIT,
TIME_UNIT,
};
-/* private data for RAPL MSR Interface */
-static struct rapl_if_priv rapl_msr_priv = {
- .reg_unit = MSR_RAPL_POWER_UNIT,
- .regs[RAPL_DOMAIN_PACKAGE] = {
- MSR_PKG_POWER_LIMIT, MSR_PKG_ENERGY_STATUS, MSR_PKG_PERF_STATUS, 0, MSR_PKG_POWER_INFO },
- .regs[RAPL_DOMAIN_PP0] = {
- MSR_PP0_POWER_LIMIT, MSR_PP0_ENERGY_STATUS, 0, MSR_PP0_POLICY, 0 },
- .regs[RAPL_DOMAIN_PP1] = {
- MSR_PP1_POWER_LIMIT, MSR_PP1_ENERGY_STATUS, 0, MSR_PP1_POLICY, 0 },
- .regs[RAPL_DOMAIN_DRAM] = {
- MSR_DRAM_POWER_LIMIT, MSR_DRAM_ENERGY_STATUS, MSR_DRAM_PERF_STATUS, 0, MSR_DRAM_POWER_INFO },
- .regs[RAPL_DOMAIN_PLATFORM] = {
- MSR_PLATFORM_POWER_LIMIT, MSR_PLATFORM_ENERGY_STATUS, 0, 0, 0},
-};
-
/* per domain data, some are optional */
#define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
int (*check_unit)(struct rapl_package *rp, int cpu);
void (*set_floor_freq)(struct rapl_domain *rd, bool mode);
u64 (*compute_time_window)(struct rapl_package *rp, u64 val,
- bool to_raw);
+ bool to_raw);
unsigned int dram_domain_energy_unit;
};
static struct rapl_defaults *rapl_defaults;
static void rapl_init_domains(struct rapl_package *rp);
static int rapl_read_data_raw(struct rapl_domain *rd,
- enum rapl_primitives prim,
- bool xlate, u64 *data);
+ enum rapl_primitives prim,
+ bool xlate, u64 *data);
static int rapl_write_data_raw(struct rapl_domain *rd,
- enum rapl_primitives prim,
- unsigned long long value);
+ enum rapl_primitives prim,
+ unsigned long long value);
static u64 rapl_unit_xlate(struct rapl_domain *rd,
- enum unit_type type, u64 value,
- int to_raw);
+ enum unit_type type, u64 value, int to_raw);
static void package_power_limit_irq_save(struct rapl_package *rp);
+static int rapl_init_core(void);
+static void rapl_remove_core(void);
-static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */
+static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */
-static const char * const rapl_domain_names[] = {
+static const char *const rapl_domain_names[] = {
"package",
"core",
"uncore",
"psys",
};
-/* caller to ensure CPU hotplug lock is held */
-static struct rapl_package *rapl_find_package_domain(int cpu, struct rapl_if_priv *priv)
-{
- int id = topology_logical_die_id(cpu);
- struct rapl_package *rp;
-
- list_for_each_entry(rp, &rapl_packages, plist) {
- if (rp->id == id && rp->priv->control_type == priv->control_type)
- return rp;
- }
-
- return NULL;
-}
-
-static int get_energy_counter(struct powercap_zone *power_zone, u64 *energy_raw)
+static int get_energy_counter(struct powercap_zone *power_zone,
+ u64 *energy_raw)
{
struct rapl_domain *rd;
u64 energy_now;
static const struct powercap_zone_ops zone_ops[] = {
/* RAPL_DOMAIN_PACKAGE */
{
- .get_energy_uj = get_energy_counter,
- .get_max_energy_range_uj = get_max_energy_counter,
- .release = release_zone,
- .set_enable = set_domain_enable,
- .get_enable = get_domain_enable,
- },
+ .get_energy_uj = get_energy_counter,
+ .get_max_energy_range_uj = get_max_energy_counter,
+ .release = release_zone,
+ .set_enable = set_domain_enable,
+ .get_enable = get_domain_enable,
+ },
/* RAPL_DOMAIN_PP0 */
{
- .get_energy_uj = get_energy_counter,
- .get_max_energy_range_uj = get_max_energy_counter,
- .release = release_zone,
- .set_enable = set_domain_enable,
- .get_enable = get_domain_enable,
- },
+ .get_energy_uj = get_energy_counter,
+ .get_max_energy_range_uj = get_max_energy_counter,
+ .release = release_zone,
+ .set_enable = set_domain_enable,
+ .get_enable = get_domain_enable,
+ },
/* RAPL_DOMAIN_PP1 */
{
- .get_energy_uj = get_energy_counter,
- .get_max_energy_range_uj = get_max_energy_counter,
- .release = release_zone,
- .set_enable = set_domain_enable,
- .get_enable = get_domain_enable,
- },
+ .get_energy_uj = get_energy_counter,
+ .get_max_energy_range_uj = get_max_energy_counter,
+ .release = release_zone,
+ .set_enable = set_domain_enable,
+ .get_enable = get_domain_enable,
+ },
/* RAPL_DOMAIN_DRAM */
{
- .get_energy_uj = get_energy_counter,
- .get_max_energy_range_uj = get_max_energy_counter,
- .release = release_zone,
- .set_enable = set_domain_enable,
- .get_enable = get_domain_enable,
- },
+ .get_energy_uj = get_energy_counter,
+ .get_max_energy_range_uj = get_max_energy_counter,
+ .release = release_zone,
+ .set_enable = set_domain_enable,
+ .get_enable = get_domain_enable,
+ },
/* RAPL_DOMAIN_PLATFORM */
{
- .get_energy_uj = get_energy_counter,
- .get_max_energy_range_uj = get_max_energy_counter,
- .release = release_zone,
- .set_enable = set_domain_enable,
- .get_enable = get_domain_enable,
- },
+ .get_energy_uj = get_energy_counter,
+ .get_max_energy_range_uj = get_max_energy_counter,
+ .release = release_zone,
+ .set_enable = set_domain_enable,
+ .get_enable = get_domain_enable,
+ },
};
-
/*
* Constraint index used by powercap can be different than power limit (PL)
- * index in that some PLs maybe missing due to non-existant MSRs. So we
+ * index in that some PLs maybe missing due to non-existent MSRs. So we
* need to convert here by finding the valid PLs only (name populated).
*/
static int contraint_to_pl(struct rapl_domain *rd, int cid)
}
static int set_power_limit(struct powercap_zone *power_zone, int cid,
- u64 power_limit)
+ u64 power_limit)
{
struct rapl_domain *rd;
struct rapl_package *rp;
rp = rd->rp;
if (rd->state & DOMAIN_STATE_BIOS_LOCKED) {
- dev_warn(&power_zone->dev, "%s locked by BIOS, monitoring only\n",
- rd->name);
+ dev_warn(&power_zone->dev,
+ "%s locked by BIOS, monitoring only\n", rd->name);
ret = -EACCES;
goto set_exit;
}
}
static int get_current_power_limit(struct powercap_zone *power_zone, int cid,
- u64 *data)
+ u64 *data)
{
struct rapl_domain *rd;
u64 val;
}
static int set_time_window(struct powercap_zone *power_zone, int cid,
- u64 window)
+ u64 window)
{
struct rapl_domain *rd;
int ret = 0;
return ret;
}
-static int get_time_window(struct powercap_zone *power_zone, int cid, u64 *data)
+static int get_time_window(struct powercap_zone *power_zone, int cid,
+ u64 *data)
{
struct rapl_domain *rd;
u64 val;
return ret;
}
-static const char *get_constraint_name(struct powercap_zone *power_zone, int cid)
+static const char *get_constraint_name(struct powercap_zone *power_zone,
+ int cid)
{
struct rapl_domain *rd;
int id;
return NULL;
}
-
-static int get_max_power(struct powercap_zone *power_zone, int id,
- u64 *data)
+static int get_max_power(struct powercap_zone *power_zone, int id, u64 *data)
{
struct rapl_domain *rd;
u64 val;
for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
unsigned int mask = rp->domain_map & (1 << i);
- rd->regs[RAPL_DOMAIN_REG_LIMIT] = rp->priv->regs[i][RAPL_DOMAIN_REG_LIMIT];
- rd->regs[RAPL_DOMAIN_REG_STATUS] = rp->priv->regs[i][RAPL_DOMAIN_REG_STATUS];
- rd->regs[RAPL_DOMAIN_REG_PERF] = rp->priv->regs[i][RAPL_DOMAIN_REG_PERF];
- rd->regs[RAPL_DOMAIN_REG_POLICY] = rp->priv->regs[i][RAPL_DOMAIN_REG_POLICY];
- rd->regs[RAPL_DOMAIN_REG_INFO] = rp->priv->regs[i][RAPL_DOMAIN_REG_INFO];
+ rd->regs[RAPL_DOMAIN_REG_LIMIT] =
+ rp->priv->regs[i][RAPL_DOMAIN_REG_LIMIT];
+ rd->regs[RAPL_DOMAIN_REG_STATUS] =
+ rp->priv->regs[i][RAPL_DOMAIN_REG_STATUS];
+ rd->regs[RAPL_DOMAIN_REG_PERF] =
+ rp->priv->regs[i][RAPL_DOMAIN_REG_PERF];
+ rd->regs[RAPL_DOMAIN_REG_POLICY] =
+ rp->priv->regs[i][RAPL_DOMAIN_REG_POLICY];
+ rd->regs[RAPL_DOMAIN_REG_INFO] =
+ rp->priv->regs[i][RAPL_DOMAIN_REG_INFO];
switch (mask) {
case BIT(RAPL_DOMAIN_PACKAGE):
rd->rpl[0].prim_id = PL1_ENABLE;
rd->rpl[0].name = pl1_name;
rd->domain_energy_unit =
- rapl_defaults->dram_domain_energy_unit;
+ rapl_defaults->dram_domain_energy_unit;
if (rd->domain_energy_unit)
pr_info("DRAM domain energy unit %dpj\n",
rd->domain_energy_unit);
}
static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type,
- u64 value, int to_raw)
+ u64 value, int to_raw)
{
u64 units = 1;
struct rapl_package *rp = rd->rp;
static struct rapl_primitive_info rpi[] = {
/* name, mask, shift, msr index, unit divisor */
PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
- RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
+ RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0,
- RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
+ RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32,
- RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
+ RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
PRIMITIVE_INFO_INIT(FW_LOCK, POWER_PP_LOCK, 31,
- RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+ RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15,
- RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+ RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16,
- RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+ RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47,
- RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+ RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48,
- RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
+ RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17,
- RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
+ RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49,
- RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
+ RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK,
- 0, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
+ 0, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32,
- RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
+ RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16,
- RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
+ RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48,
- RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
+ RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,
- RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
+ RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0,
- RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0),
+ RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0),
/* non-hardware */
PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT,
- RAPL_PRIMITIVE_DERIVED),
+ RAPL_PRIMITIVE_DERIVED),
{NULL, 0, 0, 0},
};
* 63-------------------------- 31--------------------------- 0
*/
static int rapl_read_data_raw(struct rapl_domain *rd,
- enum rapl_primitives prim,
- bool xlate, u64 *data)
+ enum rapl_primitives prim, bool xlate, u64 *data)
{
u64 value;
struct rapl_primitive_info *rp = &rpi[prim];
cpu = rd->rp->lead_cpu;
- /* special-case package domain, which uses a different bit*/
+ /* special-case package domain, which uses a different bit */
if (prim == FW_LOCK && rd->id == RAPL_DOMAIN_PACKAGE) {
rp->mask = POWER_PACKAGE_LOCK;
rp->shift = 63;
/* Similar use of primitive info in the read counterpart */
static int rapl_write_data_raw(struct rapl_domain *rd,
- enum rapl_primitives prim,
- unsigned long long value)
+ enum rapl_primitives prim,
+ unsigned long long value)
{
struct rapl_primitive_info *rp = &rpi[prim];
int cpu;
ra.mask = ~0;
if (rp->priv->read_raw(cpu, &ra)) {
pr_err("Failed to read power unit REG 0x%x on CPU %d, exit.\n",
- rp->priv->reg_unit, cpu);
+ rp->priv->reg_unit, cpu);
return -ENODEV;
}
rp->time_unit = 1000000 / (1 << value);
pr_debug("Core CPU %s energy=%dpJ, time=%dus, power=%duW\n",
- rp->name, rp->energy_unit, rp->time_unit, rp->power_unit);
+ rp->name, rp->energy_unit, rp->time_unit, rp->power_unit);
return 0;
}
ra.mask = ~0;
if (rp->priv->read_raw(cpu, &ra)) {
pr_err("Failed to read power unit REG 0x%x on CPU %d, exit.\n",
- rp->priv->reg_unit, cpu);
+ rp->priv->reg_unit, cpu);
return -ENODEV;
}
rp->time_unit = 1000000 / (1 << value);
pr_debug("Atom %s energy=%dpJ, time=%dus, power=%duW\n",
- rp->name, rp->energy_unit, rp->time_unit, rp->power_unit);
+ rp->name, rp->energy_unit, rp->time_unit, rp->power_unit);
return 0;
}
wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
}
-
/* REVISIT:
* When package power limit is set artificially low by RAPL, LVT
* thermal interrupt for package power limit should be ignored
}
static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value,
- bool to_raw)
+ bool to_raw)
{
- u64 f, y; /* fraction and exp. used for time unit */
+ u64 f, y; /* fraction and exp. used for time unit */
/*
* Special processing based on 2^Y*(1+F/4), refer
}
static u64 rapl_compute_time_window_atom(struct rapl_package *rp, u64 value,
- bool to_raw)
+ bool to_raw)
{
/*
* Atom time unit encoding is straight forward val * time_unit,
*/
if (!to_raw)
return (value) ? value *= rp->time_unit : rp->time_unit;
- else
- value = div64_u64(value, rp->time_unit);
+
+ value = div64_u64(value, rp->time_unit);
return value;
}
};
static const struct x86_cpu_id rapl_ids[] __initconst = {
- INTEL_CPU_FAM6(SANDYBRIDGE, rapl_defaults_core),
- INTEL_CPU_FAM6(SANDYBRIDGE_X, rapl_defaults_core),
-
- INTEL_CPU_FAM6(IVYBRIDGE, rapl_defaults_core),
- INTEL_CPU_FAM6(IVYBRIDGE_X, rapl_defaults_core),
-
- INTEL_CPU_FAM6(HASWELL_CORE, rapl_defaults_core),
- INTEL_CPU_FAM6(HASWELL_ULT, rapl_defaults_core),
- INTEL_CPU_FAM6(HASWELL_GT3E, rapl_defaults_core),
- INTEL_CPU_FAM6(HASWELL_X, rapl_defaults_hsw_server),
-
- INTEL_CPU_FAM6(BROADWELL_CORE, rapl_defaults_core),
- INTEL_CPU_FAM6(BROADWELL_GT3E, rapl_defaults_core),
- INTEL_CPU_FAM6(BROADWELL_XEON_D, rapl_defaults_core),
- INTEL_CPU_FAM6(BROADWELL_X, rapl_defaults_hsw_server),
-
- INTEL_CPU_FAM6(SKYLAKE_DESKTOP, rapl_defaults_core),
- INTEL_CPU_FAM6(SKYLAKE_MOBILE, rapl_defaults_core),
- INTEL_CPU_FAM6(SKYLAKE_X, rapl_defaults_hsw_server),
- INTEL_CPU_FAM6(KABYLAKE_MOBILE, rapl_defaults_core),
- INTEL_CPU_FAM6(KABYLAKE_DESKTOP, rapl_defaults_core),
- INTEL_CPU_FAM6(CANNONLAKE_MOBILE, rapl_defaults_core),
- INTEL_CPU_FAM6(ICELAKE_MOBILE, rapl_defaults_core),
-
- INTEL_CPU_FAM6(ATOM_SILVERMONT, rapl_defaults_byt),
- INTEL_CPU_FAM6(ATOM_AIRMONT, rapl_defaults_cht),
- INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, rapl_defaults_tng),
- INTEL_CPU_FAM6(ATOM_AIRMONT_MID, rapl_defaults_ann),
- INTEL_CPU_FAM6(ATOM_GOLDMONT, rapl_defaults_core),
- INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, rapl_defaults_core),
- INTEL_CPU_FAM6(ATOM_GOLDMONT_X, rapl_defaults_core),
- INTEL_CPU_FAM6(ATOM_TREMONT_X, rapl_defaults_core),
-
- INTEL_CPU_FAM6(XEON_PHI_KNL, rapl_defaults_hsw_server),
- INTEL_CPU_FAM6(XEON_PHI_KNM, rapl_defaults_hsw_server),
+ INTEL_CPU_FAM6(SANDYBRIDGE, rapl_defaults_core),
+ INTEL_CPU_FAM6(SANDYBRIDGE_X, rapl_defaults_core),
+
+ INTEL_CPU_FAM6(IVYBRIDGE, rapl_defaults_core),
+ INTEL_CPU_FAM6(IVYBRIDGE_X, rapl_defaults_core),
+
+ INTEL_CPU_FAM6(HASWELL_CORE, rapl_defaults_core),
+ INTEL_CPU_FAM6(HASWELL_ULT, rapl_defaults_core),
+ INTEL_CPU_FAM6(HASWELL_GT3E, rapl_defaults_core),
+ INTEL_CPU_FAM6(HASWELL_X, rapl_defaults_hsw_server),
+
+ INTEL_CPU_FAM6(BROADWELL_CORE, rapl_defaults_core),
+ INTEL_CPU_FAM6(BROADWELL_GT3E, rapl_defaults_core),
+ INTEL_CPU_FAM6(BROADWELL_XEON_D, rapl_defaults_core),
+ INTEL_CPU_FAM6(BROADWELL_X, rapl_defaults_hsw_server),
+
+ INTEL_CPU_FAM6(SKYLAKE_DESKTOP, rapl_defaults_core),
+ INTEL_CPU_FAM6(SKYLAKE_MOBILE, rapl_defaults_core),
+ INTEL_CPU_FAM6(SKYLAKE_X, rapl_defaults_hsw_server),
+ INTEL_CPU_FAM6(KABYLAKE_MOBILE, rapl_defaults_core),
+ INTEL_CPU_FAM6(KABYLAKE_DESKTOP, rapl_defaults_core),
+ INTEL_CPU_FAM6(CANNONLAKE_MOBILE, rapl_defaults_core),
+ INTEL_CPU_FAM6(ICELAKE_MOBILE, rapl_defaults_core),
+
+ INTEL_CPU_FAM6(ATOM_SILVERMONT, rapl_defaults_byt),
+ INTEL_CPU_FAM6(ATOM_AIRMONT, rapl_defaults_cht),
+ INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, rapl_defaults_tng),
+ INTEL_CPU_FAM6(ATOM_AIRMONT_MID, rapl_defaults_ann),
+ INTEL_CPU_FAM6(ATOM_GOLDMONT, rapl_defaults_core),
+ INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, rapl_defaults_core),
+ INTEL_CPU_FAM6(ATOM_GOLDMONT_X, rapl_defaults_core),
+ INTEL_CPU_FAM6(ATOM_TREMONT_X, rapl_defaults_core),
+
+ INTEL_CPU_FAM6(XEON_PHI_KNL, rapl_defaults_hsw_server),
+ INTEL_CPU_FAM6(XEON_PHI_KNM, rapl_defaults_hsw_server),
{}
};
+
MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
/* Read once for all raw primitive data for domains */
for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {
if (!rapl_read_data_raw(&rp->domains[dmn], prim,
rpi[prim].unit, &val))
- rp->domains[dmn].rdd.primitives[prim] = val;
+ rp->domains[dmn].rdd.primitives[prim] = val;
}
}
/* Update the domain data of the new package */
rapl_update_domain_data(rp);
- /* first we register package domain as the parent zone*/
+ /* first we register package domain as the parent zone */
for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
if (rd->id == RAPL_DOMAIN_PACKAGE) {
nr_pl = find_nr_power_limit(rd);
pr_debug("register package domain %s\n", rp->name);
power_zone = powercap_register_zone(&rd->power_zone,
- rp->priv->control_type,
- rp->name, NULL,
- &zone_ops[rd->id],
- nr_pl,
- &constraint_ops);
+ rp->priv->control_type, rp->name,
+ NULL, &zone_ops[rd->id], nr_pl,
+ &constraint_ops);
if (IS_ERR(power_zone)) {
pr_debug("failed to register power zone %s\n",
- rp->name);
+ rp->name);
return PTR_ERR(power_zone);
}
/* track parent zone in per package/socket data */
pr_err("no package domain found, unknown topology!\n");
return -ENODEV;
}
- /* now register domains as children of the socket/package*/
+ /* now register domains as children of the socket/package */
for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
if (rd->id == RAPL_DOMAIN_PACKAGE)
continue;
/* number of power limits per domain varies */
nr_pl = find_nr_power_limit(rd);
power_zone = powercap_register_zone(&rd->power_zone,
- rp->priv->control_type, rd->name,
- rp->power_zone,
- &zone_ops[rd->id], nr_pl,
- &constraint_ops);
+ rp->priv->control_type,
+ rd->name, rp->power_zone,
+ &zone_ops[rd->id], nr_pl,
+ &constraint_ops);
if (IS_ERR(power_zone)) {
pr_debug("failed to register power_zone, %s:%s\n",
- rp->name, rd->name);
+ rp->name, rd->name);
ret = PTR_ERR(power_zone);
goto err_cleanup;
}
*/
while (--rd >= rp->domains) {
pr_debug("unregister %s domain %s\n", rp->name, rd->name);
- powercap_unregister_zone(rp->priv->control_type, &rd->power_zone);
+ powercap_unregister_zone(rp->priv->control_type,
+ &rd->power_zone);
}
return ret;
}
-static int __init rapl_add_platform_domain(struct rapl_if_priv *priv)
+int rapl_add_platform_domain(struct rapl_if_priv *priv)
{
struct rapl_domain *rd;
struct powercap_zone *power_zone;
rd->name = rapl_domain_names[RAPL_DOMAIN_PLATFORM];
rd->id = RAPL_DOMAIN_PLATFORM;
- rd->regs[RAPL_DOMAIN_REG_LIMIT] = priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_LIMIT];
- rd->regs[RAPL_DOMAIN_REG_STATUS] = priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_STATUS];
+ rd->regs[RAPL_DOMAIN_REG_LIMIT] =
+ priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_LIMIT];
+ rd->regs[RAPL_DOMAIN_REG_STATUS] =
+ priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_STATUS];
rd->rpl[0].prim_id = PL1_ENABLE;
rd->rpl[0].name = pl1_name;
rd->rpl[1].prim_id = PL2_ENABLE;
return 0;
}
+EXPORT_SYMBOL_GPL(rapl_add_platform_domain);
-static void rapl_remove_platform_domain(struct rapl_if_priv *priv)
+void rapl_remove_platform_domain(struct rapl_if_priv *priv)
{
if (priv->platform_rapl_domain) {
powercap_unregister_zone(priv->control_type,
- &priv->platform_rapl_domain->power_zone);
+ &priv->platform_rapl_domain->power_zone);
kfree(priv->platform_rapl_domain);
}
}
+EXPORT_SYMBOL_GPL(rapl_remove_platform_domain);
static int rapl_check_domain(int cpu, int domain, struct rapl_package *rp)
{
return 0;
}
-
/*
* Check if power limits are available. Two cases when they are not available:
* 1. Locked by BIOS, in this case we still provide read-only access so that
* users can see what limit is set by the BIOS.
* 2. Some CPUs make some domains monitoring only which means PLx MSRs may not
- * exist at all. In this case, we do not show the contraints in powercap.
+ * exist at all. In this case, we do not show the constraints in powercap.
*
* Called after domains are detected and initialized.
*/
rd->state |= DOMAIN_STATE_BIOS_LOCKED;
}
}
- /* check if power limit MSRs exists, otherwise domain is monitoring only */
+ /* check if power limit MSR exists, otherwise domain is monitoring only */
for (i = 0; i < NR_POWER_LIMITS; i++) {
int prim = rd->rpl[i].prim_id;
+
if (rapl_read_data_raw(rd, prim, false, &val64))
rd->rpl[i].name = NULL;
}
pr_info("Found RAPL domain %s\n", rapl_domain_names[i]);
}
}
- rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);
+ rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);
if (!rp->nr_domains) {
pr_debug("no valid rapl domains found in %s\n", rp->name);
return -ENODEV;
pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name);
rp->domains = kcalloc(rp->nr_domains + 1, sizeof(struct rapl_domain),
- GFP_KERNEL);
+ GFP_KERNEL);
if (!rp->domains)
return -ENOMEM;
}
/* called from CPU hotplug notifier, hotplug lock held */
-static void rapl_remove_package(struct rapl_package *rp)
+void rapl_remove_package(struct rapl_package *rp)
{
struct rapl_domain *rd, *rd_package = NULL;
}
pr_debug("remove package, undo power limit on %s: %s\n",
rp->name, rd->name);
- powercap_unregister_zone(rp->priv->control_type, &rd->power_zone);
+ powercap_unregister_zone(rp->priv->control_type,
+ &rd->power_zone);
}
/* do parent zone last */
- powercap_unregister_zone(rp->priv->control_type, &rd_package->power_zone);
+ powercap_unregister_zone(rp->priv->control_type,
+ &rd_package->power_zone);
list_del(&rp->plist);
+ if (list_empty(&rapl_packages))
+ rapl_remove_core();
kfree(rp);
}
+EXPORT_SYMBOL_GPL(rapl_remove_package);
+
+/* caller to ensure CPU hotplug lock is held */
+struct rapl_package *rapl_find_package_domain(int cpu, struct rapl_if_priv *priv)
+{
+ int id = topology_logical_die_id(cpu);
+ struct rapl_package *rp;
+
+ list_for_each_entry(rp, &rapl_packages, plist) {
+ if (rp->id == id
+ && rp->priv->control_type == priv->control_type)
+ return rp;
+ }
+
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(rapl_find_package_domain);
/* called from CPU hotplug notifier, hotplug lock held */
-static struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv)
+struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv)
{
int id = topology_logical_die_id(cpu);
struct rapl_package *rp;
struct cpuinfo_x86 *c = &cpu_data(cpu);
int ret;
+ ret = rapl_init_core();
+ if (ret)
+ return ERR_PTR(ret);
+
rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);
if (!rp)
return ERR_PTR(-ENOMEM);
if (topology_max_die_per_package() > 1)
snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH,
- "package-%d-die-%d", c->phys_proc_id, c->cpu_die_id);
+ "package-%d-die-%d", c->phys_proc_id, c->cpu_die_id);
else
snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d",
- c->phys_proc_id);
+ c->phys_proc_id);
/* check if the package contains valid domains */
- if (rapl_detect_domains(rp, cpu) ||
- rapl_defaults->check_unit(rp, cpu)) {
+ if (rapl_detect_domains(rp, cpu) || rapl_defaults->check_unit(rp, cpu)) {
ret = -ENODEV;
goto err_free_package;
}
kfree(rp);
return ERR_PTR(ret);
}
-
-/* Handles CPU hotplug on multi-socket systems.
- * If a CPU goes online as the first CPU of the physical package
- * we add the RAPL package to the system. Similarly, when the last
- * CPU of the package is removed, we remove the RAPL package and its
- * associated domains. Cooling devices are handled accordingly at
- * per-domain level.
- */
-static int rapl_cpu_online(unsigned int cpu)
-{
- struct rapl_package *rp;
-
- rp = rapl_find_package_domain(cpu, &rapl_msr_priv);
- if (!rp) {
- rp = rapl_add_package(cpu, &rapl_msr_priv);
- if (IS_ERR(rp))
- return PTR_ERR(rp);
- }
- cpumask_set_cpu(cpu, &rp->cpumask);
- return 0;
-}
-
-static int rapl_cpu_down_prep(unsigned int cpu)
-{
- struct rapl_package *rp;
- int lead_cpu;
-
- rp = rapl_find_package_domain(cpu, &rapl_msr_priv);
- if (!rp)
- return 0;
-
- cpumask_clear_cpu(cpu, &rp->cpumask);
- lead_cpu = cpumask_first(&rp->cpumask);
- if (lead_cpu >= nr_cpu_ids)
- rapl_remove_package(rp);
- else if (rp->lead_cpu == cpu)
- rp->lead_cpu = lead_cpu;
- return 0;
-}
+EXPORT_SYMBOL_GPL(rapl_add_package);
static void power_limit_state_save(void)
{
switch (rd->rpl[i].prim_id) {
case PL1_ENABLE:
ret = rapl_read_data_raw(rd,
- POWER_LIMIT1,
- true,
- &rd->rpl[i].last_power_limit);
+ POWER_LIMIT1, true,
+ &rd->rpl[i].last_power_limit);
if (ret)
rd->rpl[i].last_power_limit = 0;
break;
case PL2_ENABLE:
ret = rapl_read_data_raw(rd,
- POWER_LIMIT2,
- true,
- &rd->rpl[i].last_power_limit);
+ POWER_LIMIT2, true,
+ &rd->rpl[i].last_power_limit);
if (ret)
rd->rpl[i].last_power_limit = 0;
break;
switch (rd->rpl[i].prim_id) {
case PL1_ENABLE:
if (rd->rpl[i].last_power_limit)
- rapl_write_data_raw(rd,
- POWER_LIMIT1,
- rd->rpl[i].last_power_limit);
+ rapl_write_data_raw(rd, POWER_LIMIT1,
+ rd->rpl[i].last_power_limit);
break;
case PL2_ENABLE:
if (rd->rpl[i].last_power_limit)
- rapl_write_data_raw(rd,
- POWER_LIMIT2,
- rd->rpl[i].last_power_limit);
+ rapl_write_data_raw(rd, POWER_LIMIT2,
+ rd->rpl[i].last_power_limit);
break;
}
}
}
static int rapl_pm_callback(struct notifier_block *nb,
- unsigned long mode, void *_unused)
+ unsigned long mode, void *_unused)
{
switch (mode) {
case PM_SUSPEND_PREPARE:
.notifier_call = rapl_pm_callback,
};
-static int rapl_msr_read_raw(int cpu, struct reg_action *ra)
-{
- if (rdmsrl_safe_on_cpu(cpu, ra->reg, &ra->value)) {
- pr_debug("failed to read msr 0x%x on cpu %d\n", ra->reg, cpu);
- return -EIO;
- }
- ra->value &= ra->mask;
- return 0;
-}
-
-static void rapl_msr_update_func(void *info)
-{
- struct reg_action *ra = info;
- u64 val;
-
- ra->err = rdmsrl_safe(ra->reg, &val);
- if (ra->err)
- return;
-
- val &= ~ra->mask;
- val |= ra->value;
-
- ra->err = wrmsrl_safe(ra->reg, val);
-}
-
-
-static int rapl_msr_write_raw(int cpu, struct reg_action *ra)
-{
- int ret;
-
- ret = smp_call_function_single(cpu, rapl_msr_update_func, ra, 1);
- if (WARN_ON_ONCE(ret))
- return ret;
-
- return ra->err;
-}
-
-static int __init rapl_init(void)
+static int rapl_init_core(void)
{
const struct x86_cpu_id *id;
int ret;
+ if (rapl_defaults)
+ return 0;
+
id = x86_match_cpu(rapl_ids);
if (!id) {
pr_err("driver does not support CPU family %d model %d\n",
- boot_cpu_data.x86, boot_cpu_data.x86_model);
+ boot_cpu_data.x86, boot_cpu_data.x86_model);
return -ENODEV;
}
rapl_defaults = (struct rapl_defaults *)id->driver_data;
- rapl_msr_priv.read_raw = rapl_msr_read_raw;
- rapl_msr_priv.write_raw = rapl_msr_write_raw;
-
- rapl_msr_priv.control_type = powercap_register_control_type(NULL, "intel-rapl", NULL);
- if (IS_ERR(rapl_msr_priv.control_type)) {
- pr_debug("failed to register powercap control_type.\n");
- return PTR_ERR(rapl_msr_priv.control_type);
- }
-
- ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powercap/rapl:online",
- rapl_cpu_online, rapl_cpu_down_prep);
- if (ret < 0)
- goto err_unreg;
- rapl_msr_priv.pcap_rapl_online = ret;
-
- /* Don't bail out if PSys is not supported */
- rapl_add_platform_domain(&rapl_msr_priv);
-
ret = register_pm_notifier(&rapl_pm_notifier);
- if (ret)
- goto err_unreg_all;
return 0;
-
-err_unreg_all:
- cpuhp_remove_state(rapl_msr_priv.pcap_rapl_online);
-
-err_unreg:
- powercap_unregister_control_type(rapl_msr_priv.control_type);
- return ret;
}
-static void __exit rapl_exit(void)
+static void rapl_remove_core(void)
{
unregister_pm_notifier(&rapl_pm_notifier);
- cpuhp_remove_state(rapl_msr_priv.pcap_rapl_online);
- rapl_remove_platform_domain(&rapl_msr_priv);
- powercap_unregister_control_type(rapl_msr_priv.control_type);
+ rapl_defaults = NULL;
}
-module_init(rapl_init);
-module_exit(rapl_exit);
-
-MODULE_DESCRIPTION("Driver for Intel RAPL (Running Average Power Limit)");
+MODULE_DESCRIPTION("Intel Runtime Average Power Limit (RAPL) common code");
MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>");
MODULE_LICENSE("GPL v2");