drivers/perf/arm_cspmu/nvidia_cspmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
   4  *
   5  */
   6
   7 /* Support for NVIDIA specific attributes. */
   8
   9 #include <linux/topology.h>
  10
  11 #include "nvidia_cspmu.h"
  12
  13 #define NV_PCIE_PORT_COUNT           10ULL
  14 #define NV_PCIE_FILTER_ID_MASK       GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0)
  15
  16 #define NV_NVL_C2C_PORT_COUNT        2ULL
  17 #define NV_NVL_C2C_FILTER_ID_MASK    GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0)
  18
  19 #define NV_CNVL_PORT_COUNT           4ULL
  20 #define NV_CNVL_FILTER_ID_MASK       GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0)
  21
  22 #define NV_GENERIC_FILTER_ID_MASK    GENMASK_ULL(31, 0)
  23
  24 #define NV_PRODID_MASK               GENMASK(31, 0)
  25
  26 #define NV_FORMAT_NAME_GENERIC  0
  27
  28 #define to_nv_cspmu_ctx(cspmu)  ((struct nv_cspmu_ctx *)(cspmu->impl.ctx))
  29
  30 #define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config)        \
  31         ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config)
  32
  33 #define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config)                    \
  34         NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config),        \
  35         NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1),    \
  36         NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2),    \
  37         NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3)
  38
  39 struct nv_cspmu_ctx {
  40         const char *name;
  41         u32 filter_mask;
  42         u32 filter_default_val;
  43         struct attribute **event_attr;
  44         struct attribute **format_attr;
  45 };
  46
  47 static struct attribute *scf_pmu_event_attrs[] = {
  48         ARM_CSPMU_EVENT_ATTR(bus_cycles,                        0x1d),
  49
  50         ARM_CSPMU_EVENT_ATTR(scf_cache_allocate,                0xF0),
  51         ARM_CSPMU_EVENT_ATTR(scf_cache_refill,                  0xF1),
  52         ARM_CSPMU_EVENT_ATTR(scf_cache,                         0xF2),
  53         ARM_CSPMU_EVENT_ATTR(scf_cache_wb,                      0xF3),
  54
  55         NV_CSPMU_EVENT_ATTR_4(socket, rd_data,                  0x101),
  56         NV_CSPMU_EVENT_ATTR_4(socket, dl_rsp,                   0x105),
  57         NV_CSPMU_EVENT_ATTR_4(socket, wb_data,                  0x109),
  58         NV_CSPMU_EVENT_ATTR_4(socket, ev_rsp,                   0x10d),
  59         NV_CSPMU_EVENT_ATTR_4(socket, prb_data,                 0x111),
  60
  61         NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding,           0x115),
  62         NV_CSPMU_EVENT_ATTR_4(socket, dl_outstanding,           0x119),
  63         NV_CSPMU_EVENT_ATTR_4(socket, wb_outstanding,           0x11d),
  64         NV_CSPMU_EVENT_ATTR_4(socket, wr_outstanding,           0x121),
  65         NV_CSPMU_EVENT_ATTR_4(socket, ev_outstanding,           0x125),
  66         NV_CSPMU_EVENT_ATTR_4(socket, prb_outstanding,          0x129),
  67
  68         NV_CSPMU_EVENT_ATTR_4(socket, rd_access,                0x12d),
  69         NV_CSPMU_EVENT_ATTR_4(socket, dl_access,                0x131),
  70         NV_CSPMU_EVENT_ATTR_4(socket, wb_access,                0x135),
  71         NV_CSPMU_EVENT_ATTR_4(socket, wr_access,                0x139),
  72         NV_CSPMU_EVENT_ATTR_4(socket, ev_access,                0x13d),
  73         NV_CSPMU_EVENT_ATTR_4(socket, prb_access,               0x141),
  74
  75         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_data,                0x145),
  76         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_access,              0x149),
  77         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_access,              0x14d),
  78         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_outstanding,         0x151),
  79         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_outstanding,         0x155),
  80
  81         NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_data,                 0x159),
  82         NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_access,               0x15d),
  83         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_access,               0x161),
  84         NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_outstanding,          0x165),
  85         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_outstanding,          0x169),
  86
  87         ARM_CSPMU_EVENT_ATTR(gmem_rd_data,                      0x16d),
  88         ARM_CSPMU_EVENT_ATTR(gmem_rd_access,                    0x16e),
  89         ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding,               0x16f),
  90         ARM_CSPMU_EVENT_ATTR(gmem_dl_rsp,                       0x170),
  91         ARM_CSPMU_EVENT_ATTR(gmem_dl_access,                    0x171),
  92         ARM_CSPMU_EVENT_ATTR(gmem_dl_outstanding,               0x172),
  93         ARM_CSPMU_EVENT_ATTR(gmem_wb_data,                      0x173),
  94         ARM_CSPMU_EVENT_ATTR(gmem_wb_access,                    0x174),
  95         ARM_CSPMU_EVENT_ATTR(gmem_wb_outstanding,               0x175),
  96         ARM_CSPMU_EVENT_ATTR(gmem_ev_rsp,                       0x176),
  97         ARM_CSPMU_EVENT_ATTR(gmem_ev_access,                    0x177),
  98         ARM_CSPMU_EVENT_ATTR(gmem_ev_outstanding,               0x178),
  99         ARM_CSPMU_EVENT_ATTR(gmem_wr_data,                      0x179),
 100         ARM_CSPMU_EVENT_ATTR(gmem_wr_outstanding,               0x17a),
 101         ARM_CSPMU_EVENT_ATTR(gmem_wr_access,                    0x17b),
 102
 103         NV_CSPMU_EVENT_ATTR_4(socket, wr_data,                  0x17c),
 104
 105         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_data,                0x180),
 106         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_data,                0x184),
 107         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_access,              0x188),
 108         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_outstanding,         0x18c),
 109
 110         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_data,                 0x190),
 111         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_data,                 0x194),
 112         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_access,               0x198),
 113         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_outstanding,          0x19c),
 114
 115         ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes,               0x1a0),
 116         ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes,      0x1a1),
 117         ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data,             0x1a2),
 118         ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding,      0x1a3),
 119         ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access,           0x1a4),
 120
 121         ARM_CSPMU_EVENT_ATTR(cmem_rd_data,                      0x1a5),
 122         ARM_CSPMU_EVENT_ATTR(cmem_rd_access,                    0x1a6),
 123         ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding,               0x1a7),
 124         ARM_CSPMU_EVENT_ATTR(cmem_dl_rsp,                       0x1a8),
 125         ARM_CSPMU_EVENT_ATTR(cmem_dl_access,                    0x1a9),
 126         ARM_CSPMU_EVENT_ATTR(cmem_dl_outstanding,               0x1aa),
 127         ARM_CSPMU_EVENT_ATTR(cmem_wb_data,                      0x1ab),
 128         ARM_CSPMU_EVENT_ATTR(cmem_wb_access,                    0x1ac),
 129         ARM_CSPMU_EVENT_ATTR(cmem_wb_outstanding,               0x1ad),
 130         ARM_CSPMU_EVENT_ATTR(cmem_ev_rsp,                       0x1ae),
 131         ARM_CSPMU_EVENT_ATTR(cmem_ev_access,                    0x1af),
 132         ARM_CSPMU_EVENT_ATTR(cmem_ev_outstanding,               0x1b0),
 133         ARM_CSPMU_EVENT_ATTR(cmem_wr_data,                      0x1b1),
 134         ARM_CSPMU_EVENT_ATTR(cmem_wr_outstanding,               0x1b2),
 135
 136         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_data,                0x1b3),
 137         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_access,              0x1b7),
 138         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_access,              0x1bb),
 139         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_outstanding,         0x1bf),
 140         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_outstanding,         0x1c3),
 141
 142         ARM_CSPMU_EVENT_ATTR(ocu_prb_access,                    0x1c7),
 143         ARM_CSPMU_EVENT_ATTR(ocu_prb_data,                      0x1c8),
 144         ARM_CSPMU_EVENT_ATTR(ocu_prb_outstanding,               0x1c9),
 145
 146         ARM_CSPMU_EVENT_ATTR(cmem_wr_access,                    0x1ca),
 147
 148         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_access,              0x1cb),
 149         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_data,                0x1cf),
 150         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_data,                0x1d3),
 151         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_outstanding,         0x1d7),
 152
 153         ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes,               0x1db),
 154
 155         ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
 156         NULL,
 157 };
 158
 159 static struct attribute *mcf_pmu_event_attrs[] = {
 160         ARM_CSPMU_EVENT_ATTR(rd_bytes_loc,                      0x0),
 161         ARM_CSPMU_EVENT_ATTR(rd_bytes_rem,                      0x1),
 162         ARM_CSPMU_EVENT_ATTR(wr_bytes_loc,                      0x2),
 163         ARM_CSPMU_EVENT_ATTR(wr_bytes_rem,                      0x3),
 164         ARM_CSPMU_EVENT_ATTR(total_bytes_loc,                   0x4),
 165         ARM_CSPMU_EVENT_ATTR(total_bytes_rem,                   0x5),
 166         ARM_CSPMU_EVENT_ATTR(rd_req_loc,                        0x6),
 167         ARM_CSPMU_EVENT_ATTR(rd_req_rem,                        0x7),
 168         ARM_CSPMU_EVENT_ATTR(wr_req_loc,                        0x8),
 169         ARM_CSPMU_EVENT_ATTR(wr_req_rem,                        0x9),
 170         ARM_CSPMU_EVENT_ATTR(total_req_loc,                     0xa),
 171         ARM_CSPMU_EVENT_ATTR(total_req_rem,                     0xb),
 172         ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc,                   0xc),
 173         ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem,                   0xd),
 174         ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
 175         NULL,
 176 };
 177
 178 static struct attribute *generic_pmu_event_attrs[] = {
 179         ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
 180         NULL,
 181 };
 182
 183 static struct attribute *scf_pmu_format_attrs[] = {
 184         ARM_CSPMU_FORMAT_EVENT_ATTR,
 185         NULL,
 186 };
 187
 188 static struct attribute *pcie_pmu_format_attrs[] = {
 189         ARM_CSPMU_FORMAT_EVENT_ATTR,
 190         ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"),
 191         NULL,
 192 };
 193
 194 static struct attribute *nvlink_c2c_pmu_format_attrs[] = {
 195         ARM_CSPMU_FORMAT_EVENT_ATTR,
 196         NULL,
 197 };
 198
 199 static struct attribute *cnvlink_pmu_format_attrs[] = {
 200         ARM_CSPMU_FORMAT_EVENT_ATTR,
 201         ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"),
 202         NULL,
 203 };
 204
 205 static struct attribute *generic_pmu_format_attrs[] = {
 206         ARM_CSPMU_FORMAT_EVENT_ATTR,
 207         ARM_CSPMU_FORMAT_FILTER_ATTR,
 208         NULL,
 209 };
 210
 211 static struct attribute **
 212 nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
 213 {
 214         const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
 215
 216         return ctx->event_attr;
 217 }
 218
 219 static struct attribute **
 220 nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
 221 {
 222         const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
 223
 224         return ctx->format_attr;
 225 }
 226
 227 static const char *
 228 nv_cspmu_get_name(const struct arm_cspmu *cspmu)
 229 {
 230         const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
 231
 232         return ctx->name;
 233 }
 234
 235 static u32 nv_cspmu_event_filter(const struct perf_event *event)
 236 {
 237         const struct nv_cspmu_ctx *ctx =
 238                 to_nv_cspmu_ctx(to_arm_cspmu(event->pmu));
 239
 240         if (ctx->filter_mask == 0)
 241                 return ctx->filter_default_val;
 242
 243         return event->attr.config1 & ctx->filter_mask;
 244 }
 245
 246 enum nv_cspmu_name_fmt {
 247         NAME_FMT_GENERIC,
 248         NAME_FMT_SOCKET
 249 };
 250
 251 struct nv_cspmu_match {
 252         u32 prodid;
 253         u32 prodid_mask;
 254         u64 filter_mask;
 255         u32 filter_default_val;
 256         const char *name_pattern;
 257         enum nv_cspmu_name_fmt name_fmt;
 258         struct attribute **event_attr;
 259         struct attribute **format_attr;
 260 };
 261
 262 static const struct nv_cspmu_match nv_cspmu_match[] = {
 263         {
 264           .prodid = 0x103,
 265           .prodid_mask = NV_PRODID_MASK,
 266           .filter_mask = NV_PCIE_FILTER_ID_MASK,
 267           .filter_default_val = NV_PCIE_FILTER_ID_MASK,
 268           .name_pattern = "nvidia_pcie_pmu_%u",
 269           .name_fmt = NAME_FMT_SOCKET,
 270           .event_attr = mcf_pmu_event_attrs,
 271           .format_attr = pcie_pmu_format_attrs
 272         },
 273         {
 274           .prodid = 0x104,
 275           .prodid_mask = NV_PRODID_MASK,
 276           .filter_mask = 0x0,
 277           .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
 278           .name_pattern = "nvidia_nvlink_c2c1_pmu_%u",
 279           .name_fmt = NAME_FMT_SOCKET,
 280           .event_attr = mcf_pmu_event_attrs,
 281           .format_attr = nvlink_c2c_pmu_format_attrs
 282         },
 283         {
 284           .prodid = 0x105,
 285           .prodid_mask = NV_PRODID_MASK,
 286           .filter_mask = 0x0,
 287           .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
 288           .name_pattern = "nvidia_nvlink_c2c0_pmu_%u",
 289           .name_fmt = NAME_FMT_SOCKET,
 290           .event_attr = mcf_pmu_event_attrs,
 291           .format_attr = nvlink_c2c_pmu_format_attrs
 292         },
 293         {
 294           .prodid = 0x106,
 295           .prodid_mask = NV_PRODID_MASK,
 296           .filter_mask = NV_CNVL_FILTER_ID_MASK,
 297           .filter_default_val = NV_CNVL_FILTER_ID_MASK,
 298           .name_pattern = "nvidia_cnvlink_pmu_%u",
 299           .name_fmt = NAME_FMT_SOCKET,
 300           .event_attr = mcf_pmu_event_attrs,
 301           .format_attr = cnvlink_pmu_format_attrs
 302         },
 303         {
 304           .prodid = 0x2CF,
 305           .prodid_mask = NV_PRODID_MASK,
 306           .filter_mask = 0x0,
 307           .filter_default_val = 0x0,
 308           .name_pattern = "nvidia_scf_pmu_%u",
 309           .name_fmt = NAME_FMT_SOCKET,
 310           .event_attr = scf_pmu_event_attrs,
 311           .format_attr = scf_pmu_format_attrs
 312         },
 313         {
 314           .prodid = 0,
 315           .prodid_mask = 0,
 316           .filter_mask = NV_GENERIC_FILTER_ID_MASK,
 317           .filter_default_val = NV_GENERIC_FILTER_ID_MASK,
 318           .name_pattern = "nvidia_uncore_pmu_%u",
 319           .name_fmt = NAME_FMT_GENERIC,
 320           .event_attr = generic_pmu_event_attrs,
 321           .format_attr = generic_pmu_format_attrs
 322         },
 323 };
 324
 325 static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu,
 326                                   const struct nv_cspmu_match *match)
 327 {
 328         char *name;
 329         struct device *dev = cspmu->dev;
 330
 331         static atomic_t pmu_generic_idx = {0};
 332
 333         switch (match->name_fmt) {
 334         case NAME_FMT_SOCKET: {
 335                 const int cpu = cpumask_first(&cspmu->associated_cpus);
 336                 const int socket = cpu_to_node(cpu);
 337
 338                 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
 339                                        socket);
 340                 break;
 341         }
 342         case NAME_FMT_GENERIC:
 343                 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
 344                                        atomic_fetch_inc(&pmu_generic_idx));
 345                 break;
 346         default:
 347                 name = NULL;
 348                 break;
 349         }
 350
 351         return name;
 352 }
 353
 354 int nv_cspmu_init_ops(struct arm_cspmu *cspmu)
 355 {
 356         u32 prodid;
 357         struct nv_cspmu_ctx *ctx;
 358         struct device *dev = cspmu->dev;
 359         struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
 360         const struct nv_cspmu_match *match = nv_cspmu_match;
 361
 362         ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL);
 363         if (!ctx)
 364                 return -ENOMEM;
 365
 366         prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr);
 367
 368         /* Find matching PMU. */
 369         for (; match->prodid; match++) {
 370                 const u32 prodid_mask = match->prodid_mask;
 371
 372                 if ((match->prodid & prodid_mask) == (prodid & prodid_mask))
 373                         break;
 374         }
 375
 376         ctx->name               = nv_cspmu_format_name(cspmu, match);
 377         ctx->filter_mask        = match->filter_mask;
 378         ctx->filter_default_val = match->filter_default_val;
 379         ctx->event_attr         = match->event_attr;
 380         ctx->format_attr        = match->format_attr;
 381
 382         cspmu->impl.ctx = ctx;
 383
 384         /* NVIDIA specific callbacks. */
 385         impl_ops->event_filter                  = nv_cspmu_event_filter;
 386         impl_ops->get_event_attrs               = nv_cspmu_get_event_attrs;
 387         impl_ops->get_format_attrs              = nv_cspmu_get_format_attrs;
 388         impl_ops->get_name                      = nv_cspmu_get_name;
 389
 390         /* Set others to NULL to use default callback. */
 391         impl_ops->event_type                    = NULL;
 392         impl_ops->event_attr_is_visible         = NULL;
 393         impl_ops->get_identifier                = NULL;
 394         impl_ops->is_cycle_counter_event        = NULL;
 395
 396         return 0;
 397 }
 398 EXPORT_SYMBOL_GPL(nv_cspmu_init_ops);
 399
 400 MODULE_LICENSE("GPL v2");