arch/x86/kernel/cpu/perf_event_intel_uncore.c

   1 #include "perf_event_intel_uncore.h"
   2
   3 static struct intel_uncore_type *empty_uncore[] = { NULL, };
   4 static struct intel_uncore_type **msr_uncores = empty_uncore;
   5 static struct intel_uncore_type **pci_uncores = empty_uncore;
   6 /* pci bus to socket mapping */
   7 static int pcibus_to_physid[256] = { [0 ... 255] = -1, };
   8
   9 static DEFINE_RAW_SPINLOCK(uncore_box_lock);
  10
  11 /* mask of cpus that collect uncore events */
  12 static cpumask_t uncore_cpu_mask;
  13
  14 /* constraint for the fixed counter */
  15 static struct event_constraint constraint_fixed =
  16         EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL);
  17
  18 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
  19 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
  20 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
  21 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
  22 DEFINE_UNCORE_FORMAT_ATTR(cmask5, cmask, "config:24-28");
  23 DEFINE_UNCORE_FORMAT_ATTR(cmask8, cmask, "config:24-31");
  24 DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31");
  25 DEFINE_UNCORE_FORMAT_ATTR(thresh5, thresh, "config:24-28");
  26 DEFINE_UNCORE_FORMAT_ATTR(occ_sel, occ_sel, "config:14-15");
  27 DEFINE_UNCORE_FORMAT_ATTR(occ_invert, occ_invert, "config:30");
  28 DEFINE_UNCORE_FORMAT_ATTR(occ_edge, occ_edge, "config:14-51");
  29
  30 /* Sandy Bridge-EP uncore support */
  31 static void snbep_uncore_pci_disable_box(struct intel_uncore_box *box)
  32 {
  33         struct pci_dev *pdev = box->pci_dev;
  34         int box_ctl = uncore_pci_box_ctl(box);
  35         u32 config;
  36
  37         pci_read_config_dword(pdev, box_ctl, &config);
  38         config |= SNBEP_PMON_BOX_CTL_FRZ;
  39         pci_write_config_dword(pdev, box_ctl, config);
  40 }
  41
  42 static void snbep_uncore_pci_enable_box(struct intel_uncore_box *box)
  43 {
  44         struct pci_dev *pdev = box->pci_dev;
  45         int box_ctl = uncore_pci_box_ctl(box);
  46         u32 config;
  47
  48         pci_read_config_dword(pdev, box_ctl, &config);
  49         config &= ~SNBEP_PMON_BOX_CTL_FRZ;
  50         pci_write_config_dword(pdev, box_ctl, config);
  51 }
  52
  53 static void snbep_uncore_pci_enable_event(struct intel_uncore_box *box,
  54                                         struct perf_event *event)
  55 {
  56         struct pci_dev *pdev = box->pci_dev;
  57         struct hw_perf_event *hwc = &event->hw;
  58
  59         pci_write_config_dword(pdev, hwc->config_base, hwc->config |
  60                                 SNBEP_PMON_CTL_EN);
  61 }
  62
  63 static void snbep_uncore_pci_disable_event(struct intel_uncore_box *box,
  64                                         struct perf_event *event)
  65 {
  66         struct pci_dev *pdev = box->pci_dev;
  67         struct hw_perf_event *hwc = &event->hw;
  68
  69         pci_write_config_dword(pdev, hwc->config_base, hwc->config);
  70 }
  71
  72 static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box,
  73                                         struct perf_event *event)
  74 {
  75         struct pci_dev *pdev = box->pci_dev;
  76         struct hw_perf_event *hwc = &event->hw;
  77         u64 count;
  78
  79         pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
  80         pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
  81         return count;
  82 }
  83
  84 static void snbep_uncore_pci_init_box(struct intel_uncore_box *box)
  85 {
  86         struct pci_dev *pdev = box->pci_dev;
  87         pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL,
  88                                 SNBEP_PMON_BOX_CTL_INT);
  89 }
  90
  91 static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box)
  92 {
  93         u64 config;
  94         unsigned msr;
  95
  96         msr = uncore_msr_box_ctl(box);
  97         if (msr) {
  98                 rdmsrl(msr, config);
  99                 config |= SNBEP_PMON_BOX_CTL_FRZ;
 100                 wrmsrl(msr, config);
 101                 return;
 102         }
 103 }
 104
 105 static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box)
 106 {
 107         u64 config;
 108         unsigned msr;
 109
 110         msr = uncore_msr_box_ctl(box);
 111         if (msr) {
 112                 rdmsrl(msr, config);
 113                 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
 114                 wrmsrl(msr, config);
 115                 return;
 116         }
 117 }
 118
 119 static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box,
 120                                         struct perf_event *event)
 121 {
 122         struct hw_perf_event *hwc = &event->hw;
 123
 124         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 125 }
 126
 127 static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box,
 128                                         struct perf_event *event)
 129 {
 130         struct hw_perf_event *hwc = &event->hw;
 131
 132         wrmsrl(hwc->config_base, hwc->config);
 133 }
 134
 135 static u64 snbep_uncore_msr_read_counter(struct intel_uncore_box *box,
 136                                         struct perf_event *event)
 137 {
 138         struct hw_perf_event *hwc = &event->hw;
 139         u64 count;
 140
 141         rdmsrl(hwc->event_base, count);
 142         return count;
 143 }
 144
 145 static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
 146 {
 147         unsigned msr = uncore_msr_box_ctl(box);
 148         if (msr)
 149                 wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT);
 150 }
 151
 152 static struct attribute *snbep_uncore_formats_attr[] = {
 153         &format_attr_event.attr,
 154         &format_attr_umask.attr,
 155         &format_attr_edge.attr,
 156         &format_attr_inv.attr,
 157         &format_attr_thresh8.attr,
 158         NULL,
 159 };
 160
 161 static struct attribute *snbep_uncore_ubox_formats_attr[] = {
 162         &format_attr_event.attr,
 163         &format_attr_umask.attr,
 164         &format_attr_edge.attr,
 165         &format_attr_inv.attr,
 166         &format_attr_thresh5.attr,
 167         NULL,
 168 };
 169
 170 static struct attribute *snbep_uncore_pcu_formats_attr[] = {
 171         &format_attr_event.attr,
 172         &format_attr_occ_sel.attr,
 173         &format_attr_edge.attr,
 174         &format_attr_inv.attr,
 175         &format_attr_thresh5.attr,
 176         &format_attr_occ_invert.attr,
 177         &format_attr_occ_edge.attr,
 178         NULL,
 179 };
 180
 181 static struct uncore_event_desc snbep_uncore_imc_events[] = {
 182         INTEL_UNCORE_EVENT_DESC(clockticks,      "event=0xff,umask=0xff"),
 183         INTEL_UNCORE_EVENT_DESC(cas_count_read,  "event=0x04,umask=0x03"),
 184         INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"),
 185         { /* end: all zeroes */ },
 186 };
 187
 188 static struct uncore_event_desc snbep_uncore_qpi_events[] = {
 189         INTEL_UNCORE_EVENT_DESC(clockticks,       "event=0x14"),
 190         INTEL_UNCORE_EVENT_DESC(txl_flits_active, "event=0x00,umask=0x06"),
 191         INTEL_UNCORE_EVENT_DESC(drs_data,         "event=0x02,umask=0x08"),
 192         INTEL_UNCORE_EVENT_DESC(ncb_data,         "event=0x03,umask=0x04"),
 193         { /* end: all zeroes */ },
 194 };
 195
 196 static struct attribute_group snbep_uncore_format_group = {
 197         .name = "format",
 198         .attrs = snbep_uncore_formats_attr,
 199 };
 200
 201 static struct attribute_group snbep_uncore_ubox_format_group = {
 202         .name = "format",
 203         .attrs = snbep_uncore_ubox_formats_attr,
 204 };
 205
 206 static struct attribute_group snbep_uncore_pcu_format_group = {
 207         .name = "format",
 208         .attrs = snbep_uncore_pcu_formats_attr,
 209 };
 210
 211 static struct intel_uncore_ops snbep_uncore_msr_ops = {
 212         .init_box       = snbep_uncore_msr_init_box,
 213         .disable_box    = snbep_uncore_msr_disable_box,
 214         .enable_box     = snbep_uncore_msr_enable_box,
 215         .disable_event  = snbep_uncore_msr_disable_event,
 216         .enable_event   = snbep_uncore_msr_enable_event,
 217         .read_counter   = snbep_uncore_msr_read_counter,
 218 };
 219
 220 static struct intel_uncore_ops snbep_uncore_pci_ops = {
 221         .init_box       = snbep_uncore_pci_init_box,
 222         .disable_box    = snbep_uncore_pci_disable_box,
 223         .enable_box     = snbep_uncore_pci_enable_box,
 224         .disable_event  = snbep_uncore_pci_disable_event,
 225         .enable_event   = snbep_uncore_pci_enable_event,
 226         .read_counter   = snbep_uncore_pci_read_counter,
 227 };
 228
 229 static struct event_constraint snbep_uncore_cbox_constraints[] = {
 230         UNCORE_EVENT_CONSTRAINT(0x01, 0x1),
 231         UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
 232         UNCORE_EVENT_CONSTRAINT(0x04, 0x3),
 233         UNCORE_EVENT_CONSTRAINT(0x05, 0x3),
 234         UNCORE_EVENT_CONSTRAINT(0x07, 0x3),
 235         UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
 236         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
 237         UNCORE_EVENT_CONSTRAINT(0x13, 0x3),
 238         UNCORE_EVENT_CONSTRAINT(0x1b, 0xc),
 239         UNCORE_EVENT_CONSTRAINT(0x1c, 0xc),
 240         UNCORE_EVENT_CONSTRAINT(0x1d, 0xc),
 241         UNCORE_EVENT_CONSTRAINT(0x1e, 0xc),
 242         UNCORE_EVENT_CONSTRAINT(0x1f, 0xe),
 243         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
 244         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 245         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
 246         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 247         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 248         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 249         UNCORE_EVENT_CONSTRAINT(0x35, 0x3),
 250         UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
 251         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
 252         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
 253         UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
 254         UNCORE_EVENT_CONSTRAINT(0x3b, 0x1),
 255         EVENT_CONSTRAINT_END
 256 };
 257
 258 static struct event_constraint snbep_uncore_r2pcie_constraints[] = {
 259         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
 260         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
 261         UNCORE_EVENT_CONSTRAINT(0x12, 0x1),
 262         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 263         UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
 264         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
 265         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
 266         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 267         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 268         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 269         EVENT_CONSTRAINT_END
 270 };
 271
 272 static struct event_constraint snbep_uncore_r3qpi_constraints[] = {
 273         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
 274         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
 275         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
 276         UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
 277         UNCORE_EVENT_CONSTRAINT(0x20, 0x3),
 278         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
 279         UNCORE_EVENT_CONSTRAINT(0x22, 0x3),
 280         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 281         UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
 282         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
 283         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
 284         UNCORE_EVENT_CONSTRAINT(0x30, 0x3),
 285         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
 286         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 287         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 288         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 289         UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
 290         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
 291         EVENT_CONSTRAINT_END
 292 };
 293
 294 static struct intel_uncore_type snbep_uncore_ubox = {
 295         .name           = "ubox",
 296         .num_counters   = 2,
 297         .num_boxes      = 1,
 298         .perf_ctr_bits  = 44,
 299         .fixed_ctr_bits = 48,
 300         .perf_ctr       = SNBEP_U_MSR_PMON_CTR0,
 301         .event_ctl      = SNBEP_U_MSR_PMON_CTL0,
 302         .event_mask     = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
 303         .fixed_ctr      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
 304         .fixed_ctl      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
 305         .ops            = &snbep_uncore_msr_ops,
 306         .format_group   = &snbep_uncore_ubox_format_group,
 307 };
 308
 309 static struct intel_uncore_type snbep_uncore_cbox = {
 310         .name           = "cbox",
 311         .num_counters   = 4,
 312         .num_boxes      = 8,
 313         .perf_ctr_bits  = 44,
 314         .event_ctl      = SNBEP_C0_MSR_PMON_CTL0,
 315         .perf_ctr       = SNBEP_C0_MSR_PMON_CTR0,
 316         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
 317         .box_ctl        = SNBEP_C0_MSR_PMON_BOX_CTL,
 318         .msr_offset     = SNBEP_CBO_MSR_OFFSET,
 319         .constraints    = snbep_uncore_cbox_constraints,
 320         .ops            = &snbep_uncore_msr_ops,
 321         .format_group   = &snbep_uncore_format_group,
 322 };
 323
 324 static struct intel_uncore_type snbep_uncore_pcu = {
 325         .name           = "pcu",
 326         .num_counters   = 4,
 327         .num_boxes      = 1,
 328         .perf_ctr_bits  = 48,
 329         .perf_ctr       = SNBEP_PCU_MSR_PMON_CTR0,
 330         .event_ctl      = SNBEP_PCU_MSR_PMON_CTL0,
 331         .event_mask     = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
 332         .box_ctl        = SNBEP_PCU_MSR_PMON_BOX_CTL,
 333         .ops            = &snbep_uncore_msr_ops,
 334         .format_group   = &snbep_uncore_pcu_format_group,
 335 };
 336
 337 static struct intel_uncore_type *snbep_msr_uncores[] = {
 338         &snbep_uncore_ubox,
 339         &snbep_uncore_cbox,
 340         &snbep_uncore_pcu,
 341         NULL,
 342 };
 343
 344 #define SNBEP_UNCORE_PCI_COMMON_INIT()                          \
 345         .perf_ctr       = SNBEP_PCI_PMON_CTR0,                  \
 346         .event_ctl      = SNBEP_PCI_PMON_CTL0,                  \
 347         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,            \
 348         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,               \
 349         .ops            = &snbep_uncore_pci_ops,                \
 350         .format_group   = &snbep_uncore_format_group
 351
 352 static struct intel_uncore_type snbep_uncore_ha = {
 353         .name           = "ha",
 354         .num_counters   = 4,
 355         .num_boxes      = 1,
 356         .perf_ctr_bits  = 48,
 357         SNBEP_UNCORE_PCI_COMMON_INIT(),
 358 };
 359
 360 static struct intel_uncore_type snbep_uncore_imc = {
 361         .name           = "imc",
 362         .num_counters   = 4,
 363         .num_boxes      = 4,
 364         .perf_ctr_bits  = 48,
 365         .fixed_ctr_bits = 48,
 366         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
 367         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
 368         .event_descs    = snbep_uncore_imc_events,
 369         SNBEP_UNCORE_PCI_COMMON_INIT(),
 370 };
 371
 372 static struct intel_uncore_type snbep_uncore_qpi = {
 373         .name           = "qpi",
 374         .num_counters   = 4,
 375         .num_boxes      = 2,
 376         .perf_ctr_bits  = 48,
 377         .event_descs    = snbep_uncore_qpi_events,
 378         SNBEP_UNCORE_PCI_COMMON_INIT(),
 379 };
 380
 381
 382 static struct intel_uncore_type snbep_uncore_r2pcie = {
 383         .name           = "r2pcie",
 384         .num_counters   = 4,
 385         .num_boxes      = 1,
 386         .perf_ctr_bits  = 44,
 387         .constraints    = snbep_uncore_r2pcie_constraints,
 388         SNBEP_UNCORE_PCI_COMMON_INIT(),
 389 };
 390
 391 static struct intel_uncore_type snbep_uncore_r3qpi = {
 392         .name           = "r3qpi",
 393         .num_counters   = 3,
 394         .num_boxes      = 2,
 395         .perf_ctr_bits  = 44,
 396         .constraints    = snbep_uncore_r3qpi_constraints,
 397         SNBEP_UNCORE_PCI_COMMON_INIT(),
 398 };
 399
 400 static struct intel_uncore_type *snbep_pci_uncores[] = {
 401         &snbep_uncore_ha,
 402         &snbep_uncore_imc,
 403         &snbep_uncore_qpi,
 404         &snbep_uncore_r2pcie,
 405         &snbep_uncore_r3qpi,
 406         NULL,
 407 };
 408
 409 static DEFINE_PCI_DEVICE_TABLE(snbep_uncore_pci_ids) = {
 410         { /* Home Agent */
 411                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA),
 412                 .driver_data = (unsigned long)&snbep_uncore_ha,
 413         },
 414         { /* MC Channel 0 */
 415                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0),
 416                 .driver_data = (unsigned long)&snbep_uncore_imc,
 417         },
 418         { /* MC Channel 1 */
 419                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1),
 420                 .driver_data = (unsigned long)&snbep_uncore_imc,
 421         },
 422         { /* MC Channel 2 */
 423                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2),
 424                 .driver_data = (unsigned long)&snbep_uncore_imc,
 425         },
 426         { /* MC Channel 3 */
 427                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3),
 428                 .driver_data = (unsigned long)&snbep_uncore_imc,
 429         },
 430         { /* QPI Port 0 */
 431                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0),
 432                 .driver_data = (unsigned long)&snbep_uncore_qpi,
 433         },
 434         { /* QPI Port 1 */
 435                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1),
 436                 .driver_data = (unsigned long)&snbep_uncore_qpi,
 437         },
 438         { /* P2PCIe */
 439                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE),
 440                 .driver_data = (unsigned long)&snbep_uncore_r2pcie,
 441         },
 442         { /* R3QPI Link 0 */
 443                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0),
 444                 .driver_data = (unsigned long)&snbep_uncore_r3qpi,
 445         },
 446         { /* R3QPI Link 1 */
 447                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1),
 448                 .driver_data = (unsigned long)&snbep_uncore_r3qpi,
 449         },
 450         { /* end: all zeroes */ }
 451 };
 452
 453 static struct pci_driver snbep_uncore_pci_driver = {
 454         .name           = "snbep_uncore",
 455         .id_table       = snbep_uncore_pci_ids,
 456 };
 457
 458 /*
 459  * build pci bus to socket mapping
 460  */
 461 static void snbep_pci2phy_map_init(void)
 462 {
 463         struct pci_dev *ubox_dev = NULL;
 464         int i, bus, nodeid;
 465         u32 config;
 466
 467         while (1) {
 468                 /* find the UBOX device */
 469                 ubox_dev = pci_get_device(PCI_VENDOR_ID_INTEL,
 470                                         PCI_DEVICE_ID_INTEL_JAKETOWN_UBOX,
 471                                         ubox_dev);
 472                 if (!ubox_dev)
 473                         break;
 474                 bus = ubox_dev->bus->number;
 475                 /* get the Node ID of the local register */
 476                 pci_read_config_dword(ubox_dev, 0x40, &config);
 477                 nodeid = config;
 478                 /* get the Node ID mapping */
 479                 pci_read_config_dword(ubox_dev, 0x54, &config);
 480                 /*
 481                  * every three bits in the Node ID mapping register maps
 482                  * to a particular node.
 483                  */
 484                 for (i = 0; i < 8; i++) {
 485                         if (nodeid == ((config >> (3 * i)) & 0x7)) {
 486                                 pcibus_to_physid[bus] = i;
 487                                 break;
 488                         }
 489                 }
 490         };
 491         return;
 492 }
 493 /* end of Sandy Bridge-EP uncore support */
 494
 495
 496 /* Sandy Bridge uncore support */
 497 static void snb_uncore_msr_enable_event(struct intel_uncore_box *box,
 498                                         struct perf_event *event)
 499 {
 500         struct hw_perf_event *hwc = &event->hw;
 501
 502         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
 503                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
 504         else
 505                 wrmsrl(hwc->config_base, SNB_UNC_CTL_EN);
 506 }
 507
 508 static void snb_uncore_msr_disable_event(struct intel_uncore_box *box,
 509                                         struct perf_event *event)
 510 {
 511         wrmsrl(event->hw.config_base, 0);
 512 }
 513
 514 static u64 snb_uncore_msr_read_counter(struct intel_uncore_box *box,
 515                                         struct perf_event *event)
 516 {
 517         u64 count;
 518         rdmsrl(event->hw.event_base, count);
 519         return count;
 520 }
 521
 522 static void snb_uncore_msr_init_box(struct intel_uncore_box *box)
 523 {
 524         if (box->pmu->pmu_idx == 0) {
 525                 wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
 526                         SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
 527         }
 528 }
 529
 530 static struct attribute *snb_uncore_formats_attr[] = {
 531         &format_attr_event.attr,
 532         &format_attr_umask.attr,
 533         &format_attr_edge.attr,
 534         &format_attr_inv.attr,
 535         &format_attr_cmask5.attr,
 536         NULL,
 537 };
 538
 539 static struct attribute_group snb_uncore_format_group = {
 540         .name = "format",
 541         .attrs = snb_uncore_formats_attr,
 542 };
 543
 544 static struct intel_uncore_ops snb_uncore_msr_ops = {
 545         .init_box       = snb_uncore_msr_init_box,
 546         .disable_event  = snb_uncore_msr_disable_event,
 547         .enable_event   = snb_uncore_msr_enable_event,
 548         .read_counter   = snb_uncore_msr_read_counter,
 549 };
 550
 551 static struct event_constraint snb_uncore_cbox_constraints[] = {
 552         UNCORE_EVENT_CONSTRAINT(0x80, 0x1),
 553         UNCORE_EVENT_CONSTRAINT(0x83, 0x1),
 554         EVENT_CONSTRAINT_END
 555 };
 556
 557 static struct intel_uncore_type snb_uncore_cbox = {
 558         .name           = "cbox",
 559         .num_counters   = 2,
 560         .num_boxes      = 4,
 561         .perf_ctr_bits  = 44,
 562         .fixed_ctr_bits = 48,
 563         .perf_ctr       = SNB_UNC_CBO_0_PER_CTR0,
 564         .event_ctl      = SNB_UNC_CBO_0_PERFEVTSEL0,
 565         .fixed_ctr      = SNB_UNC_FIXED_CTR,
 566         .fixed_ctl      = SNB_UNC_FIXED_CTR_CTRL,
 567         .single_fixed   = 1,
 568         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
 569         .msr_offset     = SNB_UNC_CBO_MSR_OFFSET,
 570         .constraints    = snb_uncore_cbox_constraints,
 571         .ops            = &snb_uncore_msr_ops,
 572         .format_group   = &snb_uncore_format_group,
 573 };
 574
 575 static struct intel_uncore_type *snb_msr_uncores[] = {
 576         &snb_uncore_cbox,
 577         NULL,
 578 };
 579 /* end of Sandy Bridge uncore support */
 580
 581 /* Nehalem uncore support */
 582 static void nhm_uncore_msr_disable_box(struct intel_uncore_box *box)
 583 {
 584         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, 0);
 585 }
 586
 587 static void nhm_uncore_msr_enable_box(struct intel_uncore_box *box)
 588 {
 589         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL,
 590                 NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC);
 591 }
 592
 593 static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box,
 594                                         struct perf_event *event)
 595 {
 596         struct hw_perf_event *hwc = &event->hw;
 597
 598         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
 599                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
 600         else
 601                 wrmsrl(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN);
 602 }
 603
 604 static struct attribute *nhm_uncore_formats_attr[] = {
 605         &format_attr_event.attr,
 606         &format_attr_umask.attr,
 607         &format_attr_edge.attr,
 608         &format_attr_inv.attr,
 609         &format_attr_cmask8.attr,
 610         NULL,
 611 };
 612
 613 static struct attribute_group nhm_uncore_format_group = {
 614         .name = "format",
 615         .attrs = nhm_uncore_formats_attr,
 616 };
 617
 618 static struct uncore_event_desc nhm_uncore_events[] = {
 619         INTEL_UNCORE_EVENT_DESC(clockticks,                "event=0xff,umask=0xff"),
 620         INTEL_UNCORE_EVENT_DESC(qmc_writes_full_any,       "event=0x2f,umask=0x0f"),
 621         INTEL_UNCORE_EVENT_DESC(qmc_normal_reads_any,      "event=0x2c,umask=0x0f"),
 622         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_reads,     "event=0x20,umask=0x01"),
 623         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_writes,    "event=0x20,umask=0x02"),
 624         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_reads,  "event=0x20,umask=0x04"),
 625         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_writes, "event=0x20,umask=0x08"),
 626         INTEL_UNCORE_EVENT_DESC(qhl_request_local_reads,   "event=0x20,umask=0x10"),
 627         INTEL_UNCORE_EVENT_DESC(qhl_request_local_writes,  "event=0x20,umask=0x20"),
 628         { /* end: all zeroes */ },
 629 };
 630
 631 static struct intel_uncore_ops nhm_uncore_msr_ops = {
 632         .disable_box    = nhm_uncore_msr_disable_box,
 633         .enable_box     = nhm_uncore_msr_enable_box,
 634         .disable_event  = snb_uncore_msr_disable_event,
 635         .enable_event   = nhm_uncore_msr_enable_event,
 636         .read_counter   = snb_uncore_msr_read_counter,
 637 };
 638
 639 static struct intel_uncore_type nhm_uncore = {
 640         .name           = "",
 641         .num_counters   = 8,
 642         .num_boxes      = 1,
 643         .perf_ctr_bits  = 48,
 644         .fixed_ctr_bits = 48,
 645         .event_ctl      = NHM_UNC_PERFEVTSEL0,
 646         .perf_ctr       = NHM_UNC_UNCORE_PMC0,
 647         .fixed_ctr      = NHM_UNC_FIXED_CTR,
 648         .fixed_ctl      = NHM_UNC_FIXED_CTR_CTRL,
 649         .event_mask     = NHM_UNC_RAW_EVENT_MASK,
 650         .event_descs    = nhm_uncore_events,
 651         .ops            = &nhm_uncore_msr_ops,
 652         .format_group   = &nhm_uncore_format_group,
 653 };
 654
 655 static struct intel_uncore_type *nhm_msr_uncores[] = {
 656         &nhm_uncore,
 657         NULL,
 658 };
 659 /* end of Nehalem uncore support */
 660
 661 static void uncore_assign_hw_event(struct intel_uncore_box *box,
 662                                 struct perf_event *event, int idx)
 663 {
 664         struct hw_perf_event *hwc = &event->hw;
 665
 666         hwc->idx = idx;
 667         hwc->last_tag = ++box->tags[idx];
 668
 669         if (hwc->idx == UNCORE_PMC_IDX_FIXED) {
 670                 hwc->event_base = uncore_fixed_ctr(box);
 671                 hwc->config_base = uncore_fixed_ctl(box);
 672                 return;
 673         }
 674
 675         hwc->config_base = uncore_event_ctl(box, hwc->idx);
 676         hwc->event_base  = uncore_perf_ctr(box, hwc->idx);
 677 }
 678
 679 static void uncore_perf_event_update(struct intel_uncore_box *box,
 680                                         struct perf_event *event)
 681 {
 682         u64 prev_count, new_count, delta;
 683         int shift;
 684
 685         if (event->hw.idx >= UNCORE_PMC_IDX_FIXED)
 686                 shift = 64 - uncore_fixed_ctr_bits(box);
 687         else
 688                 shift = 64 - uncore_perf_ctr_bits(box);
 689
 690         /* the hrtimer might modify the previous event value */
 691 again:
 692         prev_count = local64_read(&event->hw.prev_count);
 693         new_count = uncore_read_counter(box, event);
 694         if (local64_xchg(&event->hw.prev_count, new_count) != prev_count)
 695                 goto again;
 696
 697         delta = (new_count << shift) - (prev_count << shift);
 698         delta >>= shift;
 699
 700         local64_add(delta, &event->count);
 701 }
 702
 703 /*
 704  * The overflow interrupt is unavailable for SandyBridge-EP, is broken
 705  * for SandyBridge. So we use hrtimer to periodically poll the counter
 706  * to avoid overflow.
 707  */
 708 static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
 709 {
 710         struct intel_uncore_box *box;
 711         unsigned long flags;
 712         int bit;
 713
 714         box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
 715         if (!box->n_active || box->cpu != smp_processor_id())
 716                 return HRTIMER_NORESTART;
 717         /*
 718          * disable local interrupt to prevent uncore_pmu_event_start/stop
 719          * to interrupt the update process
 720          */
 721         local_irq_save(flags);
 722
 723         for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
 724                 uncore_perf_event_update(box, box->events[bit]);
 725
 726         local_irq_restore(flags);
 727
 728         hrtimer_forward_now(hrtimer, ns_to_ktime(UNCORE_PMU_HRTIMER_INTERVAL));
 729         return HRTIMER_RESTART;
 730 }
 731
 732 static void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
 733 {
 734         __hrtimer_start_range_ns(&box->hrtimer,
 735                         ns_to_ktime(UNCORE_PMU_HRTIMER_INTERVAL), 0,
 736                         HRTIMER_MODE_REL_PINNED, 0);
 737 }
 738
 739 static void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
 740 {
 741         hrtimer_cancel(&box->hrtimer);
 742 }
 743
 744 static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
 745 {
 746         hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 747         box->hrtimer.function = uncore_pmu_hrtimer;
 748 }
 749
 750 struct intel_uncore_box *uncore_alloc_box(int cpu)
 751 {
 752         struct intel_uncore_box *box;
 753
 754         box = kmalloc_node(sizeof(*box), GFP_KERNEL | __GFP_ZERO,
 755                            cpu_to_node(cpu));
 756         if (!box)
 757                 return NULL;
 758
 759         uncore_pmu_init_hrtimer(box);
 760         atomic_set(&box->refcnt, 1);
 761         box->cpu = -1;
 762         box->phys_id = -1;
 763
 764         return box;
 765 }
 766
 767 static struct intel_uncore_box *
 768 uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
 769 {
 770         static struct intel_uncore_box *box;
 771
 772         box = *per_cpu_ptr(pmu->box, cpu);
 773         if (box)
 774                 return box;
 775
 776         raw_spin_lock(&uncore_box_lock);
 777         list_for_each_entry(box, &pmu->box_list, list) {
 778                 if (box->phys_id == topology_physical_package_id(cpu)) {
 779                         atomic_inc(&box->refcnt);
 780                         *per_cpu_ptr(pmu->box, cpu) = box;
 781                         break;
 782                 }
 783         }
 784         raw_spin_unlock(&uncore_box_lock);
 785
 786         return *per_cpu_ptr(pmu->box, cpu);
 787 }
 788
 789 static struct intel_uncore_pmu *uncore_event_to_pmu(struct perf_event *event)
 790 {
 791         return container_of(event->pmu, struct intel_uncore_pmu, pmu);
 792 }
 793
 794 static struct intel_uncore_box *uncore_event_to_box(struct perf_event *event)
 795 {
 796         /*
 797          * perf core schedules event on the basis of cpu, uncore events are
 798          * collected by one of the cpus inside a physical package.
 799          */
 800         return uncore_pmu_to_box(uncore_event_to_pmu(event),
 801                                  smp_processor_id());
 802 }
 803
 804 static int uncore_collect_events(struct intel_uncore_box *box,
 805                                 struct perf_event *leader, bool dogrp)
 806 {
 807         struct perf_event *event;
 808         int n, max_count;
 809
 810         max_count = box->pmu->type->num_counters;
 811         if (box->pmu->type->fixed_ctl)
 812                 max_count++;
 813
 814         if (box->n_events >= max_count)
 815                 return -EINVAL;
 816
 817         n = box->n_events;
 818         box->event_list[n] = leader;
 819         n++;
 820         if (!dogrp)
 821                 return n;
 822
 823         list_for_each_entry(event, &leader->sibling_list, group_entry) {
 824                 if (event->state <= PERF_EVENT_STATE_OFF)
 825                         continue;
 826
 827                 if (n >= max_count)
 828                         return -EINVAL;
 829
 830                 box->event_list[n] = event;
 831                 n++;
 832         }
 833         return n;
 834 }
 835
 836 static struct event_constraint *
 837 uncore_event_constraint(struct intel_uncore_type *type,
 838                         struct perf_event *event)
 839 {
 840         struct event_constraint *c;
 841
 842         if (event->hw.config == ~0ULL)
 843                 return &constraint_fixed;
 844
 845         if (type->constraints) {
 846                 for_each_event_constraint(c, type->constraints) {
 847                         if ((event->hw.config & c->cmask) == c->code)
 848                                 return c;
 849                 }
 850         }
 851
 852         return &type->unconstrainted;
 853 }
 854
 855 static int uncore_assign_events(struct intel_uncore_box *box,
 856                                 int assign[], int n)
 857 {
 858         unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
 859         struct event_constraint *c, *constraints[UNCORE_PMC_IDX_MAX];
 860         int i, ret, wmin, wmax;
 861         struct hw_perf_event *hwc;
 862
 863         bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX);
 864
 865         for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) {
 866                 c = uncore_event_constraint(box->pmu->type,
 867                                 box->event_list[i]);
 868                 constraints[i] = c;
 869                 wmin = min(wmin, c->weight);
 870                 wmax = max(wmax, c->weight);
 871         }
 872
 873         /* fastpath, try to reuse previous register */
 874         for (i = 0; i < n; i++) {
 875                 hwc = &box->event_list[i]->hw;
 876                 c = constraints[i];
 877
 878                 /* never assigned */
 879                 if (hwc->idx == -1)
 880                         break;
 881
 882                 /* constraint still honored */
 883                 if (!test_bit(hwc->idx, c->idxmsk))
 884                         break;
 885
 886                 /* not already used */
 887                 if (test_bit(hwc->idx, used_mask))
 888                         break;
 889
 890                 __set_bit(hwc->idx, used_mask);
 891                 assign[i] = hwc->idx;
 892         }
 893         if (i == n)
 894                 return 0;
 895
 896         /* slow path */
 897         ret = perf_assign_events(constraints, n, wmin, wmax, assign);
 898         return ret ? -EINVAL : 0;
 899 }
 900
 901 static void uncore_pmu_event_start(struct perf_event *event, int flags)
 902 {
 903         struct intel_uncore_box *box = uncore_event_to_box(event);
 904         int idx = event->hw.idx;
 905
 906         if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
 907                 return;
 908
 909         if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX))
 910                 return;
 911
 912         event->hw.state = 0;
 913         box->events[idx] = event;
 914         box->n_active++;
 915         __set_bit(idx, box->active_mask);
 916
 917         local64_set(&event->hw.prev_count, uncore_read_counter(box, event));
 918         uncore_enable_event(box, event);
 919
 920         if (box->n_active == 1) {
 921                 uncore_enable_box(box);
 922                 uncore_pmu_start_hrtimer(box);
 923         }
 924 }
 925
 926 static void uncore_pmu_event_stop(struct perf_event *event, int flags)
 927 {
 928         struct intel_uncore_box *box = uncore_event_to_box(event);
 929         struct hw_perf_event *hwc = &event->hw;
 930
 931         if (__test_and_clear_bit(hwc->idx, box->active_mask)) {
 932                 uncore_disable_event(box, event);
 933                 box->n_active--;
 934                 box->events[hwc->idx] = NULL;
 935                 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
 936                 hwc->state |= PERF_HES_STOPPED;
 937
 938                 if (box->n_active == 0) {
 939                         uncore_disable_box(box);
 940                         uncore_pmu_cancel_hrtimer(box);
 941                 }
 942         }
 943
 944         if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
 945                 /*
 946                  * Drain the remaining delta count out of a event
 947                  * that we are disabling:
 948                  */
 949                 uncore_perf_event_update(box, event);
 950                 hwc->state |= PERF_HES_UPTODATE;
 951         }
 952 }
 953
 954 static int uncore_pmu_event_add(struct perf_event *event, int flags)
 955 {
 956         struct intel_uncore_box *box = uncore_event_to_box(event);
 957         struct hw_perf_event *hwc = &event->hw;
 958         int assign[UNCORE_PMC_IDX_MAX];
 959         int i, n, ret;
 960
 961         if (!box)
 962                 return -ENODEV;
 963
 964         ret = n = uncore_collect_events(box, event, false);
 965         if (ret < 0)
 966                 return ret;
 967
 968         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 969         if (!(flags & PERF_EF_START))
 970                 hwc->state |= PERF_HES_ARCH;
 971
 972         ret = uncore_assign_events(box, assign, n);
 973         if (ret)
 974                 return ret;
 975
 976         /* save events moving to new counters */
 977         for (i = 0; i < box->n_events; i++) {
 978                 event = box->event_list[i];
 979                 hwc = &event->hw;
 980
 981                 if (hwc->idx == assign[i] &&
 982                         hwc->last_tag == box->tags[assign[i]])
 983                         continue;
 984                 /*
 985                  * Ensure we don't accidentally enable a stopped
 986                  * counter simply because we rescheduled.
 987                  */
 988                 if (hwc->state & PERF_HES_STOPPED)
 989                         hwc->state |= PERF_HES_ARCH;
 990
 991                 uncore_pmu_event_stop(event, PERF_EF_UPDATE);
 992         }
 993
 994         /* reprogram moved events into new counters */
 995         for (i = 0; i < n; i++) {
 996                 event = box->event_list[i];
 997                 hwc = &event->hw;
 998
 999                 if (hwc->idx != assign[i] ||
1000                         hwc->last_tag != box->tags[assign[i]])
1001                         uncore_assign_hw_event(box, event, assign[i]);
1002                 else if (i < box->n_events)
1003                         continue;
1004
1005                 if (hwc->state & PERF_HES_ARCH)
1006                         continue;
1007
1008                 uncore_pmu_event_start(event, 0);
1009         }
1010         box->n_events = n;
1011
1012         return 0;
1013 }
1014
1015 static void uncore_pmu_event_del(struct perf_event *event, int flags)
1016 {
1017         struct intel_uncore_box *box = uncore_event_to_box(event);
1018         int i;
1019
1020         uncore_pmu_event_stop(event, PERF_EF_UPDATE);
1021
1022         for (i = 0; i < box->n_events; i++) {
1023                 if (event == box->event_list[i]) {
1024                         while (++i < box->n_events)
1025                                 box->event_list[i - 1] = box->event_list[i];
1026
1027                         --box->n_events;
1028                         break;
1029                 }
1030         }
1031
1032         event->hw.idx = -1;
1033         event->hw.last_tag = ~0ULL;
1034 }
1035
1036 static void uncore_pmu_event_read(struct perf_event *event)
1037 {
1038         struct intel_uncore_box *box = uncore_event_to_box(event);
1039         uncore_perf_event_update(box, event);
1040 }
1041
1042 /*
1043  * validation ensures the group can be loaded onto the
1044  * PMU if it was the only group available.
1045  */
1046 static int uncore_validate_group(struct intel_uncore_pmu *pmu,
1047                                 struct perf_event *event)
1048 {
1049         struct perf_event *leader = event->group_leader;
1050         struct intel_uncore_box *fake_box;
1051         int assign[UNCORE_PMC_IDX_MAX];
1052         int ret = -EINVAL, n;
1053
1054         fake_box = uncore_alloc_box(smp_processor_id());
1055         if (!fake_box)
1056                 return -ENOMEM;
1057
1058         fake_box->pmu = pmu;
1059         /*
1060          * the event is not yet connected with its
1061          * siblings therefore we must first collect
1062          * existing siblings, then add the new event
1063          * before we can simulate the scheduling
1064          */
1065         n = uncore_collect_events(fake_box, leader, true);
1066         if (n < 0)
1067                 goto out;
1068
1069         fake_box->n_events = n;
1070         n = uncore_collect_events(fake_box, event, false);
1071         if (n < 0)
1072                 goto out;
1073
1074         fake_box->n_events = n;
1075
1076         ret = uncore_assign_events(fake_box, assign, n);
1077 out:
1078         kfree(fake_box);
1079         return ret;
1080 }
1081
1082 int uncore_pmu_event_init(struct perf_event *event)
1083 {
1084         struct intel_uncore_pmu *pmu;
1085         struct intel_uncore_box *box;
1086         struct hw_perf_event *hwc = &event->hw;
1087         int ret;
1088
1089         if (event->attr.type != event->pmu->type)
1090                 return -ENOENT;
1091
1092         pmu = uncore_event_to_pmu(event);
1093         /* no device found for this pmu */
1094         if (pmu->func_id < 0)
1095                 return -ENOENT;
1096
1097         /*
1098          * Uncore PMU does measure at all privilege level all the time.
1099          * So it doesn't make sense to specify any exclude bits.
1100          */
1101         if (event->attr.exclude_user || event->attr.exclude_kernel ||
1102                         event->attr.exclude_hv || event->attr.exclude_idle)
1103                 return -EINVAL;
1104
1105         /* Sampling not supported yet */
1106         if (hwc->sample_period)
1107                 return -EINVAL;
1108
1109         /*
1110          * Place all uncore events for a particular physical package
1111          * onto a single cpu
1112          */
1113         if (event->cpu < 0)
1114                 return -EINVAL;
1115         box = uncore_pmu_to_box(pmu, event->cpu);
1116         if (!box || box->cpu < 0)
1117                 return -EINVAL;
1118         event->cpu = box->cpu;
1119
1120         if (event->attr.config == UNCORE_FIXED_EVENT) {
1121                 /* no fixed counter */
1122                 if (!pmu->type->fixed_ctl)
1123                         return -EINVAL;
1124                 /*
1125                  * if there is only one fixed counter, only the first pmu
1126                  * can access the fixed counter
1127                  */
1128                 if (pmu->type->single_fixed && pmu->pmu_idx > 0)
1129                         return -EINVAL;
1130                 hwc->config = ~0ULL;
1131         } else {
1132                 hwc->config = event->attr.config & pmu->type->event_mask;
1133         }
1134
1135         event->hw.idx = -1;
1136         event->hw.last_tag = ~0ULL;
1137
1138         if (event->group_leader != event)
1139                 ret = uncore_validate_group(pmu, event);
1140         else
1141                 ret = 0;
1142
1143         return ret;
1144 }
1145
1146 static int __init uncore_pmu_register(struct intel_uncore_pmu *pmu)
1147 {
1148         int ret;
1149
1150         pmu->pmu = (struct pmu) {
1151                 .attr_groups    = pmu->type->attr_groups,
1152                 .task_ctx_nr    = perf_invalid_context,
1153                 .event_init     = uncore_pmu_event_init,
1154                 .add            = uncore_pmu_event_add,
1155                 .del            = uncore_pmu_event_del,
1156                 .start          = uncore_pmu_event_start,
1157                 .stop           = uncore_pmu_event_stop,
1158                 .read           = uncore_pmu_event_read,
1159         };
1160
1161         if (pmu->type->num_boxes == 1) {
1162                 if (strlen(pmu->type->name) > 0)
1163                         sprintf(pmu->name, "uncore_%s", pmu->type->name);
1164                 else
1165                         sprintf(pmu->name, "uncore");
1166         } else {
1167                 sprintf(pmu->name, "uncore_%s_%d", pmu->type->name,
1168                         pmu->pmu_idx);
1169         }
1170
1171         ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
1172         return ret;
1173 }
1174
1175 static void __init uncore_type_exit(struct intel_uncore_type *type)
1176 {
1177         int i;
1178
1179         for (i = 0; i < type->num_boxes; i++)
1180                 free_percpu(type->pmus[i].box);
1181         kfree(type->pmus);
1182         type->pmus = NULL;
1183         kfree(type->attr_groups[1]);
1184         type->attr_groups[1] = NULL;
1185 }
1186
1187 static void uncore_types_exit(struct intel_uncore_type **types)
1188 {
1189         int i;
1190         for (i = 0; types[i]; i++)
1191                 uncore_type_exit(types[i]);
1192 }
1193
1194 static int __init uncore_type_init(struct intel_uncore_type *type)
1195 {
1196         struct intel_uncore_pmu *pmus;
1197         struct attribute_group *events_group;
1198         struct attribute **attrs;
1199         int i, j;
1200
1201         pmus = kzalloc(sizeof(*pmus) * type->num_boxes, GFP_KERNEL);
1202         if (!pmus)
1203                 return -ENOMEM;
1204
1205         type->unconstrainted = (struct event_constraint)
1206                 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
1207                                 0, type->num_counters, 0);
1208
1209         for (i = 0; i < type->num_boxes; i++) {
1210                 pmus[i].func_id = -1;
1211                 pmus[i].pmu_idx = i;
1212                 pmus[i].type = type;
1213                 INIT_LIST_HEAD(&pmus[i].box_list);
1214                 pmus[i].box = alloc_percpu(struct intel_uncore_box *);
1215                 if (!pmus[i].box)
1216                         goto fail;
1217         }
1218
1219         if (type->event_descs) {
1220                 i = 0;
1221                 while (type->event_descs[i].attr.attr.name)
1222                         i++;
1223
1224                 events_group = kzalloc(sizeof(struct attribute *) * (i + 1) +
1225                                         sizeof(*events_group), GFP_KERNEL);
1226                 if (!events_group)
1227                         goto fail;
1228
1229                 attrs = (struct attribute **)(events_group + 1);
1230                 events_group->name = "events";
1231                 events_group->attrs = attrs;
1232
1233                 for (j = 0; j < i; j++)
1234                         attrs[j] = &type->event_descs[j].attr.attr;
1235
1236                 type->attr_groups[1] = events_group;
1237         }
1238
1239         type->pmus = pmus;
1240         return 0;
1241 fail:
1242         uncore_type_exit(type);
1243         return -ENOMEM;
1244 }
1245
1246 static int __init uncore_types_init(struct intel_uncore_type **types)
1247 {
1248         int i, ret;
1249
1250         for (i = 0; types[i]; i++) {
1251                 ret = uncore_type_init(types[i]);
1252                 if (ret)
1253                         goto fail;
1254         }
1255         return 0;
1256 fail:
1257         while (--i >= 0)
1258                 uncore_type_exit(types[i]);
1259         return ret;
1260 }
1261
1262 static struct pci_driver *uncore_pci_driver;
1263 static bool pcidrv_registered;
1264
1265 /*
1266  * add a pci uncore device
1267  */
1268 static int __devinit uncore_pci_add(struct intel_uncore_type *type,
1269                                     struct pci_dev *pdev)
1270 {
1271         struct intel_uncore_pmu *pmu;
1272         struct intel_uncore_box *box;
1273         int i, phys_id;
1274
1275         phys_id = pcibus_to_physid[pdev->bus->number];
1276         if (phys_id < 0)
1277                 return -ENODEV;
1278
1279         box = uncore_alloc_box(0);
1280         if (!box)
1281                 return -ENOMEM;
1282
1283         /*
1284          * for performance monitoring unit with multiple boxes,
1285          * each box has a different function id.
1286          */
1287         for (i = 0; i < type->num_boxes; i++) {
1288                 pmu = &type->pmus[i];
1289                 if (pmu->func_id == pdev->devfn)
1290                         break;
1291                 if (pmu->func_id < 0) {
1292                         pmu->func_id = pdev->devfn;
1293                         break;
1294                 }
1295                 pmu = NULL;
1296         }
1297
1298         if (!pmu) {
1299                 kfree(box);
1300                 return -EINVAL;
1301         }
1302
1303         box->phys_id = phys_id;
1304         box->pci_dev = pdev;
1305         box->pmu = pmu;
1306         uncore_box_init(box);
1307         pci_set_drvdata(pdev, box);
1308
1309         raw_spin_lock(&uncore_box_lock);
1310         list_add_tail(&box->list, &pmu->box_list);
1311         raw_spin_unlock(&uncore_box_lock);
1312
1313         return 0;
1314 }
1315
1316 static void __devexit uncore_pci_remove(struct pci_dev *pdev)
1317 {
1318         struct intel_uncore_box *box = pci_get_drvdata(pdev);
1319         struct intel_uncore_pmu *pmu = box->pmu;
1320         int cpu, phys_id = pcibus_to_physid[pdev->bus->number];
1321
1322         if (WARN_ON_ONCE(phys_id != box->phys_id))
1323                 return;
1324
1325         raw_spin_lock(&uncore_box_lock);
1326         list_del(&box->list);
1327         raw_spin_unlock(&uncore_box_lock);
1328
1329         for_each_possible_cpu(cpu) {
1330                 if (*per_cpu_ptr(pmu->box, cpu) == box) {
1331                         *per_cpu_ptr(pmu->box, cpu) = NULL;
1332                         atomic_dec(&box->refcnt);
1333                 }
1334         }
1335
1336         WARN_ON_ONCE(atomic_read(&box->refcnt) != 1);
1337         kfree(box);
1338 }
1339
1340 static int __devinit uncore_pci_probe(struct pci_dev *pdev,
1341                                 const struct pci_device_id *id)
1342 {
1343         struct intel_uncore_type *type;
1344
1345         type = (struct intel_uncore_type *)id->driver_data;
1346         return uncore_pci_add(type, pdev);
1347 }
1348
1349 static int __init uncore_pci_init(void)
1350 {
1351         int ret;
1352
1353         switch (boot_cpu_data.x86_model) {
1354         case 45: /* Sandy Bridge-EP */
1355                 pci_uncores = snbep_pci_uncores;
1356                 uncore_pci_driver = &snbep_uncore_pci_driver;
1357                 snbep_pci2phy_map_init();
1358                 break;
1359         default:
1360                 return 0;
1361         }
1362
1363         ret = uncore_types_init(pci_uncores);
1364         if (ret)
1365                 return ret;
1366
1367         uncore_pci_driver->probe = uncore_pci_probe;
1368         uncore_pci_driver->remove = uncore_pci_remove;
1369
1370         ret = pci_register_driver(uncore_pci_driver);
1371         if (ret == 0)
1372                 pcidrv_registered = true;
1373         else
1374                 uncore_types_exit(pci_uncores);
1375
1376         return ret;
1377 }
1378
1379 static void __init uncore_pci_exit(void)
1380 {
1381         if (pcidrv_registered) {
1382                 pcidrv_registered = false;
1383                 pci_unregister_driver(uncore_pci_driver);
1384                 uncore_types_exit(pci_uncores);
1385         }
1386 }
1387
1388 static void __cpuinit uncore_cpu_dying(int cpu)
1389 {
1390         struct intel_uncore_type *type;
1391         struct intel_uncore_pmu *pmu;
1392         struct intel_uncore_box *box;
1393         int i, j;
1394
1395         for (i = 0; msr_uncores[i]; i++) {
1396                 type = msr_uncores[i];
1397                 for (j = 0; j < type->num_boxes; j++) {
1398                         pmu = &type->pmus[j];
1399                         box = *per_cpu_ptr(pmu->box, cpu);
1400                         *per_cpu_ptr(pmu->box, cpu) = NULL;
1401                         if (box && atomic_dec_and_test(&box->refcnt))
1402                                 kfree(box);
1403                 }
1404         }
1405 }
1406
1407 static int __cpuinit uncore_cpu_starting(int cpu)
1408 {
1409         struct intel_uncore_type *type;
1410         struct intel_uncore_pmu *pmu;
1411         struct intel_uncore_box *box, *exist;
1412         int i, j, k, phys_id;
1413
1414         phys_id = topology_physical_package_id(cpu);
1415
1416         for (i = 0; msr_uncores[i]; i++) {
1417                 type = msr_uncores[i];
1418                 for (j = 0; j < type->num_boxes; j++) {
1419                         pmu = &type->pmus[j];
1420                         box = *per_cpu_ptr(pmu->box, cpu);
1421                         /* called by uncore_cpu_init? */
1422                         if (box && box->phys_id >= 0) {
1423                                 uncore_box_init(box);
1424                                 continue;
1425                         }
1426
1427                         for_each_online_cpu(k) {
1428                                 exist = *per_cpu_ptr(pmu->box, k);
1429                                 if (exist && exist->phys_id == phys_id) {
1430                                         atomic_inc(&exist->refcnt);
1431                                         *per_cpu_ptr(pmu->box, cpu) = exist;
1432                                         kfree(box);
1433                                         box = NULL;
1434                                         break;
1435                                 }
1436                         }
1437
1438                         if (box) {
1439                                 box->phys_id = phys_id;
1440                                 uncore_box_init(box);
1441                         }
1442                 }
1443         }
1444         return 0;
1445 }
1446
1447 static int __cpuinit uncore_cpu_prepare(int cpu, int phys_id)
1448 {
1449         struct intel_uncore_type *type;
1450         struct intel_uncore_pmu *pmu;
1451         struct intel_uncore_box *box;
1452         int i, j;
1453
1454         for (i = 0; msr_uncores[i]; i++) {
1455                 type = msr_uncores[i];
1456                 for (j = 0; j < type->num_boxes; j++) {
1457                         pmu = &type->pmus[j];
1458                         if (pmu->func_id < 0)
1459                                 pmu->func_id = j;
1460
1461                         box = uncore_alloc_box(cpu);
1462                         if (!box)
1463                                 return -ENOMEM;
1464
1465                         box->pmu = pmu;
1466                         box->phys_id = phys_id;
1467                         *per_cpu_ptr(pmu->box, cpu) = box;
1468                 }
1469         }
1470         return 0;
1471 }
1472
1473 static void __cpuinit uncore_change_context(struct intel_uncore_type **uncores,
1474                                             int old_cpu, int new_cpu)
1475 {
1476         struct intel_uncore_type *type;
1477         struct intel_uncore_pmu *pmu;
1478         struct intel_uncore_box *box;
1479         int i, j;
1480
1481         for (i = 0; uncores[i]; i++) {
1482                 type = uncores[i];
1483                 for (j = 0; j < type->num_boxes; j++) {
1484                         pmu = &type->pmus[j];
1485                         if (old_cpu < 0)
1486                                 box = uncore_pmu_to_box(pmu, new_cpu);
1487                         else
1488                                 box = uncore_pmu_to_box(pmu, old_cpu);
1489                         if (!box)
1490                                 continue;
1491
1492                         if (old_cpu < 0) {
1493                                 WARN_ON_ONCE(box->cpu != -1);
1494                                 box->cpu = new_cpu;
1495                                 continue;
1496                         }
1497
1498                         WARN_ON_ONCE(box->cpu != old_cpu);
1499                         if (new_cpu >= 0) {
1500                                 uncore_pmu_cancel_hrtimer(box);
1501                                 perf_pmu_migrate_context(&pmu->pmu,
1502                                                 old_cpu, new_cpu);
1503                                 box->cpu = new_cpu;
1504                         } else {
1505                                 box->cpu = -1;
1506                         }
1507                 }
1508         }
1509 }
1510
1511 static void __cpuinit uncore_event_exit_cpu(int cpu)
1512 {
1513         int i, phys_id, target;
1514
1515         /* if exiting cpu is used for collecting uncore events */
1516         if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
1517                 return;
1518
1519         /* find a new cpu to collect uncore events */
1520         phys_id = topology_physical_package_id(cpu);
1521         target = -1;
1522         for_each_online_cpu(i) {
1523                 if (i == cpu)
1524                         continue;
1525                 if (phys_id == topology_physical_package_id(i)) {
1526                         target = i;
1527                         break;
1528                 }
1529         }
1530
1531         /* migrate uncore events to the new cpu */
1532         if (target >= 0)
1533                 cpumask_set_cpu(target, &uncore_cpu_mask);
1534
1535         uncore_change_context(msr_uncores, cpu, target);
1536         uncore_change_context(pci_uncores, cpu, target);
1537 }
1538
1539 static void __cpuinit uncore_event_init_cpu(int cpu)
1540 {
1541         int i, phys_id;
1542
1543         phys_id = topology_physical_package_id(cpu);
1544         for_each_cpu(i, &uncore_cpu_mask) {
1545                 if (phys_id == topology_physical_package_id(i))
1546                         return;
1547         }
1548
1549         cpumask_set_cpu(cpu, &uncore_cpu_mask);
1550
1551         uncore_change_context(msr_uncores, -1, cpu);
1552         uncore_change_context(pci_uncores, -1, cpu);
1553 }
1554
1555 static int __cpuinit uncore_cpu_notifier(struct notifier_block *self,
1556                                          unsigned long action, void *hcpu)
1557 {
1558         unsigned int cpu = (long)hcpu;
1559
1560         /* allocate/free data structure for uncore box */
1561         switch (action & ~CPU_TASKS_FROZEN) {
1562         case CPU_UP_PREPARE:
1563                 uncore_cpu_prepare(cpu, -1);
1564                 break;
1565         case CPU_STARTING:
1566                 uncore_cpu_starting(cpu);
1567                 break;
1568         case CPU_UP_CANCELED:
1569         case CPU_DYING:
1570                 uncore_cpu_dying(cpu);
1571                 break;
1572         default:
1573                 break;
1574         }
1575
1576         /* select the cpu that collects uncore events */
1577         switch (action & ~CPU_TASKS_FROZEN) {
1578         case CPU_DOWN_FAILED:
1579         case CPU_STARTING:
1580                 uncore_event_init_cpu(cpu);
1581                 break;
1582         case CPU_DOWN_PREPARE:
1583                 uncore_event_exit_cpu(cpu);
1584                 break;
1585         default:
1586                 break;
1587         }
1588
1589         return NOTIFY_OK;
1590 }
1591
1592 static struct notifier_block uncore_cpu_nb __cpuinitdata = {
1593         .notifier_call = uncore_cpu_notifier,
1594         /*
1595          * to migrate uncore events, our notifier should be executed
1596          * before perf core's notifier.
1597          */
1598         .priority = CPU_PRI_PERF + 1,
1599 };
1600
1601 static void __init uncore_cpu_setup(void *dummy)
1602 {
1603         uncore_cpu_starting(smp_processor_id());
1604 }
1605
1606 static int __init uncore_cpu_init(void)
1607 {
1608         int ret, cpu;
1609
1610         switch (boot_cpu_data.x86_model) {
1611         case 26: /* Nehalem */
1612         case 30:
1613         case 37: /* Westmere */
1614         case 44:
1615                 msr_uncores = nhm_msr_uncores;
1616                 break;
1617         case 42: /* Sandy Bridge */
1618                 msr_uncores = snb_msr_uncores;
1619                 break;
1620         case 45: /* Sandy Birdge-EP */
1621                 msr_uncores = snbep_msr_uncores;
1622                 break;
1623         default:
1624                 return 0;
1625         }
1626
1627         ret = uncore_types_init(msr_uncores);
1628         if (ret)
1629                 return ret;
1630
1631         get_online_cpus();
1632
1633         for_each_online_cpu(cpu) {
1634                 int i, phys_id = topology_physical_package_id(cpu);
1635
1636                 for_each_cpu(i, &uncore_cpu_mask) {
1637                         if (phys_id == topology_physical_package_id(i)) {
1638                                 phys_id = -1;
1639                                 break;
1640                         }
1641                 }
1642                 if (phys_id < 0)
1643                         continue;
1644
1645                 uncore_cpu_prepare(cpu, phys_id);
1646                 uncore_event_init_cpu(cpu);
1647         }
1648         on_each_cpu(uncore_cpu_setup, NULL, 1);
1649
1650         register_cpu_notifier(&uncore_cpu_nb);
1651
1652         put_online_cpus();
1653
1654         return 0;
1655 }
1656
1657 static int __init uncore_pmus_register(void)
1658 {
1659         struct intel_uncore_pmu *pmu;
1660         struct intel_uncore_type *type;
1661         int i, j;
1662
1663         for (i = 0; msr_uncores[i]; i++) {
1664                 type = msr_uncores[i];
1665                 for (j = 0; j < type->num_boxes; j++) {
1666                         pmu = &type->pmus[j];
1667                         uncore_pmu_register(pmu);
1668                 }
1669         }
1670
1671         for (i = 0; pci_uncores[i]; i++) {
1672                 type = pci_uncores[i];
1673                 for (j = 0; j < type->num_boxes; j++) {
1674                         pmu = &type->pmus[j];
1675                         uncore_pmu_register(pmu);
1676                 }
1677         }
1678
1679         return 0;
1680 }
1681
1682 static int __init intel_uncore_init(void)
1683 {
1684         int ret;
1685
1686         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
1687                 return -ENODEV;
1688
1689         ret = uncore_pci_init();
1690         if (ret)
1691                 goto fail;
1692         ret = uncore_cpu_init();
1693         if (ret) {
1694                 uncore_pci_exit();
1695                 goto fail;
1696         }
1697
1698         uncore_pmus_register();
1699         return 0;
1700 fail:
1701         return ret;
1702 }
1703 device_initcall(intel_uncore_init);