arch/x86/events/core.c

   1 /*
   2  * Performance events x86 architecture code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2009 Jaswinder Singh Rajput
   7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
   9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10  *  Copyright (C) 2009 Google, Inc., Stephane Eranian
  11  *
  12  *  For licencing details see kernel-base/COPYING
  13  */
  14
  15 #include <linux/perf_event.h>
  16 #include <linux/capability.h>
  17 #include <linux/notifier.h>
  18 #include <linux/hardirq.h>
  19 #include <linux/kprobes.h>
  20 #include <linux/export.h>
  21 #include <linux/init.h>
  22 #include <linux/kdebug.h>
  23 #include <linux/sched/mm.h>
  24 #include <linux/sched/clock.h>
  25 #include <linux/uaccess.h>
  26 #include <linux/slab.h>
  27 #include <linux/cpu.h>
  28 #include <linux/bitops.h>
  29 #include <linux/device.h>
  30
  31 #include <asm/apic.h>
  32 #include <asm/stacktrace.h>
  33 #include <asm/nmi.h>
  34 #include <asm/smp.h>
  35 #include <asm/alternative.h>
  36 #include <asm/mmu_context.h>
  37 #include <asm/tlbflush.h>
  38 #include <asm/timer.h>
  39 #include <asm/desc.h>
  40 #include <asm/ldt.h>
  41 #include <asm/unwind.h>
  42
  43 #include "perf_event.h"
  44
  45 struct x86_pmu x86_pmu __read_mostly;
  46
  47 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
  48         .enabled = 1,
  49 };
  50
  51 struct static_key rdpmc_always_available = STATIC_KEY_INIT_FALSE;
  52
  53 u64 __read_mostly hw_cache_event_ids
  54                                 [PERF_COUNT_HW_CACHE_MAX]
  55                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  56                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
  57 u64 __read_mostly hw_cache_extra_regs
  58                                 [PERF_COUNT_HW_CACHE_MAX]
  59                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  60                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
  61
  62 /*
  63  * Propagate event elapsed time into the generic event.
  64  * Can only be executed on the CPU where the event is active.
  65  * Returns the delta events processed.
  66  */
  67 u64 x86_perf_event_update(struct perf_event *event)
  68 {
  69         struct hw_perf_event *hwc = &event->hw;
  70         int shift = 64 - x86_pmu.cntval_bits;
  71         u64 prev_raw_count, new_raw_count;
  72         int idx = hwc->idx;
  73         u64 delta;
  74
  75         if (idx == INTEL_PMC_IDX_FIXED_BTS)
  76                 return 0;
  77
  78         /*
  79          * Careful: an NMI might modify the previous event value.
  80          *
  81          * Our tactic to handle this is to first atomically read and
  82          * exchange a new raw count - then add that new-prev delta
  83          * count to the generic event atomically:
  84          */
  85 again:
  86         prev_raw_count = local64_read(&hwc->prev_count);
  87         rdpmcl(hwc->event_base_rdpmc, new_raw_count);
  88
  89         if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  90                                         new_raw_count) != prev_raw_count)
  91                 goto again;
  92
  93         /*
  94          * Now we have the new raw value and have updated the prev
  95          * timestamp already. We can now calculate the elapsed delta
  96          * (event-)time and add that to the generic event.
  97          *
  98          * Careful, not all hw sign-extends above the physical width
  99          * of the count.
 100          */
 101         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 102         delta >>= shift;
 103
 104         local64_add(delta, &event->count);
 105         local64_sub(delta, &hwc->period_left);
 106
 107         return new_raw_count;
 108 }
 109
 110 /*
 111  * Find and validate any extra registers to set up.
 112  */
 113 static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
 114 {
 115         struct hw_perf_event_extra *reg;
 116         struct extra_reg *er;
 117
 118         reg = &event->hw.extra_reg;
 119
 120         if (!x86_pmu.extra_regs)
 121                 return 0;
 122
 123         for (er = x86_pmu.extra_regs; er->msr; er++) {
 124                 if (er->event != (config & er->config_mask))
 125                         continue;
 126                 if (event->attr.config1 & ~er->valid_mask)
 127                         return -EINVAL;
 128                 /* Check if the extra msrs can be safely accessed*/
 129                 if (!er->extra_msr_access)
 130                         return -ENXIO;
 131
 132                 reg->idx = er->idx;
 133                 reg->config = event->attr.config1;
 134                 reg->reg = er->msr;
 135                 break;
 136         }
 137         return 0;
 138 }
 139
 140 static atomic_t active_events;
 141 static atomic_t pmc_refcount;
 142 static DEFINE_MUTEX(pmc_reserve_mutex);
 143
 144 #ifdef CONFIG_X86_LOCAL_APIC
 145
 146 static bool reserve_pmc_hardware(void)
 147 {
 148         int i;
 149
 150         for (i = 0; i < x86_pmu.num_counters; i++) {
 151                 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
 152                         goto perfctr_fail;
 153         }
 154
 155         for (i = 0; i < x86_pmu.num_counters; i++) {
 156                 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
 157                         goto eventsel_fail;
 158         }
 159
 160         return true;
 161
 162 eventsel_fail:
 163         for (i--; i >= 0; i--)
 164                 release_evntsel_nmi(x86_pmu_config_addr(i));
 165
 166         i = x86_pmu.num_counters;
 167
 168 perfctr_fail:
 169         for (i--; i >= 0; i--)
 170                 release_perfctr_nmi(x86_pmu_event_addr(i));
 171
 172         return false;
 173 }
 174
 175 static void release_pmc_hardware(void)
 176 {
 177         int i;
 178
 179         for (i = 0; i < x86_pmu.num_counters; i++) {
 180                 release_perfctr_nmi(x86_pmu_event_addr(i));
 181                 release_evntsel_nmi(x86_pmu_config_addr(i));
 182         }
 183 }
 184
 185 #else
 186
 187 static bool reserve_pmc_hardware(void) { return true; }
 188 static void release_pmc_hardware(void) {}
 189
 190 #endif
 191
 192 static bool check_hw_exists(void)
 193 {
 194         u64 val, val_fail = -1, val_new= ~0;
 195         int i, reg, reg_fail = -1, ret = 0;
 196         int bios_fail = 0;
 197         int reg_safe = -1;
 198
 199         /*
 200          * Check to see if the BIOS enabled any of the counters, if so
 201          * complain and bail.
 202          */
 203         for (i = 0; i < x86_pmu.num_counters; i++) {
 204                 reg = x86_pmu_config_addr(i);
 205                 ret = rdmsrl_safe(reg, &val);
 206                 if (ret)
 207                         goto msr_fail;
 208                 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
 209                         bios_fail = 1;
 210                         val_fail = val;
 211                         reg_fail = reg;
 212                 } else {
 213                         reg_safe = i;
 214                 }
 215         }
 216
 217         if (x86_pmu.num_counters_fixed) {
 218                 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 219                 ret = rdmsrl_safe(reg, &val);
 220                 if (ret)
 221                         goto msr_fail;
 222                 for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
 223                         if (val & (0x03 << i*4)) {
 224                                 bios_fail = 1;
 225                                 val_fail = val;
 226                                 reg_fail = reg;
 227                         }
 228                 }
 229         }
 230
 231         /*
 232          * If all the counters are enabled, the below test will always
 233          * fail.  The tools will also become useless in this scenario.
 234          * Just fail and disable the hardware counters.
 235          */
 236
 237         if (reg_safe == -1) {
 238                 reg = reg_safe;
 239                 goto msr_fail;
 240         }
 241
 242         /*
 243          * Read the current value, change it and read it back to see if it
 244          * matches, this is needed to detect certain hardware emulators
 245          * (qemu/kvm) that don't trap on the MSR access and always return 0s.
 246          */
 247         reg = x86_pmu_event_addr(reg_safe);
 248         if (rdmsrl_safe(reg, &val))
 249                 goto msr_fail;
 250         val ^= 0xffffUL;
 251         ret = wrmsrl_safe(reg, val);
 252         ret |= rdmsrl_safe(reg, &val_new);
 253         if (ret || val != val_new)
 254                 goto msr_fail;
 255
 256         /*
 257          * We still allow the PMU driver to operate:
 258          */
 259         if (bios_fail) {
 260                 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n");
 261                 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n",
 262                               reg_fail, val_fail);
 263         }
 264
 265         return true;
 266
 267 msr_fail:
 268         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
 269                 pr_cont("PMU not available due to virtualization, using software events only.\n");
 270         } else {
 271                 pr_cont("Broken PMU hardware detected, using software events only.\n");
 272                 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n",
 273                        reg, val_new);
 274         }
 275
 276         return false;
 277 }
 278
 279 static void hw_perf_event_destroy(struct perf_event *event)
 280 {
 281         x86_release_hardware();
 282         atomic_dec(&active_events);
 283 }
 284
 285 void hw_perf_lbr_event_destroy(struct perf_event *event)
 286 {
 287         hw_perf_event_destroy(event);
 288
 289         /* undo the lbr/bts event accounting */
 290         x86_del_exclusive(x86_lbr_exclusive_lbr);
 291 }
 292
 293 static inline int x86_pmu_initialized(void)
 294 {
 295         return x86_pmu.handle_irq != NULL;
 296 }
 297
 298 static inline int
 299 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
 300 {
 301         struct perf_event_attr *attr = &event->attr;
 302         unsigned int cache_type, cache_op, cache_result;
 303         u64 config, val;
 304
 305         config = attr->config;
 306
 307         cache_type = (config >>  0) & 0xff;
 308         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 309                 return -EINVAL;
 310
 311         cache_op = (config >>  8) & 0xff;
 312         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 313                 return -EINVAL;
 314
 315         cache_result = (config >> 16) & 0xff;
 316         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 317                 return -EINVAL;
 318
 319         val = hw_cache_event_ids[cache_type][cache_op][cache_result];
 320
 321         if (val == 0)
 322                 return -ENOENT;
 323
 324         if (val == -1)
 325                 return -EINVAL;
 326
 327         hwc->config |= val;
 328         attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
 329         return x86_pmu_extra_regs(val, event);
 330 }
 331
 332 int x86_reserve_hardware(void)
 333 {
 334         int err = 0;
 335
 336         if (!atomic_inc_not_zero(&pmc_refcount)) {
 337                 mutex_lock(&pmc_reserve_mutex);
 338                 if (atomic_read(&pmc_refcount) == 0) {
 339                         if (!reserve_pmc_hardware())
 340                                 err = -EBUSY;
 341                         else
 342                                 reserve_ds_buffers();
 343                 }
 344                 if (!err)
 345                         atomic_inc(&pmc_refcount);
 346                 mutex_unlock(&pmc_reserve_mutex);
 347         }
 348
 349         return err;
 350 }
 351
 352 void x86_release_hardware(void)
 353 {
 354         if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
 355                 release_pmc_hardware();
 356                 release_ds_buffers();
 357                 mutex_unlock(&pmc_reserve_mutex);
 358         }
 359 }
 360
 361 /*
 362  * Check if we can create event of a certain type (that no conflicting events
 363  * are present).
 364  */
 365 int x86_add_exclusive(unsigned int what)
 366 {
 367         int i;
 368
 369         /*
 370          * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS.
 371          * LBR and BTS are still mutually exclusive.
 372          */
 373         if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
 374                 return 0;
 375
 376         if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) {
 377                 mutex_lock(&pmc_reserve_mutex);
 378                 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) {
 379                         if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i]))
 380                                 goto fail_unlock;
 381                 }
 382                 atomic_inc(&x86_pmu.lbr_exclusive[what]);
 383                 mutex_unlock(&pmc_reserve_mutex);
 384         }
 385
 386         atomic_inc(&active_events);
 387         return 0;
 388
 389 fail_unlock:
 390         mutex_unlock(&pmc_reserve_mutex);
 391         return -EBUSY;
 392 }
 393
 394 void x86_del_exclusive(unsigned int what)
 395 {
 396         if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
 397                 return;
 398
 399         atomic_dec(&x86_pmu.lbr_exclusive[what]);
 400         atomic_dec(&active_events);
 401 }
 402
 403 int x86_setup_perfctr(struct perf_event *event)
 404 {
 405         struct perf_event_attr *attr = &event->attr;
 406         struct hw_perf_event *hwc = &event->hw;
 407         u64 config;
 408
 409         if (!is_sampling_event(event)) {
 410                 hwc->sample_period = x86_pmu.max_period;
 411                 hwc->last_period = hwc->sample_period;
 412                 local64_set(&hwc->period_left, hwc->sample_period);
 413         }
 414
 415         if (attr->type == PERF_TYPE_RAW)
 416                 return x86_pmu_extra_regs(event->attr.config, event);
 417
 418         if (attr->type == PERF_TYPE_HW_CACHE)
 419                 return set_ext_hw_attr(hwc, event);
 420
 421         if (attr->config >= x86_pmu.max_events)
 422                 return -EINVAL;
 423
 424         /*
 425          * The generic map:
 426          */
 427         config = x86_pmu.event_map(attr->config);
 428
 429         if (config == 0)
 430                 return -ENOENT;
 431
 432         if (config == -1LL)
 433                 return -EINVAL;
 434
 435         /*
 436          * Branch tracing:
 437          */
 438         if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS &&
 439             !attr->freq && hwc->sample_period == 1) {
 440                 /* BTS is not supported by this architecture. */
 441                 if (!x86_pmu.bts_active)
 442                         return -EOPNOTSUPP;
 443
 444                 /* BTS is currently only allowed for user-mode. */
 445                 if (!attr->exclude_kernel)
 446                         return -EOPNOTSUPP;
 447
 448                 /* disallow bts if conflicting events are present */
 449                 if (x86_add_exclusive(x86_lbr_exclusive_lbr))
 450                         return -EBUSY;
 451
 452                 event->destroy = hw_perf_lbr_event_destroy;
 453         }
 454
 455         hwc->config |= config;
 456
 457         return 0;
 458 }
 459
 460 /*
 461  * check that branch_sample_type is compatible with
 462  * settings needed for precise_ip > 1 which implies
 463  * using the LBR to capture ALL taken branches at the
 464  * priv levels of the measurement
 465  */
 466 static inline int precise_br_compat(struct perf_event *event)
 467 {
 468         u64 m = event->attr.branch_sample_type;
 469         u64 b = 0;
 470
 471         /* must capture all branches */
 472         if (!(m & PERF_SAMPLE_BRANCH_ANY))
 473                 return 0;
 474
 475         m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
 476
 477         if (!event->attr.exclude_user)
 478                 b |= PERF_SAMPLE_BRANCH_USER;
 479
 480         if (!event->attr.exclude_kernel)
 481                 b |= PERF_SAMPLE_BRANCH_KERNEL;
 482
 483         /*
 484          * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
 485          */
 486
 487         return m == b;
 488 }
 489
 490 int x86_pmu_max_precise(void)
 491 {
 492         int precise = 0;
 493
 494         /* Support for constant skid */
 495         if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
 496                 precise++;
 497
 498                 /* Support for IP fixup */
 499                 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
 500                         precise++;
 501
 502                 if (x86_pmu.pebs_prec_dist)
 503                         precise++;
 504         }
 505         return precise;
 506 }
 507
 508 int x86_pmu_hw_config(struct perf_event *event)
 509 {
 510         if (event->attr.precise_ip) {
 511                 int precise = x86_pmu_max_precise();
 512
 513                 if (event->attr.precise_ip > precise)
 514                         return -EOPNOTSUPP;
 515
 516                 /* There's no sense in having PEBS for non sampling events: */
 517                 if (!is_sampling_event(event))
 518                         return -EINVAL;
 519         }
 520         /*
 521          * check that PEBS LBR correction does not conflict with
 522          * whatever the user is asking with attr->branch_sample_type
 523          */
 524         if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) {
 525                 u64 *br_type = &event->attr.branch_sample_type;
 526
 527                 if (has_branch_stack(event)) {
 528                         if (!precise_br_compat(event))
 529                                 return -EOPNOTSUPP;
 530
 531                         /* branch_sample_type is compatible */
 532
 533                 } else {
 534                         /*
 535                          * user did not specify  branch_sample_type
 536                          *
 537                          * For PEBS fixups, we capture all
 538                          * the branches at the priv level of the
 539                          * event.
 540                          */
 541                         *br_type = PERF_SAMPLE_BRANCH_ANY;
 542
 543                         if (!event->attr.exclude_user)
 544                                 *br_type |= PERF_SAMPLE_BRANCH_USER;
 545
 546                         if (!event->attr.exclude_kernel)
 547                                 *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
 548                 }
 549         }
 550
 551         if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK)
 552                 event->attach_state |= PERF_ATTACH_TASK_DATA;
 553
 554         /*
 555          * Generate PMC IRQs:
 556          * (keep 'enabled' bit clear for now)
 557          */
 558         event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
 559
 560         /*
 561          * Count user and OS events unless requested not to
 562          */
 563         if (!event->attr.exclude_user)
 564                 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
 565         if (!event->attr.exclude_kernel)
 566                 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
 567
 568         if (event->attr.type == PERF_TYPE_RAW)
 569                 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
 570
 571         if (event->attr.sample_period && x86_pmu.limit_period) {
 572                 if (x86_pmu.limit_period(event, event->attr.sample_period) >
 573                                 event->attr.sample_period)
 574                         return -EINVAL;
 575         }
 576
 577         return x86_setup_perfctr(event);
 578 }
 579
 580 /*
 581  * Setup the hardware configuration for a given attr_type
 582  */
 583 static int __x86_pmu_event_init(struct perf_event *event)
 584 {
 585         int err;
 586
 587         if (!x86_pmu_initialized())
 588                 return -ENODEV;
 589
 590         err = x86_reserve_hardware();
 591         if (err)
 592                 return err;
 593
 594         atomic_inc(&active_events);
 595         event->destroy = hw_perf_event_destroy;
 596
 597         event->hw.idx = -1;
 598         event->hw.last_cpu = -1;
 599         event->hw.last_tag = ~0ULL;
 600
 601         /* mark unused */
 602         event->hw.extra_reg.idx = EXTRA_REG_NONE;
 603         event->hw.branch_reg.idx = EXTRA_REG_NONE;
 604
 605         return x86_pmu.hw_config(event);
 606 }
 607
 608 void x86_pmu_disable_all(void)
 609 {
 610         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 611         int idx;
 612
 613         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 614                 u64 val;
 615
 616                 if (!test_bit(idx, cpuc->active_mask))
 617                         continue;
 618                 rdmsrl(x86_pmu_config_addr(idx), val);
 619                 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
 620                         continue;
 621                 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
 622                 wrmsrl(x86_pmu_config_addr(idx), val);
 623         }
 624 }
 625
 626 /*
 627  * There may be PMI landing after enabled=0. The PMI hitting could be before or
 628  * after disable_all.
 629  *
 630  * If PMI hits before disable_all, the PMU will be disabled in the NMI handler.
 631  * It will not be re-enabled in the NMI handler again, because enabled=0. After
 632  * handling the NMI, disable_all will be called, which will not change the
 633  * state either. If PMI hits after disable_all, the PMU is already disabled
 634  * before entering NMI handler. The NMI handler will not change the state
 635  * either.
 636  *
 637  * So either situation is harmless.
 638  */
 639 static void x86_pmu_disable(struct pmu *pmu)
 640 {
 641         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 642
 643         if (!x86_pmu_initialized())
 644                 return;
 645
 646         if (!cpuc->enabled)
 647                 return;
 648
 649         cpuc->n_added = 0;
 650         cpuc->enabled = 0;
 651         barrier();
 652
 653         x86_pmu.disable_all();
 654 }
 655
 656 void x86_pmu_enable_all(int added)
 657 {
 658         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 659         int idx;
 660
 661         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 662                 struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
 663
 664                 if (!test_bit(idx, cpuc->active_mask))
 665                         continue;
 666
 667                 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
 668         }
 669 }
 670
 671 static struct pmu pmu;
 672
 673 static inline int is_x86_event(struct perf_event *event)
 674 {
 675         return event->pmu == &pmu;
 676 }
 677
 678 /*
 679  * Event scheduler state:
 680  *
 681  * Assign events iterating over all events and counters, beginning
 682  * with events with least weights first. Keep the current iterator
 683  * state in struct sched_state.
 684  */
 685 struct sched_state {
 686         int     weight;
 687         int     event;          /* event index */
 688         int     counter;        /* counter index */
 689         int     unassigned;     /* number of events to be assigned left */
 690         int     nr_gp;          /* number of GP counters used */
 691         unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 692 };
 693
 694 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
 695 #define SCHED_STATES_MAX        2
 696
 697 struct perf_sched {
 698         int                     max_weight;
 699         int                     max_events;
 700         int                     max_gp;
 701         int                     saved_states;
 702         struct event_constraint **constraints;
 703         struct sched_state      state;
 704         struct sched_state      saved[SCHED_STATES_MAX];
 705 };
 706
 707 /*
 708  * Initialize interator that runs through all events and counters.
 709  */
 710 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
 711                             int num, int wmin, int wmax, int gpmax)
 712 {
 713         int idx;
 714
 715         memset(sched, 0, sizeof(*sched));
 716         sched->max_events       = num;
 717         sched->max_weight       = wmax;
 718         sched->max_gp           = gpmax;
 719         sched->constraints      = constraints;
 720
 721         for (idx = 0; idx < num; idx++) {
 722                 if (constraints[idx]->weight == wmin)
 723                         break;
 724         }
 725
 726         sched->state.event      = idx;          /* start with min weight */
 727         sched->state.weight     = wmin;
 728         sched->state.unassigned = num;
 729 }
 730
 731 static void perf_sched_save_state(struct perf_sched *sched)
 732 {
 733         if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
 734                 return;
 735
 736         sched->saved[sched->saved_states] = sched->state;
 737         sched->saved_states++;
 738 }
 739
 740 static bool perf_sched_restore_state(struct perf_sched *sched)
 741 {
 742         if (!sched->saved_states)
 743                 return false;
 744
 745         sched->saved_states--;
 746         sched->state = sched->saved[sched->saved_states];
 747
 748         /* continue with next counter: */
 749         clear_bit(sched->state.counter++, sched->state.used);
 750
 751         return true;
 752 }
 753
 754 /*
 755  * Select a counter for the current event to schedule. Return true on
 756  * success.
 757  */
 758 static bool __perf_sched_find_counter(struct perf_sched *sched)
 759 {
 760         struct event_constraint *c;
 761         int idx;
 762
 763         if (!sched->state.unassigned)
 764                 return false;
 765
 766         if (sched->state.event >= sched->max_events)
 767                 return false;
 768
 769         c = sched->constraints[sched->state.event];
 770         /* Prefer fixed purpose counters */
 771         if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
 772                 idx = INTEL_PMC_IDX_FIXED;
 773                 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
 774                         if (!__test_and_set_bit(idx, sched->state.used))
 775                                 goto done;
 776                 }
 777         }
 778
 779         /* Grab the first unused counter starting with idx */
 780         idx = sched->state.counter;
 781         for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
 782                 if (!__test_and_set_bit(idx, sched->state.used)) {
 783                         if (sched->state.nr_gp++ >= sched->max_gp)
 784                                 return false;
 785
 786                         goto done;
 787                 }
 788         }
 789
 790         return false;
 791
 792 done:
 793         sched->state.counter = idx;
 794
 795         if (c->overlap)
 796                 perf_sched_save_state(sched);
 797
 798         return true;
 799 }
 800
 801 static bool perf_sched_find_counter(struct perf_sched *sched)
 802 {
 803         while (!__perf_sched_find_counter(sched)) {
 804                 if (!perf_sched_restore_state(sched))
 805                         return false;
 806         }
 807
 808         return true;
 809 }
 810
 811 /*
 812  * Go through all unassigned events and find the next one to schedule.
 813  * Take events with the least weight first. Return true on success.
 814  */
 815 static bool perf_sched_next_event(struct perf_sched *sched)
 816 {
 817         struct event_constraint *c;
 818
 819         if (!sched->state.unassigned || !--sched->state.unassigned)
 820                 return false;
 821
 822         do {
 823                 /* next event */
 824                 sched->state.event++;
 825                 if (sched->state.event >= sched->max_events) {
 826                         /* next weight */
 827                         sched->state.event = 0;
 828                         sched->state.weight++;
 829                         if (sched->state.weight > sched->max_weight)
 830                                 return false;
 831                 }
 832                 c = sched->constraints[sched->state.event];
 833         } while (c->weight != sched->state.weight);
 834
 835         sched->state.counter = 0;       /* start with first counter */
 836
 837         return true;
 838 }
 839
 840 /*
 841  * Assign a counter for each event.
 842  */
 843 int perf_assign_events(struct event_constraint **constraints, int n,
 844                         int wmin, int wmax, int gpmax, int *assign)
 845 {
 846         struct perf_sched sched;
 847
 848         perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);
 849
 850         do {
 851                 if (!perf_sched_find_counter(&sched))
 852                         break;  /* failed */
 853                 if (assign)
 854                         assign[sched.state.event] = sched.state.counter;
 855         } while (perf_sched_next_event(&sched));
 856
 857         return sched.state.unassigned;
 858 }
 859 EXPORT_SYMBOL_GPL(perf_assign_events);
 860
 861 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
 862 {
 863         struct event_constraint *c;
 864         unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 865         struct perf_event *e;
 866         int i, wmin, wmax, unsched = 0;
 867         struct hw_perf_event *hwc;
 868
 869         bitmap_zero(used_mask, X86_PMC_IDX_MAX);
 870
 871         if (x86_pmu.start_scheduling)
 872                 x86_pmu.start_scheduling(cpuc);
 873
 874         for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
 875                 cpuc->event_constraint[i] = NULL;
 876                 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
 877                 cpuc->event_constraint[i] = c;
 878
 879                 wmin = min(wmin, c->weight);
 880                 wmax = max(wmax, c->weight);
 881         }
 882
 883         /*
 884          * fastpath, try to reuse previous register
 885          */
 886         for (i = 0; i < n; i++) {
 887                 hwc = &cpuc->event_list[i]->hw;
 888                 c = cpuc->event_constraint[i];
 889
 890                 /* never assigned */
 891                 if (hwc->idx == -1)
 892                         break;
 893
 894                 /* constraint still honored */
 895                 if (!test_bit(hwc->idx, c->idxmsk))
 896                         break;
 897
 898                 /* not already used */
 899                 if (test_bit(hwc->idx, used_mask))
 900                         break;
 901
 902                 __set_bit(hwc->idx, used_mask);
 903                 if (assign)
 904                         assign[i] = hwc->idx;
 905         }
 906
 907         /* slow path */
 908         if (i != n) {
 909                 int gpmax = x86_pmu.num_counters;
 910
 911                 /*
 912                  * Do not allow scheduling of more than half the available
 913                  * generic counters.
 914                  *
 915                  * This helps avoid counter starvation of sibling thread by
 916                  * ensuring at most half the counters cannot be in exclusive
 917                  * mode. There is no designated counters for the limits. Any
 918                  * N/2 counters can be used. This helps with events with
 919                  * specific counter constraints.
 920                  */
 921                 if (is_ht_workaround_enabled() && !cpuc->is_fake &&
 922                     READ_ONCE(cpuc->excl_cntrs->exclusive_present))
 923                         gpmax /= 2;
 924
 925                 unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
 926                                              wmax, gpmax, assign);
 927         }
 928
 929         /*
 930          * In case of success (unsched = 0), mark events as committed,
 931          * so we do not put_constraint() in case new events are added
 932          * and fail to be scheduled
 933          *
 934          * We invoke the lower level commit callback to lock the resource
 935          *
 936          * We do not need to do all of this in case we are called to
 937          * validate an event group (assign == NULL)
 938          */
 939         if (!unsched && assign) {
 940                 for (i = 0; i < n; i++) {
 941                         e = cpuc->event_list[i];
 942                         e->hw.flags |= PERF_X86_EVENT_COMMITTED;
 943                         if (x86_pmu.commit_scheduling)
 944                                 x86_pmu.commit_scheduling(cpuc, i, assign[i]);
 945                 }
 946         } else {
 947                 for (i = 0; i < n; i++) {
 948                         e = cpuc->event_list[i];
 949                         /*
 950                          * do not put_constraint() on comitted events,
 951                          * because they are good to go
 952                          */
 953                         if ((e->hw.flags & PERF_X86_EVENT_COMMITTED))
 954                                 continue;
 955
 956                         /*
 957                          * release events that failed scheduling
 958                          */
 959                         if (x86_pmu.put_event_constraints)
 960                                 x86_pmu.put_event_constraints(cpuc, e);
 961                 }
 962         }
 963
 964         if (x86_pmu.stop_scheduling)
 965                 x86_pmu.stop_scheduling(cpuc);
 966
 967         return unsched ? -EINVAL : 0;
 968 }
 969
 970 /*
 971  * dogrp: true if must collect siblings events (group)
 972  * returns total number of events and error code
 973  */
 974 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
 975 {
 976         struct perf_event *event;
 977         int n, max_count;
 978
 979         max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
 980
 981         /* current number of events already accepted */
 982         n = cpuc->n_events;
 983
 984         if (is_x86_event(leader)) {
 985                 if (n >= max_count)
 986                         return -EINVAL;
 987                 cpuc->event_list[n] = leader;
 988                 n++;
 989         }
 990         if (!dogrp)
 991                 return n;
 992
 993         list_for_each_entry(event, &leader->sibling_list, group_entry) {
 994                 if (!is_x86_event(event) ||
 995                     event->state <= PERF_EVENT_STATE_OFF)
 996                         continue;
 997
 998                 if (n >= max_count)
 999                         return -EINVAL;
1000
1001                 cpuc->event_list[n] = event;
1002                 n++;
1003         }
1004         return n;
1005 }
1006
1007 static inline void x86_assign_hw_event(struct perf_event *event,
1008                                 struct cpu_hw_events *cpuc, int i)
1009 {
1010         struct hw_perf_event *hwc = &event->hw;
1011
1012         hwc->idx = cpuc->assign[i];
1013         hwc->last_cpu = smp_processor_id();
1014         hwc->last_tag = ++cpuc->tags[i];
1015
1016         if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) {
1017                 hwc->config_base = 0;
1018                 hwc->event_base = 0;
1019         } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
1020                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1021                 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED);
1022                 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30;
1023         } else {
1024                 hwc->config_base = x86_pmu_config_addr(hwc->idx);
1025                 hwc->event_base  = x86_pmu_event_addr(hwc->idx);
1026                 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
1027         }
1028 }
1029
1030 static inline int match_prev_assignment(struct hw_perf_event *hwc,
1031                                         struct cpu_hw_events *cpuc,
1032                                         int i)
1033 {
1034         return hwc->idx == cpuc->assign[i] &&
1035                 hwc->last_cpu == smp_processor_id() &&
1036                 hwc->last_tag == cpuc->tags[i];
1037 }
1038
1039 static void x86_pmu_start(struct perf_event *event, int flags);
1040
1041 static void x86_pmu_enable(struct pmu *pmu)
1042 {
1043         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1044         struct perf_event *event;
1045         struct hw_perf_event *hwc;
1046         int i, added = cpuc->n_added;
1047
1048         if (!x86_pmu_initialized())
1049                 return;
1050
1051         if (cpuc->enabled)
1052                 return;
1053
1054         if (cpuc->n_added) {
1055                 int n_running = cpuc->n_events - cpuc->n_added;
1056                 /*
1057                  * apply assignment obtained either from
1058                  * hw_perf_group_sched_in() or x86_pmu_enable()
1059                  *
1060                  * step1: save events moving to new counters
1061                  */
1062                 for (i = 0; i < n_running; i++) {
1063                         event = cpuc->event_list[i];
1064                         hwc = &event->hw;
1065
1066                         /*
1067                          * we can avoid reprogramming counter if:
1068                          * - assigned same counter as last time
1069                          * - running on same CPU as last time
1070                          * - no other event has used the counter since
1071                          */
1072                         if (hwc->idx == -1 ||
1073                             match_prev_assignment(hwc, cpuc, i))
1074                                 continue;
1075
1076                         /*
1077                          * Ensure we don't accidentally enable a stopped
1078                          * counter simply because we rescheduled.
1079                          */
1080                         if (hwc->state & PERF_HES_STOPPED)
1081                                 hwc->state |= PERF_HES_ARCH;
1082
1083                         x86_pmu_stop(event, PERF_EF_UPDATE);
1084                 }
1085
1086                 /*
1087                  * step2: reprogram moved events into new counters
1088                  */
1089                 for (i = 0; i < cpuc->n_events; i++) {
1090                         event = cpuc->event_list[i];
1091                         hwc = &event->hw;
1092
1093                         if (!match_prev_assignment(hwc, cpuc, i))
1094                                 x86_assign_hw_event(event, cpuc, i);
1095                         else if (i < n_running)
1096                                 continue;
1097
1098                         if (hwc->state & PERF_HES_ARCH)
1099                                 continue;
1100
1101                         x86_pmu_start(event, PERF_EF_RELOAD);
1102                 }
1103                 cpuc->n_added = 0;
1104                 perf_events_lapic_init();
1105         }
1106
1107         cpuc->enabled = 1;
1108         barrier();
1109
1110         x86_pmu.enable_all(added);
1111 }
1112
1113 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1114
1115 /*
1116  * Set the next IRQ period, based on the hwc->period_left value.
1117  * To be called with the event disabled in hw:
1118  */
1119 int x86_perf_event_set_period(struct perf_event *event)
1120 {
1121         struct hw_perf_event *hwc = &event->hw;
1122         s64 left = local64_read(&hwc->period_left);
1123         s64 period = hwc->sample_period;
1124         int ret = 0, idx = hwc->idx;
1125
1126         if (idx == INTEL_PMC_IDX_FIXED_BTS)
1127                 return 0;
1128
1129         /*
1130          * If we are way outside a reasonable range then just skip forward:
1131          */
1132         if (unlikely(left <= -period)) {
1133                 left = period;
1134                 local64_set(&hwc->period_left, left);
1135                 hwc->last_period = period;
1136                 ret = 1;
1137         }
1138
1139         if (unlikely(left <= 0)) {
1140                 left += period;
1141                 local64_set(&hwc->period_left, left);
1142                 hwc->last_period = period;
1143                 ret = 1;
1144         }
1145         /*
1146          * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1147          */
1148         if (unlikely(left < 2))
1149                 left = 2;
1150
1151         if (left > x86_pmu.max_period)
1152                 left = x86_pmu.max_period;
1153
1154         if (x86_pmu.limit_period)
1155                 left = x86_pmu.limit_period(event, left);
1156
1157         per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1158
1159         if (!(hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) ||
1160             local64_read(&hwc->prev_count) != (u64)-left) {
1161                 /*
1162                  * The hw event starts counting from this event offset,
1163                  * mark it to be able to extra future deltas:
1164                  */
1165                 local64_set(&hwc->prev_count, (u64)-left);
1166
1167                 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
1168         }
1169
1170         /*
1171          * Due to erratum on certan cpu we need
1172          * a second write to be sure the register
1173          * is updated properly
1174          */
1175         if (x86_pmu.perfctr_second_write) {
1176                 wrmsrl(hwc->event_base,
1177                         (u64)(-left) & x86_pmu.cntval_mask);
1178         }
1179
1180         perf_event_update_userpage(event);
1181
1182         return ret;
1183 }
1184
1185 void x86_pmu_enable_event(struct perf_event *event)
1186 {
1187         if (__this_cpu_read(cpu_hw_events.enabled))
1188                 __x86_pmu_enable_event(&event->hw,
1189                                        ARCH_PERFMON_EVENTSEL_ENABLE);
1190 }
1191
1192 /*
1193  * Add a single event to the PMU.
1194  *
1195  * The event is added to the group of enabled events
1196  * but only if it can be scehduled with existing events.
1197  */
1198 static int x86_pmu_add(struct perf_event *event, int flags)
1199 {
1200         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1201         struct hw_perf_event *hwc;
1202         int assign[X86_PMC_IDX_MAX];
1203         int n, n0, ret;
1204
1205         hwc = &event->hw;
1206
1207         n0 = cpuc->n_events;
1208         ret = n = collect_events(cpuc, event, false);
1209         if (ret < 0)
1210                 goto out;
1211
1212         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1213         if (!(flags & PERF_EF_START))
1214                 hwc->state |= PERF_HES_ARCH;
1215
1216         /*
1217          * If group events scheduling transaction was started,
1218          * skip the schedulability test here, it will be performed
1219          * at commit time (->commit_txn) as a whole.
1220          *
1221          * If commit fails, we'll call ->del() on all events
1222          * for which ->add() was called.
1223          */
1224         if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1225                 goto done_collect;
1226
1227         ret = x86_pmu.schedule_events(cpuc, n, assign);
1228         if (ret)
1229                 goto out;
1230         /*
1231          * copy new assignment, now we know it is possible
1232          * will be used by hw_perf_enable()
1233          */
1234         memcpy(cpuc->assign, assign, n*sizeof(int));
1235
1236 done_collect:
1237         /*
1238          * Commit the collect_events() state. See x86_pmu_del() and
1239          * x86_pmu_*_txn().
1240          */
1241         cpuc->n_events = n;
1242         cpuc->n_added += n - n0;
1243         cpuc->n_txn += n - n0;
1244
1245         if (x86_pmu.add) {
1246                 /*
1247                  * This is before x86_pmu_enable() will call x86_pmu_start(),
1248                  * so we enable LBRs before an event needs them etc..
1249                  */
1250                 x86_pmu.add(event);
1251         }
1252
1253         ret = 0;
1254 out:
1255         return ret;
1256 }
1257
1258 static void x86_pmu_start(struct perf_event *event, int flags)
1259 {
1260         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1261         int idx = event->hw.idx;
1262
1263         if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1264                 return;
1265
1266         if (WARN_ON_ONCE(idx == -1))
1267                 return;
1268
1269         if (flags & PERF_EF_RELOAD) {
1270                 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1271                 x86_perf_event_set_period(event);
1272         }
1273
1274         event->hw.state = 0;
1275
1276         cpuc->events[idx] = event;
1277         __set_bit(idx, cpuc->active_mask);
1278         __set_bit(idx, cpuc->running);
1279         x86_pmu.enable(event);
1280         perf_event_update_userpage(event);
1281 }
1282
1283 void perf_event_print_debug(void)
1284 {
1285         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1286         u64 pebs, debugctl;
1287         struct cpu_hw_events *cpuc;
1288         unsigned long flags;
1289         int cpu, idx;
1290
1291         if (!x86_pmu.num_counters)
1292                 return;
1293
1294         local_irq_save(flags);
1295
1296         cpu = smp_processor_id();
1297         cpuc = &per_cpu(cpu_hw_events, cpu);
1298
1299         if (x86_pmu.version >= 2) {
1300                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1301                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1302                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1303                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1304
1305                 pr_info("\n");
1306                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1307                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1308                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1309                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1310                 if (x86_pmu.pebs_constraints) {
1311                         rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1312                         pr_info("CPU#%d: pebs:       %016llx\n", cpu, pebs);
1313                 }
1314                 if (x86_pmu.lbr_nr) {
1315                         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1316                         pr_info("CPU#%d: debugctl:   %016llx\n", cpu, debugctl);
1317                 }
1318         }
1319         pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);
1320
1321         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1322                 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
1323                 rdmsrl(x86_pmu_event_addr(idx), pmc_count);
1324
1325                 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1326
1327                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1328                         cpu, idx, pmc_ctrl);
1329                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1330                         cpu, idx, pmc_count);
1331                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1332                         cpu, idx, prev_left);
1333         }
1334         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1335                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1336
1337                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1338                         cpu, idx, pmc_count);
1339         }
1340         local_irq_restore(flags);
1341 }
1342
1343 void x86_pmu_stop(struct perf_event *event, int flags)
1344 {
1345         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1346         struct hw_perf_event *hwc = &event->hw;
1347
1348         if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
1349                 x86_pmu.disable(event);
1350                 cpuc->events[hwc->idx] = NULL;
1351                 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1352                 hwc->state |= PERF_HES_STOPPED;
1353         }
1354
1355         if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1356                 /*
1357                  * Drain the remaining delta count out of a event
1358                  * that we are disabling:
1359                  */
1360                 x86_perf_event_update(event);
1361                 hwc->state |= PERF_HES_UPTODATE;
1362         }
1363 }
1364
1365 static void x86_pmu_del(struct perf_event *event, int flags)
1366 {
1367         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1368         int i;
1369
1370         /*
1371          * event is descheduled
1372          */
1373         event->hw.flags &= ~PERF_X86_EVENT_COMMITTED;
1374
1375         /*
1376          * If we're called during a txn, we only need to undo x86_pmu.add.
1377          * The events never got scheduled and ->cancel_txn will truncate
1378          * the event_list.
1379          *
1380          * XXX assumes any ->del() called during a TXN will only be on
1381          * an event added during that same TXN.
1382          */
1383         if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1384                 goto do_del;
1385
1386         /*
1387          * Not a TXN, therefore cleanup properly.
1388          */
1389         x86_pmu_stop(event, PERF_EF_UPDATE);
1390
1391         for (i = 0; i < cpuc->n_events; i++) {
1392                 if (event == cpuc->event_list[i])
1393                         break;
1394         }
1395
1396         if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
1397                 return;
1398
1399         /* If we have a newly added event; make sure to decrease n_added. */
1400         if (i >= cpuc->n_events - cpuc->n_added)
1401                 --cpuc->n_added;
1402
1403         if (x86_pmu.put_event_constraints)
1404                 x86_pmu.put_event_constraints(cpuc, event);
1405
1406         /* Delete the array entry. */
1407         while (++i < cpuc->n_events) {
1408                 cpuc->event_list[i-1] = cpuc->event_list[i];
1409                 cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
1410         }
1411         --cpuc->n_events;
1412
1413         perf_event_update_userpage(event);
1414
1415 do_del:
1416         if (x86_pmu.del) {
1417                 /*
1418                  * This is after x86_pmu_stop(); so we disable LBRs after any
1419                  * event can need them etc..
1420                  */
1421                 x86_pmu.del(event);
1422         }
1423 }
1424
1425 int x86_pmu_handle_irq(struct pt_regs *regs)
1426 {
1427         struct perf_sample_data data;
1428         struct cpu_hw_events *cpuc;
1429         struct perf_event *event;
1430         int idx, handled = 0;
1431         u64 val;
1432
1433         cpuc = this_cpu_ptr(&cpu_hw_events);
1434
1435         /*
1436          * Some chipsets need to unmask the LVTPC in a particular spot
1437          * inside the nmi handler.  As a result, the unmasking was pushed
1438          * into all the nmi handlers.
1439          *
1440          * This generic handler doesn't seem to have any issues where the
1441          * unmasking occurs so it was left at the top.
1442          */
1443         apic_write(APIC_LVTPC, APIC_DM_NMI);
1444
1445         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1446                 if (!test_bit(idx, cpuc->active_mask)) {
1447                         /*
1448                          * Though we deactivated the counter some cpus
1449                          * might still deliver spurious interrupts still
1450                          * in flight. Catch them:
1451                          */
1452                         if (__test_and_clear_bit(idx, cpuc->running))
1453                                 handled++;
1454                         continue;
1455                 }
1456
1457                 event = cpuc->events[idx];
1458
1459                 val = x86_perf_event_update(event);
1460                 if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1461                         continue;
1462
1463                 /*
1464                  * event overflow
1465                  */
1466                 handled++;
1467                 perf_sample_data_init(&data, 0, event->hw.last_period);
1468
1469                 if (!x86_perf_event_set_period(event))
1470                         continue;
1471
1472                 if (perf_event_overflow(event, &data, regs))
1473                         x86_pmu_stop(event, 0);
1474         }
1475
1476         if (handled)
1477                 inc_irq_stat(apic_perf_irqs);
1478
1479         return handled;
1480 }
1481
1482 void perf_events_lapic_init(void)
1483 {
1484         if (!x86_pmu.apic || !x86_pmu_initialized())
1485                 return;
1486
1487         /*
1488          * Always use NMI for PMU
1489          */
1490         apic_write(APIC_LVTPC, APIC_DM_NMI);
1491 }
1492
1493 static int
1494 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1495 {
1496         u64 start_clock;
1497         u64 finish_clock;
1498         int ret;
1499
1500         /*
1501          * All PMUs/events that share this PMI handler should make sure to
1502          * increment active_events for their events.
1503          */
1504         if (!atomic_read(&active_events))
1505                 return NMI_DONE;
1506
1507         start_clock = sched_clock();
1508         ret = x86_pmu.handle_irq(regs);
1509         finish_clock = sched_clock();
1510
1511         perf_sample_event_took(finish_clock - start_clock);
1512
1513         return ret;
1514 }
1515 NOKPROBE_SYMBOL(perf_event_nmi_handler);
1516
1517 struct event_constraint emptyconstraint;
1518 struct event_constraint unconstrained;
1519
1520 static int x86_pmu_prepare_cpu(unsigned int cpu)
1521 {
1522         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1523         int i;
1524
1525         for (i = 0 ; i < X86_PERF_KFREE_MAX; i++)
1526                 cpuc->kfree_on_online[i] = NULL;
1527         if (x86_pmu.cpu_prepare)
1528                 return x86_pmu.cpu_prepare(cpu);
1529         return 0;
1530 }
1531
1532 static int x86_pmu_dead_cpu(unsigned int cpu)
1533 {
1534         if (x86_pmu.cpu_dead)
1535                 x86_pmu.cpu_dead(cpu);
1536         return 0;
1537 }
1538
1539 static int x86_pmu_online_cpu(unsigned int cpu)
1540 {
1541         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1542         int i;
1543
1544         for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) {
1545                 kfree(cpuc->kfree_on_online[i]);
1546                 cpuc->kfree_on_online[i] = NULL;
1547         }
1548         return 0;
1549 }
1550
1551 static int x86_pmu_starting_cpu(unsigned int cpu)
1552 {
1553         if (x86_pmu.cpu_starting)
1554                 x86_pmu.cpu_starting(cpu);
1555         return 0;
1556 }
1557
1558 static int x86_pmu_dying_cpu(unsigned int cpu)
1559 {
1560         if (x86_pmu.cpu_dying)
1561                 x86_pmu.cpu_dying(cpu);
1562         return 0;
1563 }
1564
1565 static void __init pmu_check_apic(void)
1566 {
1567         if (boot_cpu_has(X86_FEATURE_APIC))
1568                 return;
1569
1570         x86_pmu.apic = 0;
1571         pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
1572         pr_info("no hardware sampling interrupt available.\n");
1573
1574         /*
1575          * If we have a PMU initialized but no APIC
1576          * interrupts, we cannot sample hardware
1577          * events (user-space has to fall back and
1578          * sample via a hrtimer based software event):
1579          */
1580         pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1581
1582 }
1583
1584 static struct attribute_group x86_pmu_format_group = {
1585         .name = "format",
1586         .attrs = NULL,
1587 };
1588
1589 /*
1590  * Remove all undefined events (x86_pmu.event_map(id) == 0)
1591  * out of events_attr attributes.
1592  */
1593 static void __init filter_events(struct attribute **attrs)
1594 {
1595         struct device_attribute *d;
1596         struct perf_pmu_events_attr *pmu_attr;
1597         int offset = 0;
1598         int i, j;
1599
1600         for (i = 0; attrs[i]; i++) {
1601                 d = (struct device_attribute *)attrs[i];
1602                 pmu_attr = container_of(d, struct perf_pmu_events_attr, attr);
1603                 /* str trumps id */
1604                 if (pmu_attr->event_str)
1605                         continue;
1606                 if (x86_pmu.event_map(i + offset))
1607                         continue;
1608
1609                 for (j = i; attrs[j]; j++)
1610                         attrs[j] = attrs[j + 1];
1611
1612                 /* Check the shifted attr. */
1613                 i--;
1614
1615                 /*
1616                  * event_map() is index based, the attrs array is organized
1617                  * by increasing event index. If we shift the events, then
1618                  * we need to compensate for the event_map(), otherwise
1619                  * we are looking up the wrong event in the map
1620                  */
1621                 offset++;
1622         }
1623 }
1624
1625 /* Merge two pointer arrays */
1626 __init struct attribute **merge_attr(struct attribute **a, struct attribute **b)
1627 {
1628         struct attribute **new;
1629         int j, i;
1630
1631         for (j = 0; a[j]; j++)
1632                 ;
1633         for (i = 0; b[i]; i++)
1634                 j++;
1635         j++;
1636
1637         new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL);
1638         if (!new)
1639                 return NULL;
1640
1641         j = 0;
1642         for (i = 0; a[i]; i++)
1643                 new[j++] = a[i];
1644         for (i = 0; b[i]; i++)
1645                 new[j++] = b[i];
1646         new[j] = NULL;
1647
1648         return new;
1649 }
1650
1651 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
1652 {
1653         struct perf_pmu_events_attr *pmu_attr = \
1654                 container_of(attr, struct perf_pmu_events_attr, attr);
1655         u64 config = x86_pmu.event_map(pmu_attr->id);
1656
1657         /* string trumps id */
1658         if (pmu_attr->event_str)
1659                 return sprintf(page, "%s", pmu_attr->event_str);
1660
1661         return x86_pmu.events_sysfs_show(page, config);
1662 }
1663 EXPORT_SYMBOL_GPL(events_sysfs_show);
1664
1665 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1666                           char *page)
1667 {
1668         struct perf_pmu_events_ht_attr *pmu_attr =
1669                 container_of(attr, struct perf_pmu_events_ht_attr, attr);
1670
1671         /*
1672          * Report conditional events depending on Hyper-Threading.
1673          *
1674          * This is overly conservative as usually the HT special
1675          * handling is not needed if the other CPU thread is idle.
1676          *
1677          * Note this does not (and cannot) handle the case when thread
1678          * siblings are invisible, for example with virtualization
1679          * if they are owned by some other guest.  The user tool
1680          * has to re-read when a thread sibling gets onlined later.
1681          */
1682         return sprintf(page, "%s",
1683                         topology_max_smt_threads() > 1 ?
1684                         pmu_attr->event_str_ht :
1685                         pmu_attr->event_str_noht);
1686 }
1687
1688 EVENT_ATTR(cpu-cycles,                  CPU_CYCLES              );
1689 EVENT_ATTR(instructions,                INSTRUCTIONS            );
1690 EVENT_ATTR(cache-references,            CACHE_REFERENCES        );
1691 EVENT_ATTR(cache-misses,                CACHE_MISSES            );
1692 EVENT_ATTR(branch-instructions,         BRANCH_INSTRUCTIONS     );
1693 EVENT_ATTR(branch-misses,               BRANCH_MISSES           );
1694 EVENT_ATTR(bus-cycles,                  BUS_CYCLES              );
1695 EVENT_ATTR(stalled-cycles-frontend,     STALLED_CYCLES_FRONTEND );
1696 EVENT_ATTR(stalled-cycles-backend,      STALLED_CYCLES_BACKEND  );
1697 EVENT_ATTR(ref-cycles,                  REF_CPU_CYCLES          );
1698
1699 static struct attribute *empty_attrs;
1700
1701 static struct attribute *events_attr[] = {
1702         EVENT_PTR(CPU_CYCLES),
1703         EVENT_PTR(INSTRUCTIONS),
1704         EVENT_PTR(CACHE_REFERENCES),
1705         EVENT_PTR(CACHE_MISSES),
1706         EVENT_PTR(BRANCH_INSTRUCTIONS),
1707         EVENT_PTR(BRANCH_MISSES),
1708         EVENT_PTR(BUS_CYCLES),
1709         EVENT_PTR(STALLED_CYCLES_FRONTEND),
1710         EVENT_PTR(STALLED_CYCLES_BACKEND),
1711         EVENT_PTR(REF_CPU_CYCLES),
1712         NULL,
1713 };
1714
1715 static struct attribute_group x86_pmu_events_group = {
1716         .name = "events",
1717         .attrs = events_attr,
1718 };
1719
1720 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
1721 {
1722         u64 umask  = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
1723         u64 cmask  = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
1724         bool edge  = (config & ARCH_PERFMON_EVENTSEL_EDGE);
1725         bool pc    = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
1726         bool any   = (config & ARCH_PERFMON_EVENTSEL_ANY);
1727         bool inv   = (config & ARCH_PERFMON_EVENTSEL_INV);
1728         ssize_t ret;
1729
1730         /*
1731         * We have whole page size to spend and just little data
1732         * to write, so we can safely use sprintf.
1733         */
1734         ret = sprintf(page, "event=0x%02llx", event);
1735
1736         if (umask)
1737                 ret += sprintf(page + ret, ",umask=0x%02llx", umask);
1738
1739         if (edge)
1740                 ret += sprintf(page + ret, ",edge");
1741
1742         if (pc)
1743                 ret += sprintf(page + ret, ",pc");
1744
1745         if (any)
1746                 ret += sprintf(page + ret, ",any");
1747
1748         if (inv)
1749                 ret += sprintf(page + ret, ",inv");
1750
1751         if (cmask)
1752                 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);
1753
1754         ret += sprintf(page + ret, "\n");
1755
1756         return ret;
1757 }
1758
1759 static struct attribute_group x86_pmu_attr_group;
1760 static struct attribute_group x86_pmu_caps_group;
1761
1762 static int __init init_hw_perf_events(void)
1763 {
1764         struct x86_pmu_quirk *quirk;
1765         int err;
1766
1767         pr_info("Performance Events: ");
1768
1769         switch (boot_cpu_data.x86_vendor) {
1770         case X86_VENDOR_INTEL:
1771                 err = intel_pmu_init();
1772                 break;
1773         case X86_VENDOR_AMD:
1774                 err = amd_pmu_init();
1775                 break;
1776         default:
1777                 err = -ENOTSUPP;
1778         }
1779         if (err != 0) {
1780                 pr_cont("no PMU driver, software events only.\n");
1781                 return 0;
1782         }
1783
1784         pmu_check_apic();
1785
1786         /* sanity check that the hardware exists or is emulated */
1787         if (!check_hw_exists())
1788                 return 0;
1789
1790         pr_cont("%s PMU driver.\n", x86_pmu.name);
1791
1792         x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
1793
1794         for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
1795                 quirk->func();
1796
1797         if (!x86_pmu.intel_ctrl)
1798                 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
1799
1800         perf_events_lapic_init();
1801         register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
1802
1803         unconstrained = (struct event_constraint)
1804                 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
1805                                    0, x86_pmu.num_counters, 0, 0);
1806
1807         x86_pmu_format_group.attrs = x86_pmu.format_attrs;
1808
1809         if (x86_pmu.caps_attrs) {
1810                 struct attribute **tmp;
1811
1812                 tmp = merge_attr(x86_pmu_caps_group.attrs, x86_pmu.caps_attrs);
1813                 if (!WARN_ON(!tmp))
1814                         x86_pmu_caps_group.attrs = tmp;
1815         }
1816
1817         if (x86_pmu.event_attrs)
1818                 x86_pmu_events_group.attrs = x86_pmu.event_attrs;
1819
1820         if (!x86_pmu.events_sysfs_show)
1821                 x86_pmu_events_group.attrs = &empty_attrs;
1822         else
1823                 filter_events(x86_pmu_events_group.attrs);
1824
1825         if (x86_pmu.cpu_events) {
1826                 struct attribute **tmp;
1827
1828                 tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events);
1829                 if (!WARN_ON(!tmp))
1830                         x86_pmu_events_group.attrs = tmp;
1831         }
1832
1833         if (x86_pmu.attrs) {
1834                 struct attribute **tmp;
1835
1836                 tmp = merge_attr(x86_pmu_attr_group.attrs, x86_pmu.attrs);
1837                 if (!WARN_ON(!tmp))
1838                         x86_pmu_attr_group.attrs = tmp;
1839         }
1840
1841         pr_info("... version:                %d\n",     x86_pmu.version);
1842         pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);
1843         pr_info("... generic registers:      %d\n",     x86_pmu.num_counters);
1844         pr_info("... value mask:             %016Lx\n", x86_pmu.cntval_mask);
1845         pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
1846         pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_counters_fixed);
1847         pr_info("... event mask:             %016Lx\n", x86_pmu.intel_ctrl);
1848
1849         /*
1850          * Install callbacks. Core will call them for each online
1851          * cpu.
1852          */
1853         err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare",
1854                                 x86_pmu_prepare_cpu, x86_pmu_dead_cpu);
1855         if (err)
1856                 return err;
1857
1858         err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING,
1859                                 "perf/x86:starting", x86_pmu_starting_cpu,
1860                                 x86_pmu_dying_cpu);
1861         if (err)
1862                 goto out;
1863
1864         err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online",
1865                                 x86_pmu_online_cpu, NULL);
1866         if (err)
1867                 goto out1;
1868
1869         err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1870         if (err)
1871                 goto out2;
1872
1873         return 0;
1874
1875 out2:
1876         cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE);
1877 out1:
1878         cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING);
1879 out:
1880         cpuhp_remove_state(CPUHP_PERF_X86_PREPARE);
1881         return err;
1882 }
1883 early_initcall(init_hw_perf_events);
1884
1885 static inline void x86_pmu_read(struct perf_event *event)
1886 {
1887         x86_perf_event_update(event);
1888 }
1889
1890 /*
1891  * Start group events scheduling transaction
1892  * Set the flag to make pmu::enable() not perform the
1893  * schedulability test, it will be performed at commit time
1894  *
1895  * We only support PERF_PMU_TXN_ADD transactions. Save the
1896  * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
1897  * transactions.
1898  */
1899 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
1900 {
1901         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1902
1903         WARN_ON_ONCE(cpuc->txn_flags);          /* txn already in flight */
1904
1905         cpuc->txn_flags = txn_flags;
1906         if (txn_flags & ~PERF_PMU_TXN_ADD)
1907                 return;
1908
1909         perf_pmu_disable(pmu);
1910         __this_cpu_write(cpu_hw_events.n_txn, 0);
1911 }
1912
1913 /*
1914  * Stop group events scheduling transaction
1915  * Clear the flag and pmu::enable() will perform the
1916  * schedulability test.
1917  */
1918 static void x86_pmu_cancel_txn(struct pmu *pmu)
1919 {
1920         unsigned int txn_flags;
1921         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1922
1923         WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1924
1925         txn_flags = cpuc->txn_flags;
1926         cpuc->txn_flags = 0;
1927         if (txn_flags & ~PERF_PMU_TXN_ADD)
1928                 return;
1929
1930         /*
1931          * Truncate collected array by the number of events added in this
1932          * transaction. See x86_pmu_add() and x86_pmu_*_txn().
1933          */
1934         __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
1935         __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
1936         perf_pmu_enable(pmu);
1937 }
1938
1939 /*
1940  * Commit group events scheduling transaction
1941  * Perform the group schedulability test as a whole
1942  * Return 0 if success
1943  *
1944  * Does not cancel the transaction on failure; expects the caller to do this.
1945  */
1946 static int x86_pmu_commit_txn(struct pmu *pmu)
1947 {
1948         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1949         int assign[X86_PMC_IDX_MAX];
1950         int n, ret;
1951
1952         WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1953
1954         if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
1955                 cpuc->txn_flags = 0;
1956                 return 0;
1957         }
1958
1959         n = cpuc->n_events;
1960
1961         if (!x86_pmu_initialized())
1962                 return -EAGAIN;
1963
1964         ret = x86_pmu.schedule_events(cpuc, n, assign);
1965         if (ret)
1966                 return ret;
1967
1968         /*
1969          * copy new assignment, now we know it is possible
1970          * will be used by hw_perf_enable()
1971          */
1972         memcpy(cpuc->assign, assign, n*sizeof(int));
1973
1974         cpuc->txn_flags = 0;
1975         perf_pmu_enable(pmu);
1976         return 0;
1977 }
1978 /*
1979  * a fake_cpuc is used to validate event groups. Due to
1980  * the extra reg logic, we need to also allocate a fake
1981  * per_core and per_cpu structure. Otherwise, group events
1982  * using extra reg may conflict without the kernel being
1983  * able to catch this when the last event gets added to
1984  * the group.
1985  */
1986 static void free_fake_cpuc(struct cpu_hw_events *cpuc)
1987 {
1988         kfree(cpuc->shared_regs);
1989         kfree(cpuc);
1990 }
1991
1992 static struct cpu_hw_events *allocate_fake_cpuc(void)
1993 {
1994         struct cpu_hw_events *cpuc;
1995         int cpu = raw_smp_processor_id();
1996
1997         cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
1998         if (!cpuc)
1999                 return ERR_PTR(-ENOMEM);
2000
2001         /* only needed, if we have extra_regs */
2002         if (x86_pmu.extra_regs) {
2003                 cpuc->shared_regs = allocate_shared_regs(cpu);
2004                 if (!cpuc->shared_regs)
2005                         goto error;
2006         }
2007         cpuc->is_fake = 1;
2008         return cpuc;
2009 error:
2010         free_fake_cpuc(cpuc);
2011         return ERR_PTR(-ENOMEM);
2012 }
2013
2014 /*
2015  * validate that we can schedule this event
2016  */
2017 static int validate_event(struct perf_event *event)
2018 {
2019         struct cpu_hw_events *fake_cpuc;
2020         struct event_constraint *c;
2021         int ret = 0;
2022
2023         fake_cpuc = allocate_fake_cpuc();
2024         if (IS_ERR(fake_cpuc))
2025                 return PTR_ERR(fake_cpuc);
2026
2027         c = x86_pmu.get_event_constraints(fake_cpuc, -1, event);
2028
2029         if (!c || !c->weight)
2030                 ret = -EINVAL;
2031
2032         if (x86_pmu.put_event_constraints)
2033                 x86_pmu.put_event_constraints(fake_cpuc, event);
2034
2035         free_fake_cpuc(fake_cpuc);
2036
2037         return ret;
2038 }
2039
2040 /*
2041  * validate a single event group
2042  *
2043  * validation include:
2044  *      - check events are compatible which each other
2045  *      - events do not compete for the same counter
2046  *      - number of events <= number of counters
2047  *
2048  * validation ensures the group can be loaded onto the
2049  * PMU if it was the only group available.
2050  */
2051 static int validate_group(struct perf_event *event)
2052 {
2053         struct perf_event *leader = event->group_leader;
2054         struct cpu_hw_events *fake_cpuc;
2055         int ret = -EINVAL, n;
2056
2057         fake_cpuc = allocate_fake_cpuc();
2058         if (IS_ERR(fake_cpuc))
2059                 return PTR_ERR(fake_cpuc);
2060         /*
2061          * the event is not yet connected with its
2062          * siblings therefore we must first collect
2063          * existing siblings, then add the new event
2064          * before we can simulate the scheduling
2065          */
2066         n = collect_events(fake_cpuc, leader, true);
2067         if (n < 0)
2068                 goto out;
2069
2070         fake_cpuc->n_events = n;
2071         n = collect_events(fake_cpuc, event, false);
2072         if (n < 0)
2073                 goto out;
2074
2075         fake_cpuc->n_events = n;
2076
2077         ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
2078
2079 out:
2080         free_fake_cpuc(fake_cpuc);
2081         return ret;
2082 }
2083
2084 static int x86_pmu_event_init(struct perf_event *event)
2085 {
2086         struct pmu *tmp;
2087         int err;
2088
2089         switch (event->attr.type) {
2090         case PERF_TYPE_RAW:
2091         case PERF_TYPE_HARDWARE:
2092         case PERF_TYPE_HW_CACHE:
2093                 break;
2094
2095         default:
2096                 return -ENOENT;
2097         }
2098
2099         err = __x86_pmu_event_init(event);
2100         if (!err) {
2101                 /*
2102                  * we temporarily connect event to its pmu
2103                  * such that validate_group() can classify
2104                  * it as an x86 event using is_x86_event()
2105                  */
2106                 tmp = event->pmu;
2107                 event->pmu = &pmu;
2108
2109                 if (event->group_leader != event)
2110                         err = validate_group(event);
2111                 else
2112                         err = validate_event(event);
2113
2114                 event->pmu = tmp;
2115         }
2116         if (err) {
2117                 if (event->destroy)
2118                         event->destroy(event);
2119         }
2120
2121         if (ACCESS_ONCE(x86_pmu.attr_rdpmc))
2122                 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;
2123
2124         return err;
2125 }
2126
2127 static void refresh_pce(void *ignored)
2128 {
2129         load_mm_cr4(this_cpu_read(cpu_tlbstate.loaded_mm));
2130 }
2131
2132 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
2133 {
2134         if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2135                 return;
2136
2137         /*
2138          * This function relies on not being called concurrently in two
2139          * tasks in the same mm.  Otherwise one task could observe
2140          * perf_rdpmc_allowed > 1 and return all the way back to
2141          * userspace with CR4.PCE clear while another task is still
2142          * doing on_each_cpu_mask() to propagate CR4.PCE.
2143          *
2144          * For now, this can't happen because all callers hold mmap_sem
2145          * for write.  If this changes, we'll need a different solution.
2146          */
2147         lockdep_assert_held_exclusive(&mm->mmap_sem);
2148
2149         if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
2150                 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2151 }
2152
2153 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
2154 {
2155
2156         if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2157                 return;
2158
2159         if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
2160                 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2161 }
2162
2163 static int x86_pmu_event_idx(struct perf_event *event)
2164 {
2165         int idx = event->hw.idx;
2166
2167         if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2168                 return 0;
2169
2170         if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
2171                 idx -= INTEL_PMC_IDX_FIXED;
2172                 idx |= 1 << 30;
2173         }
2174
2175         return idx + 1;
2176 }
2177
2178 static ssize_t get_attr_rdpmc(struct device *cdev,
2179                               struct device_attribute *attr,
2180                               char *buf)
2181 {
2182         return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
2183 }
2184
2185 static ssize_t set_attr_rdpmc(struct device *cdev,
2186                               struct device_attribute *attr,
2187                               const char *buf, size_t count)
2188 {
2189         unsigned long val;
2190         ssize_t ret;
2191
2192         ret = kstrtoul(buf, 0, &val);
2193         if (ret)
2194                 return ret;
2195
2196         if (val > 2)
2197                 return -EINVAL;
2198
2199         if (x86_pmu.attr_rdpmc_broken)
2200                 return -ENOTSUPP;
2201
2202         if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) {
2203                 /*
2204                  * Changing into or out of always available, aka
2205                  * perf-event-bypassing mode.  This path is extremely slow,
2206                  * but only root can trigger it, so it's okay.
2207                  */
2208                 if (val == 2)
2209                         static_key_slow_inc(&rdpmc_always_available);
2210                 else
2211                         static_key_slow_dec(&rdpmc_always_available);
2212                 on_each_cpu(refresh_pce, NULL, 1);
2213         }
2214
2215         x86_pmu.attr_rdpmc = val;
2216
2217         return count;
2218 }
2219
2220 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
2221
2222 static struct attribute *x86_pmu_attrs[] = {
2223         &dev_attr_rdpmc.attr,
2224         NULL,
2225 };
2226
2227 static struct attribute_group x86_pmu_attr_group = {
2228         .attrs = x86_pmu_attrs,
2229 };
2230
2231 static ssize_t max_precise_show(struct device *cdev,
2232                                   struct device_attribute *attr,
2233                                   char *buf)
2234 {
2235         return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
2236 }
2237
2238 static DEVICE_ATTR_RO(max_precise);
2239
2240 static struct attribute *x86_pmu_caps_attrs[] = {
2241         &dev_attr_max_precise.attr,
2242         NULL
2243 };
2244
2245 static struct attribute_group x86_pmu_caps_group = {
2246         .name = "caps",
2247         .attrs = x86_pmu_caps_attrs,
2248 };
2249
2250 static const struct attribute_group *x86_pmu_attr_groups[] = {
2251         &x86_pmu_attr_group,
2252         &x86_pmu_format_group,
2253         &x86_pmu_events_group,
2254         &x86_pmu_caps_group,
2255         NULL,
2256 };
2257
2258 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
2259 {
2260         if (x86_pmu.sched_task)
2261                 x86_pmu.sched_task(ctx, sched_in);
2262 }
2263
2264 void perf_check_microcode(void)
2265 {
2266         if (x86_pmu.check_microcode)
2267                 x86_pmu.check_microcode();
2268 }
2269
2270 static struct pmu pmu = {
2271         .pmu_enable             = x86_pmu_enable,
2272         .pmu_disable            = x86_pmu_disable,
2273
2274         .attr_groups            = x86_pmu_attr_groups,
2275
2276         .event_init             = x86_pmu_event_init,
2277
2278         .event_mapped           = x86_pmu_event_mapped,
2279         .event_unmapped         = x86_pmu_event_unmapped,
2280
2281         .add                    = x86_pmu_add,
2282         .del                    = x86_pmu_del,
2283         .start                  = x86_pmu_start,
2284         .stop                   = x86_pmu_stop,
2285         .read                   = x86_pmu_read,
2286
2287         .start_txn              = x86_pmu_start_txn,
2288         .cancel_txn             = x86_pmu_cancel_txn,
2289         .commit_txn             = x86_pmu_commit_txn,
2290
2291         .event_idx              = x86_pmu_event_idx,
2292         .sched_task             = x86_pmu_sched_task,
2293         .task_ctx_size          = sizeof(struct x86_perf_task_context),
2294 };
2295
2296 void arch_perf_update_userpage(struct perf_event *event,
2297                                struct perf_event_mmap_page *userpg, u64 now)
2298 {
2299         struct cyc2ns_data data;
2300         u64 offset;
2301
2302         userpg->cap_user_time = 0;
2303         userpg->cap_user_time_zero = 0;
2304         userpg->cap_user_rdpmc =
2305                 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
2306         userpg->pmc_width = x86_pmu.cntval_bits;
2307
2308         if (!using_native_sched_clock() || !sched_clock_stable())
2309                 return;
2310
2311         cyc2ns_read_begin(&data);
2312
2313         offset = data.cyc2ns_offset + __sched_clock_offset;
2314
2315         /*
2316          * Internal timekeeping for enabled/running/stopped times
2317          * is always in the local_clock domain.
2318          */
2319         userpg->cap_user_time = 1;
2320         userpg->time_mult = data.cyc2ns_mul;
2321         userpg->time_shift = data.cyc2ns_shift;
2322         userpg->time_offset = offset - now;
2323
2324         /*
2325          * cap_user_time_zero doesn't make sense when we're using a different
2326          * time base for the records.
2327          */
2328         if (!event->attr.use_clockid) {
2329                 userpg->cap_user_time_zero = 1;
2330                 userpg->time_zero = offset;
2331         }
2332
2333         cyc2ns_read_end();
2334 }
2335
2336 void
2337 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2338 {
2339         struct unwind_state state;
2340         unsigned long addr;
2341
2342         if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2343                 /* TODO: We don't support guest os callchain now */
2344                 return;
2345         }
2346
2347         if (perf_callchain_store(entry, regs->ip))
2348                 return;
2349
2350         for (unwind_start(&state, current, regs, NULL); !unwind_done(&state);
2351              unwind_next_frame(&state)) {
2352                 addr = unwind_get_return_address(&state);
2353                 if (!addr || perf_callchain_store(entry, addr))
2354                         return;
2355         }
2356 }
2357
2358 static inline int
2359 valid_user_frame(const void __user *fp, unsigned long size)
2360 {
2361         return (__range_not_ok(fp, size, TASK_SIZE) == 0);
2362 }
2363
2364 static unsigned long get_segment_base(unsigned int segment)
2365 {
2366         struct desc_struct *desc;
2367         unsigned int idx = segment >> 3;
2368
2369         if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2370 #ifdef CONFIG_MODIFY_LDT_SYSCALL
2371                 struct ldt_struct *ldt;
2372
2373                 /* IRQs are off, so this synchronizes with smp_store_release */
2374                 ldt = READ_ONCE(current->active_mm->context.ldt);
2375                 if (!ldt || idx >= ldt->nr_entries)
2376                         return 0;
2377
2378                 desc = &ldt->entries[idx];
2379 #else
2380                 return 0;
2381 #endif
2382         } else {
2383                 if (idx >= GDT_ENTRIES)
2384                         return 0;
2385
2386                 desc = raw_cpu_ptr(gdt_page.gdt) + idx;
2387         }
2388
2389         return get_desc_base(desc);
2390 }
2391
2392 #ifdef CONFIG_IA32_EMULATION
2393
2394 #include <asm/compat.h>
2395
2396 static inline int
2397 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2398 {
2399         /* 32-bit process in 64-bit kernel. */
2400         unsigned long ss_base, cs_base;
2401         struct stack_frame_ia32 frame;
2402         const void __user *fp;
2403
2404         if (!test_thread_flag(TIF_IA32))
2405                 return 0;
2406
2407         cs_base = get_segment_base(regs->cs);
2408         ss_base = get_segment_base(regs->ss);
2409
2410         fp = compat_ptr(ss_base + regs->bp);
2411         pagefault_disable();
2412         while (entry->nr < entry->max_stack) {
2413                 unsigned long bytes;
2414                 frame.next_frame     = 0;
2415                 frame.return_address = 0;
2416
2417                 if (!valid_user_frame(fp, sizeof(frame)))
2418                         break;
2419
2420                 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4);
2421                 if (bytes != 0)
2422                         break;
2423                 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4);
2424                 if (bytes != 0)
2425                         break;
2426
2427                 perf_callchain_store(entry, cs_base + frame.return_address);
2428                 fp = compat_ptr(ss_base + frame.next_frame);
2429         }
2430         pagefault_enable();
2431         return 1;
2432 }
2433 #else
2434 static inline int
2435 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2436 {
2437     return 0;
2438 }
2439 #endif
2440
2441 void
2442 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2443 {
2444         struct stack_frame frame;
2445         const unsigned long __user *fp;
2446
2447         if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2448                 /* TODO: We don't support guest os callchain now */
2449                 return;
2450         }
2451
2452         /*
2453          * We don't know what to do with VM86 stacks.. ignore them for now.
2454          */
2455         if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
2456                 return;
2457
2458         fp = (unsigned long __user *)regs->bp;
2459
2460         perf_callchain_store(entry, regs->ip);
2461
2462         if (!current->mm)
2463                 return;
2464
2465         if (perf_callchain_user32(regs, entry))
2466                 return;
2467
2468         pagefault_disable();
2469         while (entry->nr < entry->max_stack) {
2470                 unsigned long bytes;
2471
2472                 frame.next_frame             = NULL;
2473                 frame.return_address = 0;
2474
2475                 if (!valid_user_frame(fp, sizeof(frame)))
2476                         break;
2477
2478                 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp));
2479                 if (bytes != 0)
2480                         break;
2481                 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp));
2482                 if (bytes != 0)
2483                         break;
2484
2485                 perf_callchain_store(entry, frame.return_address);
2486                 fp = (void __user *)frame.next_frame;
2487         }
2488         pagefault_enable();
2489 }
2490
2491 /*
2492  * Deal with code segment offsets for the various execution modes:
2493  *
2494  *   VM86 - the good olde 16 bit days, where the linear address is
2495  *          20 bits and we use regs->ip + 0x10 * regs->cs.
2496  *
2497  *   IA32 - Where we need to look at GDT/LDT segment descriptor tables
2498  *          to figure out what the 32bit base address is.
2499  *
2500  *    X32 - has TIF_X32 set, but is running in x86_64
2501  *
2502  * X86_64 - CS,DS,SS,ES are all zero based.
2503  */
2504 static unsigned long code_segment_base(struct pt_regs *regs)
2505 {
2506         /*
2507          * For IA32 we look at the GDT/LDT segment base to convert the
2508          * effective IP to a linear address.
2509          */
2510
2511 #ifdef CONFIG_X86_32
2512         /*
2513          * If we are in VM86 mode, add the segment offset to convert to a
2514          * linear address.
2515          */
2516         if (regs->flags & X86_VM_MASK)
2517                 return 0x10 * regs->cs;
2518
2519         if (user_mode(regs) && regs->cs != __USER_CS)
2520                 return get_segment_base(regs->cs);
2521 #else
2522         if (user_mode(regs) && !user_64bit_mode(regs) &&
2523             regs->cs != __USER32_CS)
2524                 return get_segment_base(regs->cs);
2525 #endif
2526         return 0;
2527 }
2528
2529 unsigned long perf_instruction_pointer(struct pt_regs *regs)
2530 {
2531         if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
2532                 return perf_guest_cbs->get_guest_ip();
2533
2534         return regs->ip + code_segment_base(regs);
2535 }
2536
2537 unsigned long perf_misc_flags(struct pt_regs *regs)
2538 {
2539         int misc = 0;
2540
2541         if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2542                 if (perf_guest_cbs->is_user_mode())
2543                         misc |= PERF_RECORD_MISC_GUEST_USER;
2544                 else
2545                         misc |= PERF_RECORD_MISC_GUEST_KERNEL;
2546         } else {
2547                 if (user_mode(regs))
2548                         misc |= PERF_RECORD_MISC_USER;
2549                 else
2550                         misc |= PERF_RECORD_MISC_KERNEL;
2551         }
2552
2553         if (regs->flags & PERF_EFLAGS_EXACT)
2554                 misc |= PERF_RECORD_MISC_EXACT_IP;
2555
2556         return misc;
2557 }
2558
2559 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
2560 {
2561         cap->version            = x86_pmu.version;
2562         cap->num_counters_gp    = x86_pmu.num_counters;
2563         cap->num_counters_fixed = x86_pmu.num_counters_fixed;
2564         cap->bit_width_gp       = x86_pmu.cntval_bits;
2565         cap->bit_width_fixed    = x86_pmu.cntval_bits;
2566         cap->events_mask        = (unsigned int)x86_pmu.events_maskl;
2567         cap->events_mask_len    = x86_pmu.events_mask_len;
2568 }
2569 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);