drivers/gpu/drm/i915/i915_pmu.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2017-2018 Intel Corporation
   5  */
   6
   7 #include <linux/pm_runtime.h>
   8
   9 #include "gt/intel_engine.h"
  10 #include "gt/intel_engine_pm.h"
  11 #include "gt/intel_engine_regs.h"
  12 #include "gt/intel_engine_user.h"
  13 #include "gt/intel_gt.h"
  14 #include "gt/intel_gt_pm.h"
  15 #include "gt/intel_gt_regs.h"
  16 #include "gt/intel_rc6.h"
  17 #include "gt/intel_rps.h"
  18
  19 #include "i915_drv.h"
  20 #include "i915_pmu.h"
  21
  22 /* Frequency for the sampling timer for events which need it. */
  23 #define FREQUENCY 200
  24 #define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)
  25
  26 #define ENGINE_SAMPLE_MASK \
  27         (BIT(I915_SAMPLE_BUSY) | \
  28          BIT(I915_SAMPLE_WAIT) | \
  29          BIT(I915_SAMPLE_SEMA))
  30
  31 static cpumask_t i915_pmu_cpumask;
  32 static unsigned int i915_pmu_target_cpu = -1;
  33
  34 static u8 engine_config_sample(u64 config)
  35 {
  36         return config & I915_PMU_SAMPLE_MASK;
  37 }
  38
  39 static u8 engine_event_sample(struct perf_event *event)
  40 {
  41         return engine_config_sample(event->attr.config);
  42 }
  43
  44 static u8 engine_event_class(struct perf_event *event)
  45 {
  46         return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
  47 }
  48
  49 static u8 engine_event_instance(struct perf_event *event)
  50 {
  51         return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
  52 }
  53
  54 static bool is_engine_config(const u64 config)
  55 {
  56         return config < __I915_PMU_OTHER(0);
  57 }
  58
  59 static unsigned int config_gt_id(const u64 config)
  60 {
  61         return config >> __I915_PMU_GT_SHIFT;
  62 }
  63
  64 static u64 config_counter(const u64 config)
  65 {
  66         return config & ~(~0ULL << __I915_PMU_GT_SHIFT);
  67 }
  68
  69 static unsigned int other_bit(const u64 config)
  70 {
  71         unsigned int val;
  72
  73         switch (config_counter(config)) {
  74         case I915_PMU_ACTUAL_FREQUENCY:
  75                 val =  __I915_PMU_ACTUAL_FREQUENCY_ENABLED;
  76                 break;
  77         case I915_PMU_REQUESTED_FREQUENCY:
  78                 val = __I915_PMU_REQUESTED_FREQUENCY_ENABLED;
  79                 break;
  80         case I915_PMU_RC6_RESIDENCY:
  81                 val = __I915_PMU_RC6_RESIDENCY_ENABLED;
  82                 break;
  83         default:
  84                 /*
  85                  * Events that do not require sampling, or tracking state
  86                  * transitions between enabled and disabled can be ignored.
  87                  */
  88                 return -1;
  89         }
  90
  91         return I915_ENGINE_SAMPLE_COUNT +
  92                config_gt_id(config) * __I915_PMU_TRACKED_EVENT_COUNT +
  93                val;
  94 }
  95
  96 static unsigned int config_bit(const u64 config)
  97 {
  98         if (is_engine_config(config))
  99                 return engine_config_sample(config);
 100         else
 101                 return other_bit(config);
 102 }
 103
 104 static u32 config_mask(const u64 config)
 105 {
 106         unsigned int bit = config_bit(config);
 107
 108         if (__builtin_constant_p(config))
 109                 BUILD_BUG_ON(bit >
 110                              BITS_PER_TYPE(typeof_member(struct i915_pmu,
 111                                                          enable)) - 1);
 112         else
 113                 WARN_ON_ONCE(bit >
 114                              BITS_PER_TYPE(typeof_member(struct i915_pmu,
 115                                                          enable)) - 1);
 116
 117         return BIT(config_bit(config));
 118 }
 119
 120 static bool is_engine_event(struct perf_event *event)
 121 {
 122         return is_engine_config(event->attr.config);
 123 }
 124
 125 static unsigned int event_bit(struct perf_event *event)
 126 {
 127         return config_bit(event->attr.config);
 128 }
 129
 130 static u32 frequency_enabled_mask(void)
 131 {
 132         unsigned int i;
 133         u32 mask = 0;
 134
 135         for (i = 0; i < I915_PMU_MAX_GT; i++)
 136                 mask |= config_mask(__I915_PMU_ACTUAL_FREQUENCY(i)) |
 137                         config_mask(__I915_PMU_REQUESTED_FREQUENCY(i));
 138
 139         return mask;
 140 }
 141
 142 static bool pmu_needs_timer(struct i915_pmu *pmu)
 143 {
 144         struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu);
 145         u32 enable;
 146
 147         /*
 148          * Only some counters need the sampling timer.
 149          *
 150          * We start with a bitmask of all currently enabled events.
 151          */
 152         enable = pmu->enable;
 153
 154         /*
 155          * Mask out all the ones which do not need the timer, or in
 156          * other words keep all the ones that could need the timer.
 157          */
 158         enable &= frequency_enabled_mask() | ENGINE_SAMPLE_MASK;
 159
 160         /*
 161          * Also there is software busyness tracking available we do not
 162          * need the timer for I915_SAMPLE_BUSY counter.
 163          */
 164         if (i915->caps.scheduler & I915_SCHEDULER_CAP_ENGINE_BUSY_STATS)
 165                 enable &= ~BIT(I915_SAMPLE_BUSY);
 166
 167         /*
 168          * If some bits remain it means we need the sampling timer running.
 169          */
 170         return enable;
 171 }
 172
 173 static u64 __get_rc6(struct intel_gt *gt)
 174 {
 175         struct drm_i915_private *i915 = gt->i915;
 176         u64 val;
 177
 178         val = intel_rc6_residency_ns(&gt->rc6, INTEL_RC6_RES_RC6);
 179
 180         if (HAS_RC6p(i915))
 181                 val += intel_rc6_residency_ns(&gt->rc6, INTEL_RC6_RES_RC6p);
 182
 183         if (HAS_RC6pp(i915))
 184                 val += intel_rc6_residency_ns(&gt->rc6, INTEL_RC6_RES_RC6pp);
 185
 186         return val;
 187 }
 188
 189 static inline s64 ktime_since_raw(const ktime_t kt)
 190 {
 191         return ktime_to_ns(ktime_sub(ktime_get_raw(), kt));
 192 }
 193
 194 static u64 read_sample(struct i915_pmu *pmu, unsigned int gt_id, int sample)
 195 {
 196         return pmu->sample[gt_id][sample].cur;
 197 }
 198
 199 static void
 200 store_sample(struct i915_pmu *pmu, unsigned int gt_id, int sample, u64 val)
 201 {
 202         pmu->sample[gt_id][sample].cur = val;
 203 }
 204
 205 static void
 206 add_sample_mult(struct i915_pmu *pmu, unsigned int gt_id, int sample, u32 val, u32 mul)
 207 {
 208         pmu->sample[gt_id][sample].cur += mul_u32_u32(val, mul);
 209 }
 210
 211 static u64 get_rc6(struct intel_gt *gt)
 212 {
 213         struct drm_i915_private *i915 = gt->i915;
 214         const unsigned int gt_id = gt->info.id;
 215         struct i915_pmu *pmu = &i915->pmu;
 216         unsigned long flags;
 217         bool awake = false;
 218         u64 val;
 219
 220         if (intel_gt_pm_get_if_awake(gt)) {
 221                 val = __get_rc6(gt);
 222                 intel_gt_pm_put_async(gt);
 223                 awake = true;
 224         }
 225
 226         spin_lock_irqsave(&pmu->lock, flags);
 227
 228         if (awake) {
 229                 store_sample(pmu, gt_id, __I915_SAMPLE_RC6, val);
 230         } else {
 231                 /*
 232                  * We think we are runtime suspended.
 233                  *
 234                  * Report the delta from when the device was suspended to now,
 235                  * on top of the last known real value, as the approximated RC6
 236                  * counter value.
 237                  */
 238                 val = ktime_since_raw(pmu->sleep_last[gt_id]);
 239                 val += read_sample(pmu, gt_id, __I915_SAMPLE_RC6);
 240         }
 241
 242         if (val < read_sample(pmu, gt_id, __I915_SAMPLE_RC6_LAST_REPORTED))
 243                 val = read_sample(pmu, gt_id, __I915_SAMPLE_RC6_LAST_REPORTED);
 244         else
 245                 store_sample(pmu, gt_id, __I915_SAMPLE_RC6_LAST_REPORTED, val);
 246
 247         spin_unlock_irqrestore(&pmu->lock, flags);
 248
 249         return val;
 250 }
 251
 252 static void init_rc6(struct i915_pmu *pmu)
 253 {
 254         struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu);
 255         struct intel_gt *gt;
 256         unsigned int i;
 257
 258         for_each_gt(gt, i915, i) {
 259                 intel_wakeref_t wakeref;
 260
 261                 with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
 262                         u64 val = __get_rc6(gt);
 263
 264                         store_sample(pmu, i, __I915_SAMPLE_RC6, val);
 265                         store_sample(pmu, i, __I915_SAMPLE_RC6_LAST_REPORTED,
 266                                      val);
 267                         pmu->sleep_last[i] = ktime_get_raw();
 268                 }
 269         }
 270 }
 271
 272 static void park_rc6(struct intel_gt *gt)
 273 {
 274         struct i915_pmu *pmu = &gt->i915->pmu;
 275
 276         store_sample(pmu, gt->info.id, __I915_SAMPLE_RC6, __get_rc6(gt));
 277         pmu->sleep_last[gt->info.id] = ktime_get_raw();
 278 }
 279
 280 static void __i915_pmu_maybe_start_timer(struct i915_pmu *pmu)
 281 {
 282         if (!pmu->timer_enabled && pmu_needs_timer(pmu)) {
 283                 pmu->timer_enabled = true;
 284                 pmu->timer_last = ktime_get();
 285                 hrtimer_start_range_ns(&pmu->timer,
 286                                        ns_to_ktime(PERIOD), 0,
 287                                        HRTIMER_MODE_REL_PINNED);
 288         }
 289 }
 290
 291 void i915_pmu_gt_parked(struct intel_gt *gt)
 292 {
 293         struct i915_pmu *pmu = &gt->i915->pmu;
 294
 295         if (!pmu->base.event_init)
 296                 return;
 297
 298         spin_lock_irq(&pmu->lock);
 299
 300         park_rc6(gt);
 301
 302         /*
 303          * Signal sampling timer to stop if only engine events are enabled and
 304          * GPU went idle.
 305          */
 306         pmu->unparked &= ~BIT(gt->info.id);
 307         if (pmu->unparked == 0)
 308                 pmu->timer_enabled = false;
 309
 310         spin_unlock_irq(&pmu->lock);
 311 }
 312
 313 void i915_pmu_gt_unparked(struct intel_gt *gt)
 314 {
 315         struct i915_pmu *pmu = &gt->i915->pmu;
 316
 317         if (!pmu->base.event_init)
 318                 return;
 319
 320         spin_lock_irq(&pmu->lock);
 321
 322         /*
 323          * Re-enable sampling timer when GPU goes active.
 324          */
 325         if (pmu->unparked == 0)
 326                 __i915_pmu_maybe_start_timer(pmu);
 327
 328         pmu->unparked |= BIT(gt->info.id);
 329
 330         spin_unlock_irq(&pmu->lock);
 331 }
 332
 333 static void
 334 add_sample(struct i915_pmu_sample *sample, u32 val)
 335 {
 336         sample->cur += val;
 337 }
 338
 339 static bool exclusive_mmio_access(const struct drm_i915_private *i915)
 340 {
 341         /*
 342          * We have to avoid concurrent mmio cache line access on gen7 or
 343          * risk a machine hang. For a fun history lesson dig out the old
 344          * userspace intel_gpu_top and run it on Ivybridge or Haswell!
 345          */
 346         return GRAPHICS_VER(i915) == 7;
 347 }
 348
 349 static void engine_sample(struct intel_engine_cs *engine, unsigned int period_ns)
 350 {
 351         struct intel_engine_pmu *pmu = &engine->pmu;
 352         bool busy;
 353         u32 val;
 354
 355         val = ENGINE_READ_FW(engine, RING_CTL);
 356         if (val == 0) /* powerwell off => engine idle */
 357                 return;
 358
 359         if (val & RING_WAIT)
 360                 add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns);
 361         if (val & RING_WAIT_SEMAPHORE)
 362                 add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns);
 363
 364         /* No need to sample when busy stats are supported. */
 365         if (intel_engine_supports_stats(engine))
 366                 return;
 367
 368         /*
 369          * While waiting on a semaphore or event, MI_MODE reports the
 370          * ring as idle. However, previously using the seqno, and with
 371          * execlists sampling, we account for the ring waiting as the
 372          * engine being busy. Therefore, we record the sample as being
 373          * busy if either waiting or !idle.
 374          */
 375         busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT);
 376         if (!busy) {
 377                 val = ENGINE_READ_FW(engine, RING_MI_MODE);
 378                 busy = !(val & MODE_IDLE);
 379         }
 380         if (busy)
 381                 add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns);
 382 }
 383
 384 static void
 385 engines_sample(struct intel_gt *gt, unsigned int period_ns)
 386 {
 387         struct drm_i915_private *i915 = gt->i915;
 388         struct intel_engine_cs *engine;
 389         enum intel_engine_id id;
 390         unsigned long flags;
 391
 392         if ((i915->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
 393                 return;
 394
 395         if (!intel_gt_pm_is_awake(gt))
 396                 return;
 397
 398         for_each_engine(engine, gt, id) {
 399                 if (!engine->pmu.enable)
 400                         continue;
 401
 402                 if (!intel_engine_pm_get_if_awake(engine))
 403                         continue;
 404
 405                 if (exclusive_mmio_access(i915)) {
 406                         spin_lock_irqsave(&engine->uncore->lock, flags);
 407                         engine_sample(engine, period_ns);
 408                         spin_unlock_irqrestore(&engine->uncore->lock, flags);
 409                 } else {
 410                         engine_sample(engine, period_ns);
 411                 }
 412
 413                 intel_engine_pm_put_async(engine);
 414         }
 415 }
 416
 417 static bool
 418 frequency_sampling_enabled(struct i915_pmu *pmu, unsigned int gt)
 419 {
 420         return pmu->enable &
 421                (config_mask(__I915_PMU_ACTUAL_FREQUENCY(gt)) |
 422                 config_mask(__I915_PMU_REQUESTED_FREQUENCY(gt)));
 423 }
 424
 425 static void
 426 frequency_sample(struct intel_gt *gt, unsigned int period_ns)
 427 {
 428         struct drm_i915_private *i915 = gt->i915;
 429         const unsigned int gt_id = gt->info.id;
 430         struct i915_pmu *pmu = &i915->pmu;
 431         struct intel_rps *rps = &gt->rps;
 432
 433         if (!frequency_sampling_enabled(pmu, gt_id))
 434                 return;
 435
 436         /* Report 0/0 (actual/requested) frequency while parked. */
 437         if (!intel_gt_pm_get_if_awake(gt))
 438                 return;
 439
 440         if (pmu->enable & config_mask(__I915_PMU_ACTUAL_FREQUENCY(gt_id))) {
 441                 u32 val;
 442
 443                 /*
 444                  * We take a quick peek here without using forcewake
 445                  * so that we don't perturb the system under observation
 446                  * (forcewake => !rc6 => increased power use). We expect
 447                  * that if the read fails because it is outside of the
 448                  * mmio power well, then it will return 0 -- in which
 449                  * case we assume the system is running at the intended
 450                  * frequency. Fortunately, the read should rarely fail!
 451                  */
 452                 val = intel_rps_read_actual_frequency_fw(rps);
 453                 if (!val)
 454                         val = intel_gpu_freq(rps, rps->cur_freq);
 455
 456                 add_sample_mult(pmu, gt_id, __I915_SAMPLE_FREQ_ACT,
 457                                 val, period_ns / 1000);
 458         }
 459
 460         if (pmu->enable & config_mask(__I915_PMU_REQUESTED_FREQUENCY(gt_id))) {
 461                 add_sample_mult(pmu, gt_id, __I915_SAMPLE_FREQ_REQ,
 462                                 intel_rps_get_requested_frequency(rps),
 463                                 period_ns / 1000);
 464         }
 465
 466         intel_gt_pm_put_async(gt);
 467 }
 468
 469 static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer)
 470 {
 471         struct drm_i915_private *i915 =
 472                 container_of(hrtimer, struct drm_i915_private, pmu.timer);
 473         struct i915_pmu *pmu = &i915->pmu;
 474         unsigned int period_ns;
 475         struct intel_gt *gt;
 476         unsigned int i;
 477         ktime_t now;
 478
 479         if (!READ_ONCE(pmu->timer_enabled))
 480                 return HRTIMER_NORESTART;
 481
 482         now = ktime_get();
 483         period_ns = ktime_to_ns(ktime_sub(now, pmu->timer_last));
 484         pmu->timer_last = now;
 485
 486         /*
 487          * Strictly speaking the passed in period may not be 100% accurate for
 488          * all internal calculation, since some amount of time can be spent on
 489          * grabbing the forcewake. However the potential error from timer call-
 490          * back delay greatly dominates this so we keep it simple.
 491          */
 492
 493         for_each_gt(gt, i915, i) {
 494                 if (!(pmu->unparked & BIT(i)))
 495                         continue;
 496
 497                 engines_sample(gt, period_ns);
 498                 frequency_sample(gt, period_ns);
 499         }
 500
 501         hrtimer_forward(hrtimer, now, ns_to_ktime(PERIOD));
 502
 503         return HRTIMER_RESTART;
 504 }
 505
 506 static void i915_pmu_event_destroy(struct perf_event *event)
 507 {
 508         struct drm_i915_private *i915 =
 509                 container_of(event->pmu, typeof(*i915), pmu.base);
 510
 511         drm_WARN_ON(&i915->drm, event->parent);
 512
 513         drm_dev_put(&i915->drm);
 514 }
 515
 516 static int
 517 engine_event_status(struct intel_engine_cs *engine,
 518                     enum drm_i915_pmu_engine_sample sample)
 519 {
 520         switch (sample) {
 521         case I915_SAMPLE_BUSY:
 522         case I915_SAMPLE_WAIT:
 523                 break;
 524         case I915_SAMPLE_SEMA:
 525                 if (GRAPHICS_VER(engine->i915) < 6)
 526                         return -ENODEV;
 527                 break;
 528         default:
 529                 return -ENOENT;
 530         }
 531
 532         return 0;
 533 }
 534
 535 static int
 536 config_status(struct drm_i915_private *i915, u64 config)
 537 {
 538         struct intel_gt *gt = to_gt(i915);
 539
 540         unsigned int gt_id = config_gt_id(config);
 541         unsigned int max_gt_id = HAS_EXTRA_GT_LIST(i915) ? 1 : 0;
 542
 543         if (gt_id > max_gt_id)
 544                 return -ENOENT;
 545
 546         switch (config_counter(config)) {
 547         case I915_PMU_ACTUAL_FREQUENCY:
 548                 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 549                         /* Requires a mutex for sampling! */
 550                         return -ENODEV;
 551                 fallthrough;
 552         case I915_PMU_REQUESTED_FREQUENCY:
 553                 if (GRAPHICS_VER(i915) < 6)
 554                         return -ENODEV;
 555                 break;
 556         case I915_PMU_INTERRUPTS:
 557                 if (gt_id)
 558                         return -ENOENT;
 559                 break;
 560         case I915_PMU_RC6_RESIDENCY:
 561                 if (!gt->rc6.supported)
 562                         return -ENODEV;
 563                 break;
 564         case I915_PMU_SOFTWARE_GT_AWAKE_TIME:
 565                 break;
 566         default:
 567                 return -ENOENT;
 568         }
 569
 570         return 0;
 571 }
 572
 573 static int engine_event_init(struct perf_event *event)
 574 {
 575         struct drm_i915_private *i915 =
 576                 container_of(event->pmu, typeof(*i915), pmu.base);
 577         struct intel_engine_cs *engine;
 578
 579         engine = intel_engine_lookup_user(i915, engine_event_class(event),
 580                                           engine_event_instance(event));
 581         if (!engine)
 582                 return -ENODEV;
 583
 584         return engine_event_status(engine, engine_event_sample(event));
 585 }
 586
 587 static int i915_pmu_event_init(struct perf_event *event)
 588 {
 589         struct drm_i915_private *i915 =
 590                 container_of(event->pmu, typeof(*i915), pmu.base);
 591         struct i915_pmu *pmu = &i915->pmu;
 592         int ret;
 593
 594         if (pmu->closed)
 595                 return -ENODEV;
 596
 597         if (event->attr.type != event->pmu->type)
 598                 return -ENOENT;
 599
 600         /* unsupported modes and filters */
 601         if (event->attr.sample_period) /* no sampling */
 602                 return -EINVAL;
 603
 604         if (has_branch_stack(event))
 605                 return -EOPNOTSUPP;
 606
 607         if (event->cpu < 0)
 608                 return -EINVAL;
 609
 610         /* only allow running on one cpu at a time */
 611         if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask))
 612                 return -EINVAL;
 613
 614         if (is_engine_event(event))
 615                 ret = engine_event_init(event);
 616         else
 617                 ret = config_status(i915, event->attr.config);
 618         if (ret)
 619                 return ret;
 620
 621         if (!event->parent) {
 622                 drm_dev_get(&i915->drm);
 623                 event->destroy = i915_pmu_event_destroy;
 624         }
 625
 626         return 0;
 627 }
 628
 629 static u64 __i915_pmu_event_read(struct perf_event *event)
 630 {
 631         struct drm_i915_private *i915 =
 632                 container_of(event->pmu, typeof(*i915), pmu.base);
 633         struct i915_pmu *pmu = &i915->pmu;
 634         u64 val = 0;
 635
 636         if (is_engine_event(event)) {
 637                 u8 sample = engine_event_sample(event);
 638                 struct intel_engine_cs *engine;
 639
 640                 engine = intel_engine_lookup_user(i915,
 641                                                   engine_event_class(event),
 642                                                   engine_event_instance(event));
 643
 644                 if (drm_WARN_ON_ONCE(&i915->drm, !engine)) {
 645                         /* Do nothing */
 646                 } else if (sample == I915_SAMPLE_BUSY &&
 647                            intel_engine_supports_stats(engine)) {
 648                         ktime_t unused;
 649
 650                         val = ktime_to_ns(intel_engine_get_busy_time(engine,
 651                                                                      &unused));
 652                 } else {
 653                         val = engine->pmu.sample[sample].cur;
 654                 }
 655         } else {
 656                 const unsigned int gt_id = config_gt_id(event->attr.config);
 657                 const u64 config = config_counter(event->attr.config);
 658
 659                 switch (config) {
 660                 case I915_PMU_ACTUAL_FREQUENCY:
 661                         val =
 662                            div_u64(read_sample(pmu, gt_id,
 663                                                __I915_SAMPLE_FREQ_ACT),
 664                                    USEC_PER_SEC /* to MHz */);
 665                         break;
 666                 case I915_PMU_REQUESTED_FREQUENCY:
 667                         val =
 668                            div_u64(read_sample(pmu, gt_id,
 669                                                __I915_SAMPLE_FREQ_REQ),
 670                                    USEC_PER_SEC /* to MHz */);
 671                         break;
 672                 case I915_PMU_INTERRUPTS:
 673                         val = READ_ONCE(pmu->irq_count);
 674                         break;
 675                 case I915_PMU_RC6_RESIDENCY:
 676                         val = get_rc6(i915->gt[gt_id]);
 677                         break;
 678                 case I915_PMU_SOFTWARE_GT_AWAKE_TIME:
 679                         val = ktime_to_ns(intel_gt_get_awake_time(to_gt(i915)));
 680                         break;
 681                 }
 682         }
 683
 684         return val;
 685 }
 686
 687 static void i915_pmu_event_read(struct perf_event *event)
 688 {
 689         struct drm_i915_private *i915 =
 690                 container_of(event->pmu, typeof(*i915), pmu.base);
 691         struct hw_perf_event *hwc = &event->hw;
 692         struct i915_pmu *pmu = &i915->pmu;
 693         u64 prev, new;
 694
 695         if (pmu->closed) {
 696                 event->hw.state = PERF_HES_STOPPED;
 697                 return;
 698         }
 699 again:
 700         prev = local64_read(&hwc->prev_count);
 701         new = __i915_pmu_event_read(event);
 702
 703         if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev)
 704                 goto again;
 705
 706         local64_add(new - prev, &event->count);
 707 }
 708
 709 static void i915_pmu_enable(struct perf_event *event)
 710 {
 711         struct drm_i915_private *i915 =
 712                 container_of(event->pmu, typeof(*i915), pmu.base);
 713         const unsigned int bit = event_bit(event);
 714         struct i915_pmu *pmu = &i915->pmu;
 715         unsigned long flags;
 716
 717         if (bit == -1)
 718                 goto update;
 719
 720         spin_lock_irqsave(&pmu->lock, flags);
 721
 722         /*
 723          * Update the bitmask of enabled events and increment
 724          * the event reference counter.
 725          */
 726         BUILD_BUG_ON(ARRAY_SIZE(pmu->enable_count) != I915_PMU_MASK_BITS);
 727         GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count));
 728         GEM_BUG_ON(pmu->enable_count[bit] == ~0);
 729
 730         pmu->enable |= BIT(bit);
 731         pmu->enable_count[bit]++;
 732
 733         /*
 734          * Start the sampling timer if needed and not already enabled.
 735          */
 736         __i915_pmu_maybe_start_timer(pmu);
 737
 738         /*
 739          * For per-engine events the bitmask and reference counting
 740          * is stored per engine.
 741          */
 742         if (is_engine_event(event)) {
 743                 u8 sample = engine_event_sample(event);
 744                 struct intel_engine_cs *engine;
 745
 746                 engine = intel_engine_lookup_user(i915,
 747                                                   engine_event_class(event),
 748                                                   engine_event_instance(event));
 749
 750                 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) !=
 751                              I915_ENGINE_SAMPLE_COUNT);
 752                 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) !=
 753                              I915_ENGINE_SAMPLE_COUNT);
 754                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
 755                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
 756                 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);
 757
 758                 engine->pmu.enable |= BIT(sample);
 759                 engine->pmu.enable_count[sample]++;
 760         }
 761
 762         spin_unlock_irqrestore(&pmu->lock, flags);
 763
 764 update:
 765         /*
 766          * Store the current counter value so we can report the correct delta
 767          * for all listeners. Even when the event was already enabled and has
 768          * an existing non-zero value.
 769          */
 770         local64_set(&event->hw.prev_count, __i915_pmu_event_read(event));
 771 }
 772
 773 static void i915_pmu_disable(struct perf_event *event)
 774 {
 775         struct drm_i915_private *i915 =
 776                 container_of(event->pmu, typeof(*i915), pmu.base);
 777         const unsigned int bit = event_bit(event);
 778         struct i915_pmu *pmu = &i915->pmu;
 779         unsigned long flags;
 780
 781         if (bit == -1)
 782                 return;
 783
 784         spin_lock_irqsave(&pmu->lock, flags);
 785
 786         if (is_engine_event(event)) {
 787                 u8 sample = engine_event_sample(event);
 788                 struct intel_engine_cs *engine;
 789
 790                 engine = intel_engine_lookup_user(i915,
 791                                                   engine_event_class(event),
 792                                                   engine_event_instance(event));
 793
 794                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
 795                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
 796                 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);
 797
 798                 /*
 799                  * Decrement the reference count and clear the enabled
 800                  * bitmask when the last listener on an event goes away.
 801                  */
 802                 if (--engine->pmu.enable_count[sample] == 0)
 803                         engine->pmu.enable &= ~BIT(sample);
 804         }
 805
 806         GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count));
 807         GEM_BUG_ON(pmu->enable_count[bit] == 0);
 808         /*
 809          * Decrement the reference count and clear the enabled
 810          * bitmask when the last listener on an event goes away.
 811          */
 812         if (--pmu->enable_count[bit] == 0) {
 813                 pmu->enable &= ~BIT(bit);
 814                 pmu->timer_enabled &= pmu_needs_timer(pmu);
 815         }
 816
 817         spin_unlock_irqrestore(&pmu->lock, flags);
 818 }
 819
 820 static void i915_pmu_event_start(struct perf_event *event, int flags)
 821 {
 822         struct drm_i915_private *i915 =
 823                 container_of(event->pmu, typeof(*i915), pmu.base);
 824         struct i915_pmu *pmu = &i915->pmu;
 825
 826         if (pmu->closed)
 827                 return;
 828
 829         i915_pmu_enable(event);
 830         event->hw.state = 0;
 831 }
 832
 833 static void i915_pmu_event_stop(struct perf_event *event, int flags)
 834 {
 835         if (flags & PERF_EF_UPDATE)
 836                 i915_pmu_event_read(event);
 837         i915_pmu_disable(event);
 838         event->hw.state = PERF_HES_STOPPED;
 839 }
 840
 841 static int i915_pmu_event_add(struct perf_event *event, int flags)
 842 {
 843         struct drm_i915_private *i915 =
 844                 container_of(event->pmu, typeof(*i915), pmu.base);
 845         struct i915_pmu *pmu = &i915->pmu;
 846
 847         if (pmu->closed)
 848                 return -ENODEV;
 849
 850         if (flags & PERF_EF_START)
 851                 i915_pmu_event_start(event, flags);
 852
 853         return 0;
 854 }
 855
 856 static void i915_pmu_event_del(struct perf_event *event, int flags)
 857 {
 858         i915_pmu_event_stop(event, PERF_EF_UPDATE);
 859 }
 860
 861 static int i915_pmu_event_event_idx(struct perf_event *event)
 862 {
 863         return 0;
 864 }
 865
 866 struct i915_str_attribute {
 867         struct device_attribute attr;
 868         const char *str;
 869 };
 870
 871 static ssize_t i915_pmu_format_show(struct device *dev,
 872                                     struct device_attribute *attr, char *buf)
 873 {
 874         struct i915_str_attribute *eattr;
 875
 876         eattr = container_of(attr, struct i915_str_attribute, attr);
 877         return sprintf(buf, "%s\n", eattr->str);
 878 }
 879
 880 #define I915_PMU_FORMAT_ATTR(_name, _config) \
 881         (&((struct i915_str_attribute[]) { \
 882                 { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \
 883                   .str = _config, } \
 884         })[0].attr.attr)
 885
 886 static struct attribute *i915_pmu_format_attrs[] = {
 887         I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
 888         NULL,
 889 };
 890
 891 static const struct attribute_group i915_pmu_format_attr_group = {
 892         .name = "format",
 893         .attrs = i915_pmu_format_attrs,
 894 };
 895
 896 struct i915_ext_attribute {
 897         struct device_attribute attr;
 898         unsigned long val;
 899 };
 900
 901 static ssize_t i915_pmu_event_show(struct device *dev,
 902                                    struct device_attribute *attr, char *buf)
 903 {
 904         struct i915_ext_attribute *eattr;
 905
 906         eattr = container_of(attr, struct i915_ext_attribute, attr);
 907         return sprintf(buf, "config=0x%lx\n", eattr->val);
 908 }
 909
 910 static ssize_t cpumask_show(struct device *dev,
 911                             struct device_attribute *attr, char *buf)
 912 {
 913         return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask);
 914 }
 915
 916 static DEVICE_ATTR_RO(cpumask);
 917
 918 static struct attribute *i915_cpumask_attrs[] = {
 919         &dev_attr_cpumask.attr,
 920         NULL,
 921 };
 922
 923 static const struct attribute_group i915_pmu_cpumask_attr_group = {
 924         .attrs = i915_cpumask_attrs,
 925 };
 926
 927 #define __event(__counter, __name, __unit) \
 928 { \
 929         .counter = (__counter), \
 930         .name = (__name), \
 931         .unit = (__unit), \
 932         .global = false, \
 933 }
 934
 935 #define __global_event(__counter, __name, __unit) \
 936 { \
 937         .counter = (__counter), \
 938         .name = (__name), \
 939         .unit = (__unit), \
 940         .global = true, \
 941 }
 942
 943 #define __engine_event(__sample, __name) \
 944 { \
 945         .sample = (__sample), \
 946         .name = (__name), \
 947 }
 948
 949 static struct i915_ext_attribute *
 950 add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config)
 951 {
 952         sysfs_attr_init(&attr->attr.attr);
 953         attr->attr.attr.name = name;
 954         attr->attr.attr.mode = 0444;
 955         attr->attr.show = i915_pmu_event_show;
 956         attr->val = config;
 957
 958         return ++attr;
 959 }
 960
 961 static struct perf_pmu_events_attr *
 962 add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name,
 963              const char *str)
 964 {
 965         sysfs_attr_init(&attr->attr.attr);
 966         attr->attr.attr.name = name;
 967         attr->attr.attr.mode = 0444;
 968         attr->attr.show = perf_event_sysfs_show;
 969         attr->event_str = str;
 970
 971         return ++attr;
 972 }
 973
 974 static struct attribute **
 975 create_event_attributes(struct i915_pmu *pmu)
 976 {
 977         struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu);
 978         static const struct {
 979                 unsigned int counter;
 980                 const char *name;
 981                 const char *unit;
 982                 bool global;
 983         } events[] = {
 984                 __event(0, "actual-frequency", "M"),
 985                 __event(1, "requested-frequency", "M"),
 986                 __global_event(2, "interrupts", NULL),
 987                 __event(3, "rc6-residency", "ns"),
 988                 __event(4, "software-gt-awake-time", "ns"),
 989         };
 990         static const struct {
 991                 enum drm_i915_pmu_engine_sample sample;
 992                 char *name;
 993         } engine_events[] = {
 994                 __engine_event(I915_SAMPLE_BUSY, "busy"),
 995                 __engine_event(I915_SAMPLE_SEMA, "sema"),
 996                 __engine_event(I915_SAMPLE_WAIT, "wait"),
 997         };
 998         unsigned int count = 0;
 999         struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter;
1000         struct i915_ext_attribute *i915_attr = NULL, *i915_iter;
1001         struct attribute **attr = NULL, **attr_iter;
1002         struct intel_engine_cs *engine;
1003         struct intel_gt *gt;
1004         unsigned int i, j;
1005
1006         /* Count how many counters we will be exposing. */
1007         for_each_gt(gt, i915, j) {
1008                 for (i = 0; i < ARRAY_SIZE(events); i++) {
1009                         u64 config = ___I915_PMU_OTHER(j, events[i].counter);
1010
1011                         if (!config_status(i915, config))
1012                                 count++;
1013                 }
1014         }
1015
1016         for_each_uabi_engine(engine, i915) {
1017                 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
1018                         if (!engine_event_status(engine,
1019                                                  engine_events[i].sample))
1020                                 count++;
1021                 }
1022         }
1023
1024         /* Allocate attribute objects and table. */
1025         i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL);
1026         if (!i915_attr)
1027                 goto err_alloc;
1028
1029         pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL);
1030         if (!pmu_attr)
1031                 goto err_alloc;
1032
1033         /* Max one pointer of each attribute type plus a termination entry. */
1034         attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL);
1035         if (!attr)
1036                 goto err_alloc;
1037
1038         i915_iter = i915_attr;
1039         pmu_iter = pmu_attr;
1040         attr_iter = attr;
1041
1042         /* Initialize supported non-engine counters. */
1043         for_each_gt(gt, i915, j) {
1044                 for (i = 0; i < ARRAY_SIZE(events); i++) {
1045                         u64 config = ___I915_PMU_OTHER(j, events[i].counter);
1046                         char *str;
1047
1048                         if (config_status(i915, config))
1049                                 continue;
1050
1051                         if (events[i].global || !HAS_EXTRA_GT_LIST(i915))
1052                                 str = kstrdup(events[i].name, GFP_KERNEL);
1053                         else
1054                                 str = kasprintf(GFP_KERNEL, "%s-gt%u",
1055                                                 events[i].name, j);
1056                         if (!str)
1057                                 goto err;
1058
1059                         *attr_iter++ = &i915_iter->attr.attr;
1060                         i915_iter = add_i915_attr(i915_iter, str, config);
1061
1062                         if (events[i].unit) {
1063                                 if (events[i].global || !HAS_EXTRA_GT_LIST(i915))
1064                                         str = kasprintf(GFP_KERNEL, "%s.unit",
1065                                                         events[i].name);
1066                                 else
1067                                         str = kasprintf(GFP_KERNEL, "%s-gt%u.unit",
1068                                                         events[i].name, j);
1069                                 if (!str)
1070                                         goto err;
1071
1072                                 *attr_iter++ = &pmu_iter->attr.attr;
1073                                 pmu_iter = add_pmu_attr(pmu_iter, str,
1074                                                         events[i].unit);
1075                         }
1076                 }
1077         }
1078
1079         /* Initialize supported engine counters. */
1080         for_each_uabi_engine(engine, i915) {
1081                 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
1082                         char *str;
1083
1084                         if (engine_event_status(engine,
1085                                                 engine_events[i].sample))
1086                                 continue;
1087
1088                         str = kasprintf(GFP_KERNEL, "%s-%s",
1089                                         engine->name, engine_events[i].name);
1090                         if (!str)
1091                                 goto err;
1092
1093                         *attr_iter++ = &i915_iter->attr.attr;
1094                         i915_iter =
1095                                 add_i915_attr(i915_iter, str,
1096                                               __I915_PMU_ENGINE(engine->uabi_class,
1097                                                                 engine->uabi_instance,
1098                                                                 engine_events[i].sample));
1099
1100                         str = kasprintf(GFP_KERNEL, "%s-%s.unit",
1101                                         engine->name, engine_events[i].name);
1102                         if (!str)
1103                                 goto err;
1104
1105                         *attr_iter++ = &pmu_iter->attr.attr;
1106                         pmu_iter = add_pmu_attr(pmu_iter, str, "ns");
1107                 }
1108         }
1109
1110         pmu->i915_attr = i915_attr;
1111         pmu->pmu_attr = pmu_attr;
1112
1113         return attr;
1114
1115 err:;
1116         for (attr_iter = attr; *attr_iter; attr_iter++)
1117                 kfree((*attr_iter)->name);
1118
1119 err_alloc:
1120         kfree(attr);
1121         kfree(i915_attr);
1122         kfree(pmu_attr);
1123
1124         return NULL;
1125 }
1126
1127 static void free_event_attributes(struct i915_pmu *pmu)
1128 {
1129         struct attribute **attr_iter = pmu->events_attr_group.attrs;
1130
1131         for (; *attr_iter; attr_iter++)
1132                 kfree((*attr_iter)->name);
1133
1134         kfree(pmu->events_attr_group.attrs);
1135         kfree(pmu->i915_attr);
1136         kfree(pmu->pmu_attr);
1137
1138         pmu->events_attr_group.attrs = NULL;
1139         pmu->i915_attr = NULL;
1140         pmu->pmu_attr = NULL;
1141 }
1142
1143 static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
1144 {
1145         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node);
1146
1147         GEM_BUG_ON(!pmu->base.event_init);
1148
1149         /* Select the first online CPU as a designated reader. */
1150         if (cpumask_empty(&i915_pmu_cpumask))
1151                 cpumask_set_cpu(cpu, &i915_pmu_cpumask);
1152
1153         return 0;
1154 }
1155
1156 static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
1157 {
1158         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node);
1159         unsigned int target = i915_pmu_target_cpu;
1160
1161         GEM_BUG_ON(!pmu->base.event_init);
1162
1163         /*
1164          * Unregistering an instance generates a CPU offline event which we must
1165          * ignore to avoid incorrectly modifying the shared i915_pmu_cpumask.
1166          */
1167         if (pmu->closed)
1168                 return 0;
1169
1170         if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) {
1171                 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
1172
1173                 /* Migrate events if there is a valid target */
1174                 if (target < nr_cpu_ids) {
1175                         cpumask_set_cpu(target, &i915_pmu_cpumask);
1176                         i915_pmu_target_cpu = target;
1177                 }
1178         }
1179
1180         if (target < nr_cpu_ids && target != pmu->cpuhp.cpu) {
1181                 perf_pmu_migrate_context(&pmu->base, cpu, target);
1182                 pmu->cpuhp.cpu = target;
1183         }
1184
1185         return 0;
1186 }
1187
1188 static enum cpuhp_state cpuhp_slot = CPUHP_INVALID;
1189
1190 int i915_pmu_init(void)
1191 {
1192         int ret;
1193
1194         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1195                                       "perf/x86/intel/i915:online",
1196                                       i915_pmu_cpu_online,
1197                                       i915_pmu_cpu_offline);
1198         if (ret < 0)
1199                 pr_notice("Failed to setup cpuhp state for i915 PMU! (%d)\n",
1200                           ret);
1201         else
1202                 cpuhp_slot = ret;
1203
1204         return 0;
1205 }
1206
1207 void i915_pmu_exit(void)
1208 {
1209         if (cpuhp_slot != CPUHP_INVALID)
1210                 cpuhp_remove_multi_state(cpuhp_slot);
1211 }
1212
1213 static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu)
1214 {
1215         if (cpuhp_slot == CPUHP_INVALID)
1216                 return -EINVAL;
1217
1218         return cpuhp_state_add_instance(cpuhp_slot, &pmu->cpuhp.node);
1219 }
1220
1221 static void i915_pmu_unregister_cpuhp_state(struct i915_pmu *pmu)
1222 {
1223         cpuhp_state_remove_instance(cpuhp_slot, &pmu->cpuhp.node);
1224 }
1225
1226 static bool is_igp(struct drm_i915_private *i915)
1227 {
1228         struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
1229
1230         /* IGP is 0000:00:02.0 */
1231         return pci_domain_nr(pdev->bus) == 0 &&
1232                pdev->bus->number == 0 &&
1233                PCI_SLOT(pdev->devfn) == 2 &&
1234                PCI_FUNC(pdev->devfn) == 0;
1235 }
1236
1237 void i915_pmu_register(struct drm_i915_private *i915)
1238 {
1239         struct i915_pmu *pmu = &i915->pmu;
1240         const struct attribute_group *attr_groups[] = {
1241                 &i915_pmu_format_attr_group,
1242                 &pmu->events_attr_group,
1243                 &i915_pmu_cpumask_attr_group,
1244                 NULL
1245         };
1246
1247         int ret = -ENOMEM;
1248
1249         if (GRAPHICS_VER(i915) <= 2) {
1250                 drm_info(&i915->drm, "PMU not supported for this GPU.");
1251                 return;
1252         }
1253
1254         spin_lock_init(&pmu->lock);
1255         hrtimer_init(&pmu->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1256         pmu->timer.function = i915_sample;
1257         pmu->cpuhp.cpu = -1;
1258         init_rc6(pmu);
1259
1260         if (!is_igp(i915)) {
1261                 pmu->name = kasprintf(GFP_KERNEL,
1262                                       "i915_%s",
1263                                       dev_name(i915->drm.dev));
1264                 if (pmu->name) {
1265                         /* tools/perf reserves colons as special. */
1266                         strreplace((char *)pmu->name, ':', '_');
1267                 }
1268         } else {
1269                 pmu->name = "i915";
1270         }
1271         if (!pmu->name)
1272                 goto err;
1273
1274         pmu->events_attr_group.name = "events";
1275         pmu->events_attr_group.attrs = create_event_attributes(pmu);
1276         if (!pmu->events_attr_group.attrs)
1277                 goto err_name;
1278
1279         pmu->base.attr_groups = kmemdup(attr_groups, sizeof(attr_groups),
1280                                         GFP_KERNEL);
1281         if (!pmu->base.attr_groups)
1282                 goto err_attr;
1283
1284         pmu->base.module        = THIS_MODULE;
1285         pmu->base.task_ctx_nr   = perf_invalid_context;
1286         pmu->base.event_init    = i915_pmu_event_init;
1287         pmu->base.add           = i915_pmu_event_add;
1288         pmu->base.del           = i915_pmu_event_del;
1289         pmu->base.start         = i915_pmu_event_start;
1290         pmu->base.stop          = i915_pmu_event_stop;
1291         pmu->base.read          = i915_pmu_event_read;
1292         pmu->base.event_idx     = i915_pmu_event_event_idx;
1293
1294         ret = perf_pmu_register(&pmu->base, pmu->name, -1);
1295         if (ret)
1296                 goto err_groups;
1297
1298         ret = i915_pmu_register_cpuhp_state(pmu);
1299         if (ret)
1300                 goto err_unreg;
1301
1302         return;
1303
1304 err_unreg:
1305         perf_pmu_unregister(&pmu->base);
1306 err_groups:
1307         kfree(pmu->base.attr_groups);
1308 err_attr:
1309         pmu->base.event_init = NULL;
1310         free_event_attributes(pmu);
1311 err_name:
1312         if (!is_igp(i915))
1313                 kfree(pmu->name);
1314 err:
1315         drm_notice(&i915->drm, "Failed to register PMU!\n");
1316 }
1317
1318 void i915_pmu_unregister(struct drm_i915_private *i915)
1319 {
1320         struct i915_pmu *pmu = &i915->pmu;
1321
1322         if (!pmu->base.event_init)
1323                 return;
1324
1325         /*
1326          * "Disconnect" the PMU callbacks - since all are atomic synchronize_rcu
1327          * ensures all currently executing ones will have exited before we
1328          * proceed with unregistration.
1329          */
1330         pmu->closed = true;
1331         synchronize_rcu();
1332
1333         hrtimer_cancel(&pmu->timer);
1334
1335         i915_pmu_unregister_cpuhp_state(pmu);
1336
1337         perf_pmu_unregister(&pmu->base);
1338         pmu->base.event_init = NULL;
1339         kfree(pmu->base.attr_groups);
1340         if (!is_igp(i915))
1341                 kfree(pmu->name);
1342         free_event_attributes(pmu);
1343 }