drivers/gpu/drm/i915/gt/intel_rps.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <linux/string_helpers.h>
   7
   8 #include <drm/i915_drm.h>
   9
  10 #include "i915_drv.h"
  11 #include "i915_irq.h"
  12 #include "intel_breadcrumbs.h"
  13 #include "intel_gt.h"
  14 #include "intel_gt_clock_utils.h"
  15 #include "intel_gt_irq.h"
  16 #include "intel_gt_pm_irq.h"
  17 #include "intel_gt_regs.h"
  18 #include "intel_mchbar_regs.h"
  19 #include "intel_pcode.h"
  20 #include "intel_rps.h"
  21 #include "vlv_sideband.h"
  22 #include "../../../platform/x86/intel_ips.h"
  23
  24 #define BUSY_MAX_EI     20u /* ms */
  25
  26 /*
  27  * Lock protecting IPS related data structures
  28  */
  29 static DEFINE_SPINLOCK(mchdev_lock);
  30
  31 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
  32 {
  33         return container_of(rps, struct intel_gt, rps);
  34 }
  35
  36 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
  37 {
  38         return rps_to_gt(rps)->i915;
  39 }
  40
  41 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
  42 {
  43         return rps_to_gt(rps)->uncore;
  44 }
  45
  46 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
  47 {
  48         struct intel_gt *gt = rps_to_gt(rps);
  49
  50         return &gt->uc.guc.slpc;
  51 }
  52
  53 static bool rps_uses_slpc(struct intel_rps *rps)
  54 {
  55         struct intel_gt *gt = rps_to_gt(rps);
  56
  57         return intel_uc_uses_guc_slpc(&gt->uc);
  58 }
  59
  60 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
  61 {
  62         return mask & ~rps->pm_intrmsk_mbz;
  63 }
  64
  65 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
  66 {
  67         intel_uncore_write_fw(uncore, reg, val);
  68 }
  69
  70 static void rps_timer(struct timer_list *t)
  71 {
  72         struct intel_rps *rps = from_timer(rps, t, timer);
  73         struct intel_engine_cs *engine;
  74         ktime_t dt, last, timestamp;
  75         enum intel_engine_id id;
  76         s64 max_busy[3] = {};
  77
  78         timestamp = 0;
  79         for_each_engine(engine, rps_to_gt(rps), id) {
  80                 s64 busy;
  81                 int i;
  82
  83                 dt = intel_engine_get_busy_time(engine, &timestamp);
  84                 last = engine->stats.rps;
  85                 engine->stats.rps = dt;
  86
  87                 busy = ktime_to_ns(ktime_sub(dt, last));
  88                 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
  89                         if (busy > max_busy[i])
  90                                 swap(busy, max_busy[i]);
  91                 }
  92         }
  93         last = rps->pm_timestamp;
  94         rps->pm_timestamp = timestamp;
  95
  96         if (intel_rps_is_active(rps)) {
  97                 s64 busy;
  98                 int i;
  99
 100                 dt = ktime_sub(timestamp, last);
 101
 102                 /*
 103                  * Our goal is to evaluate each engine independently, so we run
 104                  * at the lowest clocks required to sustain the heaviest
 105                  * workload. However, a task may be split into sequential
 106                  * dependent operations across a set of engines, such that
 107                  * the independent contributions do not account for high load,
 108                  * but overall the task is GPU bound. For example, consider
 109                  * video decode on vcs followed by colour post-processing
 110                  * on vecs, followed by general post-processing on rcs.
 111                  * Since multi-engines being active does imply a single
 112                  * continuous workload across all engines, we hedge our
 113                  * bets by only contributing a factor of the distributed
 114                  * load into our busyness calculation.
 115                  */
 116                 busy = max_busy[0];
 117                 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
 118                         if (!max_busy[i])
 119                                 break;
 120
 121                         busy += div_u64(max_busy[i], 1 << i);
 122                 }
 123                 GT_TRACE(rps_to_gt(rps),
 124                          "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
 125                          busy, (int)div64_u64(100 * busy, dt),
 126                          max_busy[0], max_busy[1], max_busy[2],
 127                          rps->pm_interval);
 128
 129                 if (100 * busy > rps->power.up_threshold * dt &&
 130                     rps->cur_freq < rps->max_freq_softlimit) {
 131                         rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
 132                         rps->pm_interval = 1;
 133                         schedule_work(&rps->work);
 134                 } else if (100 * busy < rps->power.down_threshold * dt &&
 135                            rps->cur_freq > rps->min_freq_softlimit) {
 136                         rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
 137                         rps->pm_interval = 1;
 138                         schedule_work(&rps->work);
 139                 } else {
 140                         rps->last_adj = 0;
 141                 }
 142
 143                 mod_timer(&rps->timer,
 144                           jiffies + msecs_to_jiffies(rps->pm_interval));
 145                 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
 146         }
 147 }
 148
 149 static void rps_start_timer(struct intel_rps *rps)
 150 {
 151         rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
 152         rps->pm_interval = 1;
 153         mod_timer(&rps->timer, jiffies + 1);
 154 }
 155
 156 static void rps_stop_timer(struct intel_rps *rps)
 157 {
 158         del_timer_sync(&rps->timer);
 159         rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
 160         cancel_work_sync(&rps->work);
 161 }
 162
 163 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
 164 {
 165         u32 mask = 0;
 166
 167         /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
 168         if (val > rps->min_freq_softlimit)
 169                 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
 170                          GEN6_PM_RP_DOWN_THRESHOLD |
 171                          GEN6_PM_RP_DOWN_TIMEOUT);
 172
 173         if (val < rps->max_freq_softlimit)
 174                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
 175
 176         mask &= rps->pm_events;
 177
 178         return rps_pm_sanitize_mask(rps, ~mask);
 179 }
 180
 181 static void rps_reset_ei(struct intel_rps *rps)
 182 {
 183         memset(&rps->ei, 0, sizeof(rps->ei));
 184 }
 185
 186 static void rps_enable_interrupts(struct intel_rps *rps)
 187 {
 188         struct intel_gt *gt = rps_to_gt(rps);
 189
 190         GEM_BUG_ON(rps_uses_slpc(rps));
 191
 192         GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
 193                  rps->pm_events, rps_pm_mask(rps, rps->last_freq));
 194
 195         rps_reset_ei(rps);
 196
 197         spin_lock_irq(&gt->irq_lock);
 198         gen6_gt_pm_enable_irq(gt, rps->pm_events);
 199         spin_unlock_irq(&gt->irq_lock);
 200
 201         intel_uncore_write(gt->uncore,
 202                            GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
 203 }
 204
 205 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
 206 {
 207         gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
 208 }
 209
 210 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
 211 {
 212         while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
 213                 ;
 214 }
 215
 216 static void rps_reset_interrupts(struct intel_rps *rps)
 217 {
 218         struct intel_gt *gt = rps_to_gt(rps);
 219
 220         spin_lock_irq(&gt->irq_lock);
 221         if (GRAPHICS_VER(gt->i915) >= 11)
 222                 gen11_rps_reset_interrupts(rps);
 223         else
 224                 gen6_rps_reset_interrupts(rps);
 225
 226         rps->pm_iir = 0;
 227         spin_unlock_irq(&gt->irq_lock);
 228 }
 229
 230 static void rps_disable_interrupts(struct intel_rps *rps)
 231 {
 232         struct intel_gt *gt = rps_to_gt(rps);
 233
 234         intel_uncore_write(gt->uncore,
 235                            GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
 236
 237         spin_lock_irq(&gt->irq_lock);
 238         gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
 239         spin_unlock_irq(&gt->irq_lock);
 240
 241         intel_synchronize_irq(gt->i915);
 242
 243         /*
 244          * Now that we will not be generating any more work, flush any
 245          * outstanding tasks. As we are called on the RPS idle path,
 246          * we will reset the GPU to minimum frequencies, so the current
 247          * state of the worker can be discarded.
 248          */
 249         cancel_work_sync(&rps->work);
 250
 251         rps_reset_interrupts(rps);
 252         GT_TRACE(gt, "interrupts:off\n");
 253 }
 254
 255 static const struct cparams {
 256         u16 i;
 257         u16 t;
 258         u16 m;
 259         u16 c;
 260 } cparams[] = {
 261         { 1, 1333, 301, 28664 },
 262         { 1, 1066, 294, 24460 },
 263         { 1, 800, 294, 25192 },
 264         { 0, 1333, 276, 27605 },
 265         { 0, 1066, 276, 27605 },
 266         { 0, 800, 231, 23784 },
 267 };
 268
 269 static void gen5_rps_init(struct intel_rps *rps)
 270 {
 271         struct drm_i915_private *i915 = rps_to_i915(rps);
 272         struct intel_uncore *uncore = rps_to_uncore(rps);
 273         u8 fmax, fmin, fstart;
 274         u32 rgvmodectl;
 275         int c_m, i;
 276
 277         if (i915->fsb_freq <= 3200)
 278                 c_m = 0;
 279         else if (i915->fsb_freq <= 4800)
 280                 c_m = 1;
 281         else
 282                 c_m = 2;
 283
 284         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
 285                 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
 286                         rps->ips.m = cparams[i].m;
 287                         rps->ips.c = cparams[i].c;
 288                         break;
 289                 }
 290         }
 291
 292         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 293
 294         /* Set up min, max, and cur for interrupt handling */
 295         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
 296         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
 297         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 298                 MEMMODE_FSTART_SHIFT;
 299         drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
 300                 fmax, fmin, fstart);
 301
 302         rps->min_freq = fmax;
 303         rps->efficient_freq = fstart;
 304         rps->max_freq = fmin;
 305 }
 306
 307 static unsigned long
 308 __ips_chipset_val(struct intel_ips *ips)
 309 {
 310         struct intel_uncore *uncore =
 311                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 312         unsigned long now = jiffies_to_msecs(jiffies), dt;
 313         unsigned long result;
 314         u64 total, delta;
 315
 316         lockdep_assert_held(&mchdev_lock);
 317
 318         /*
 319          * Prevent division-by-zero if we are asking too fast.
 320          * Also, we don't get interesting results if we are polling
 321          * faster than once in 10ms, so just return the saved value
 322          * in such cases.
 323          */
 324         dt = now - ips->last_time1;
 325         if (dt <= 10)
 326                 return ips->chipset_power;
 327
 328         /* FIXME: handle per-counter overflow */
 329         total = intel_uncore_read(uncore, DMIEC);
 330         total += intel_uncore_read(uncore, DDREC);
 331         total += intel_uncore_read(uncore, CSIEC);
 332
 333         delta = total - ips->last_count1;
 334
 335         result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
 336
 337         ips->last_count1 = total;
 338         ips->last_time1 = now;
 339
 340         ips->chipset_power = result;
 341
 342         return result;
 343 }
 344
 345 static unsigned long ips_mch_val(struct intel_uncore *uncore)
 346 {
 347         unsigned int m, x, b;
 348         u32 tsfs;
 349
 350         tsfs = intel_uncore_read(uncore, TSFS);
 351         x = intel_uncore_read8(uncore, TR1);
 352
 353         b = tsfs & TSFS_INTR_MASK;
 354         m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
 355
 356         return m * x / 127 - b;
 357 }
 358
 359 static int _pxvid_to_vd(u8 pxvid)
 360 {
 361         if (pxvid == 0)
 362                 return 0;
 363
 364         if (pxvid >= 8 && pxvid < 31)
 365                 pxvid = 31;
 366
 367         return (pxvid + 2) * 125;
 368 }
 369
 370 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
 371 {
 372         const int vd = _pxvid_to_vd(pxvid);
 373
 374         if (INTEL_INFO(i915)->is_mobile)
 375                 return max(vd - 1125, 0);
 376
 377         return vd;
 378 }
 379
 380 static void __gen5_ips_update(struct intel_ips *ips)
 381 {
 382         struct intel_uncore *uncore =
 383                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 384         u64 now, delta, dt;
 385         u32 count;
 386
 387         lockdep_assert_held(&mchdev_lock);
 388
 389         now = ktime_get_raw_ns();
 390         dt = now - ips->last_time2;
 391         do_div(dt, NSEC_PER_MSEC);
 392
 393         /* Don't divide by 0 */
 394         if (dt <= 10)
 395                 return;
 396
 397         count = intel_uncore_read(uncore, GFXEC);
 398         delta = count - ips->last_count2;
 399
 400         ips->last_count2 = count;
 401         ips->last_time2 = now;
 402
 403         /* More magic constants... */
 404         ips->gfx_power = div_u64(delta * 1181, dt * 10);
 405 }
 406
 407 static void gen5_rps_update(struct intel_rps *rps)
 408 {
 409         spin_lock_irq(&mchdev_lock);
 410         __gen5_ips_update(&rps->ips);
 411         spin_unlock_irq(&mchdev_lock);
 412 }
 413
 414 static unsigned int gen5_invert_freq(struct intel_rps *rps,
 415                                      unsigned int val)
 416 {
 417         /* Invert the frequency bin into an ips delay */
 418         val = rps->max_freq - val;
 419         val = rps->min_freq + val;
 420
 421         return val;
 422 }
 423
 424 static int __gen5_rps_set(struct intel_rps *rps, u8 val)
 425 {
 426         struct intel_uncore *uncore = rps_to_uncore(rps);
 427         u16 rgvswctl;
 428
 429         lockdep_assert_held(&mchdev_lock);
 430
 431         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 432         if (rgvswctl & MEMCTL_CMD_STS) {
 433                 DRM_DEBUG("gpu busy, RCS change rejected\n");
 434                 return -EBUSY; /* still busy with another command */
 435         }
 436
 437         /* Invert the frequency bin into an ips delay */
 438         val = gen5_invert_freq(rps, val);
 439
 440         rgvswctl =
 441                 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
 442                 (val << MEMCTL_FREQ_SHIFT) |
 443                 MEMCTL_SFCAVM;
 444         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 445         intel_uncore_posting_read16(uncore, MEMSWCTL);
 446
 447         rgvswctl |= MEMCTL_CMD_STS;
 448         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 449
 450         return 0;
 451 }
 452
 453 static int gen5_rps_set(struct intel_rps *rps, u8 val)
 454 {
 455         int err;
 456
 457         spin_lock_irq(&mchdev_lock);
 458         err = __gen5_rps_set(rps, val);
 459         spin_unlock_irq(&mchdev_lock);
 460
 461         return err;
 462 }
 463
 464 static unsigned long intel_pxfreq(u32 vidfreq)
 465 {
 466         int div = (vidfreq & 0x3f0000) >> 16;
 467         int post = (vidfreq & 0x3000) >> 12;
 468         int pre = (vidfreq & 0x7);
 469
 470         if (!pre)
 471                 return 0;
 472
 473         return div * 133333 / (pre << post);
 474 }
 475
 476 static unsigned int init_emon(struct intel_uncore *uncore)
 477 {
 478         u8 pxw[16];
 479         int i;
 480
 481         /* Disable to program */
 482         intel_uncore_write(uncore, ECR, 0);
 483         intel_uncore_posting_read(uncore, ECR);
 484
 485         /* Program energy weights for various events */
 486         intel_uncore_write(uncore, SDEW, 0x15040d00);
 487         intel_uncore_write(uncore, CSIEW0, 0x007f0000);
 488         intel_uncore_write(uncore, CSIEW1, 0x1e220004);
 489         intel_uncore_write(uncore, CSIEW2, 0x04000004);
 490
 491         for (i = 0; i < 5; i++)
 492                 intel_uncore_write(uncore, PEW(i), 0);
 493         for (i = 0; i < 3; i++)
 494                 intel_uncore_write(uncore, DEW(i), 0);
 495
 496         /* Program P-state weights to account for frequency power adjustment */
 497         for (i = 0; i < 16; i++) {
 498                 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
 499                 unsigned int freq = intel_pxfreq(pxvidfreq);
 500                 unsigned int vid =
 501                         (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 502                 unsigned int val;
 503
 504                 val = vid * vid * freq / 1000 * 255;
 505                 val /= 127 * 127 * 900;
 506
 507                 pxw[i] = val;
 508         }
 509         /* Render standby states get 0 weight */
 510         pxw[14] = 0;
 511         pxw[15] = 0;
 512
 513         for (i = 0; i < 4; i++) {
 514                 intel_uncore_write(uncore, PXW(i),
 515                                    pxw[i * 4 + 0] << 24 |
 516                                    pxw[i * 4 + 1] << 16 |
 517                                    pxw[i * 4 + 2] <<  8 |
 518                                    pxw[i * 4 + 3] <<  0);
 519         }
 520
 521         /* Adjust magic regs to magic values (more experimental results) */
 522         intel_uncore_write(uncore, OGW0, 0);
 523         intel_uncore_write(uncore, OGW1, 0);
 524         intel_uncore_write(uncore, EG0, 0x00007f00);
 525         intel_uncore_write(uncore, EG1, 0x0000000e);
 526         intel_uncore_write(uncore, EG2, 0x000e0000);
 527         intel_uncore_write(uncore, EG3, 0x68000300);
 528         intel_uncore_write(uncore, EG4, 0x42000000);
 529         intel_uncore_write(uncore, EG5, 0x00140031);
 530         intel_uncore_write(uncore, EG6, 0);
 531         intel_uncore_write(uncore, EG7, 0);
 532
 533         for (i = 0; i < 8; i++)
 534                 intel_uncore_write(uncore, PXWL(i), 0);
 535
 536         /* Enable PMON + select events */
 537         intel_uncore_write(uncore, ECR, 0x80000019);
 538
 539         return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
 540 }
 541
 542 static bool gen5_rps_enable(struct intel_rps *rps)
 543 {
 544         struct drm_i915_private *i915 = rps_to_i915(rps);
 545         struct intel_uncore *uncore = rps_to_uncore(rps);
 546         u8 fstart, vstart;
 547         u32 rgvmodectl;
 548
 549         spin_lock_irq(&mchdev_lock);
 550
 551         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 552
 553         /* Enable temp reporting */
 554         intel_uncore_write16(uncore, PMMISC,
 555                              intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
 556         intel_uncore_write16(uncore, TSC1,
 557                              intel_uncore_read16(uncore, TSC1) | TSE);
 558
 559         /* 100ms RC evaluation intervals */
 560         intel_uncore_write(uncore, RCUPEI, 100000);
 561         intel_uncore_write(uncore, RCDNEI, 100000);
 562
 563         /* Set max/min thresholds to 90ms and 80ms respectively */
 564         intel_uncore_write(uncore, RCBMAXAVG, 90000);
 565         intel_uncore_write(uncore, RCBMINAVG, 80000);
 566
 567         intel_uncore_write(uncore, MEMIHYST, 1);
 568
 569         /* Set up min, max, and cur for interrupt handling */
 570         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 571                 MEMMODE_FSTART_SHIFT;
 572
 573         vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
 574                   PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 575
 576         intel_uncore_write(uncore,
 577                            MEMINTREN,
 578                            MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
 579
 580         intel_uncore_write(uncore, VIDSTART, vstart);
 581         intel_uncore_posting_read(uncore, VIDSTART);
 582
 583         rgvmodectl |= MEMMODE_SWMODE_EN;
 584         intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
 585
 586         if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
 587                              MEMCTL_CMD_STS) == 0, 10))
 588                 drm_err(&uncore->i915->drm,
 589                         "stuck trying to change perf mode\n");
 590         mdelay(1);
 591
 592         __gen5_rps_set(rps, rps->cur_freq);
 593
 594         rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
 595         rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
 596         rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
 597         rps->ips.last_time1 = jiffies_to_msecs(jiffies);
 598
 599         rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
 600         rps->ips.last_time2 = ktime_get_raw_ns();
 601
 602         spin_lock(&i915->irq_lock);
 603         ilk_enable_display_irq(i915, DE_PCU_EVENT);
 604         spin_unlock(&i915->irq_lock);
 605
 606         spin_unlock_irq(&mchdev_lock);
 607
 608         rps->ips.corr = init_emon(uncore);
 609
 610         return true;
 611 }
 612
 613 static void gen5_rps_disable(struct intel_rps *rps)
 614 {
 615         struct drm_i915_private *i915 = rps_to_i915(rps);
 616         struct intel_uncore *uncore = rps_to_uncore(rps);
 617         u16 rgvswctl;
 618
 619         spin_lock_irq(&mchdev_lock);
 620
 621         spin_lock(&i915->irq_lock);
 622         ilk_disable_display_irq(i915, DE_PCU_EVENT);
 623         spin_unlock(&i915->irq_lock);
 624
 625         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 626
 627         /* Ack interrupts, disable EFC interrupt */
 628         intel_uncore_write(uncore, MEMINTREN,
 629                            intel_uncore_read(uncore, MEMINTREN) &
 630                            ~MEMINT_EVAL_CHG_EN);
 631         intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
 632
 633         /* Go back to the starting frequency */
 634         __gen5_rps_set(rps, rps->idle_freq);
 635         mdelay(1);
 636         rgvswctl |= MEMCTL_CMD_STS;
 637         intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
 638         mdelay(1);
 639
 640         spin_unlock_irq(&mchdev_lock);
 641 }
 642
 643 static u32 rps_limits(struct intel_rps *rps, u8 val)
 644 {
 645         u32 limits;
 646
 647         /*
 648          * Only set the down limit when we've reached the lowest level to avoid
 649          * getting more interrupts, otherwise leave this clear. This prevents a
 650          * race in the hw when coming out of rc6: There's a tiny window where
 651          * the hw runs at the minimal clock before selecting the desired
 652          * frequency, if the down threshold expires in that window we will not
 653          * receive a down interrupt.
 654          */
 655         if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
 656                 limits = rps->max_freq_softlimit << 23;
 657                 if (val <= rps->min_freq_softlimit)
 658                         limits |= rps->min_freq_softlimit << 14;
 659         } else {
 660                 limits = rps->max_freq_softlimit << 24;
 661                 if (val <= rps->min_freq_softlimit)
 662                         limits |= rps->min_freq_softlimit << 16;
 663         }
 664
 665         return limits;
 666 }
 667
 668 static void rps_set_power(struct intel_rps *rps, int new_power)
 669 {
 670         struct intel_gt *gt = rps_to_gt(rps);
 671         struct intel_uncore *uncore = gt->uncore;
 672         u32 threshold_up = 0, threshold_down = 0; /* in % */
 673         u32 ei_up = 0, ei_down = 0;
 674
 675         lockdep_assert_held(&rps->power.mutex);
 676
 677         if (new_power == rps->power.mode)
 678                 return;
 679
 680         threshold_up = 95;
 681         threshold_down = 85;
 682
 683         /* Note the units here are not exactly 1us, but 1280ns. */
 684         switch (new_power) {
 685         case LOW_POWER:
 686                 ei_up = 16000;
 687                 ei_down = 32000;
 688                 break;
 689
 690         case BETWEEN:
 691                 ei_up = 13000;
 692                 ei_down = 32000;
 693                 break;
 694
 695         case HIGH_POWER:
 696                 ei_up = 10000;
 697                 ei_down = 32000;
 698                 break;
 699         }
 700
 701         /* When byt can survive without system hang with dynamic
 702          * sw freq adjustments, this restriction can be lifted.
 703          */
 704         if (IS_VALLEYVIEW(gt->i915))
 705                 goto skip_hw_write;
 706
 707         GT_TRACE(gt,
 708                  "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
 709                  new_power, threshold_up, ei_up, threshold_down, ei_down);
 710
 711         set(uncore, GEN6_RP_UP_EI,
 712             intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
 713         set(uncore, GEN6_RP_UP_THRESHOLD,
 714             intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10));
 715
 716         set(uncore, GEN6_RP_DOWN_EI,
 717             intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
 718         set(uncore, GEN6_RP_DOWN_THRESHOLD,
 719             intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10));
 720
 721         set(uncore, GEN6_RP_CONTROL,
 722             (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
 723             GEN6_RP_MEDIA_HW_NORMAL_MODE |
 724             GEN6_RP_MEDIA_IS_GFX |
 725             GEN6_RP_ENABLE |
 726             GEN6_RP_UP_BUSY_AVG |
 727             GEN6_RP_DOWN_IDLE_AVG);
 728
 729 skip_hw_write:
 730         rps->power.mode = new_power;
 731         rps->power.up_threshold = threshold_up;
 732         rps->power.down_threshold = threshold_down;
 733 }
 734
 735 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
 736 {
 737         int new_power;
 738
 739         new_power = rps->power.mode;
 740         switch (rps->power.mode) {
 741         case LOW_POWER:
 742                 if (val > rps->efficient_freq + 1 &&
 743                     val > rps->cur_freq)
 744                         new_power = BETWEEN;
 745                 break;
 746
 747         case BETWEEN:
 748                 if (val <= rps->efficient_freq &&
 749                     val < rps->cur_freq)
 750                         new_power = LOW_POWER;
 751                 else if (val >= rps->rp0_freq &&
 752                          val > rps->cur_freq)
 753                         new_power = HIGH_POWER;
 754                 break;
 755
 756         case HIGH_POWER:
 757                 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
 758                     val < rps->cur_freq)
 759                         new_power = BETWEEN;
 760                 break;
 761         }
 762         /* Max/min bins are special */
 763         if (val <= rps->min_freq_softlimit)
 764                 new_power = LOW_POWER;
 765         if (val >= rps->max_freq_softlimit)
 766                 new_power = HIGH_POWER;
 767
 768         mutex_lock(&rps->power.mutex);
 769         if (rps->power.interactive)
 770                 new_power = HIGH_POWER;
 771         rps_set_power(rps, new_power);
 772         mutex_unlock(&rps->power.mutex);
 773 }
 774
 775 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
 776 {
 777         GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
 778                  str_yes_no(interactive));
 779
 780         mutex_lock(&rps->power.mutex);
 781         if (interactive) {
 782                 if (!rps->power.interactive++ && intel_rps_is_active(rps))
 783                         rps_set_power(rps, HIGH_POWER);
 784         } else {
 785                 GEM_BUG_ON(!rps->power.interactive);
 786                 rps->power.interactive--;
 787         }
 788         mutex_unlock(&rps->power.mutex);
 789 }
 790
 791 static int gen6_rps_set(struct intel_rps *rps, u8 val)
 792 {
 793         struct intel_uncore *uncore = rps_to_uncore(rps);
 794         struct drm_i915_private *i915 = rps_to_i915(rps);
 795         u32 swreq;
 796
 797         GEM_BUG_ON(rps_uses_slpc(rps));
 798
 799         if (GRAPHICS_VER(i915) >= 9)
 800                 swreq = GEN9_FREQUENCY(val);
 801         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
 802                 swreq = HSW_FREQUENCY(val);
 803         else
 804                 swreq = (GEN6_FREQUENCY(val) |
 805                          GEN6_OFFSET(0) |
 806                          GEN6_AGGRESSIVE_TURBO);
 807         set(uncore, GEN6_RPNSWREQ, swreq);
 808
 809         GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
 810                  val, intel_gpu_freq(rps, val), swreq);
 811
 812         return 0;
 813 }
 814
 815 static int vlv_rps_set(struct intel_rps *rps, u8 val)
 816 {
 817         struct drm_i915_private *i915 = rps_to_i915(rps);
 818         int err;
 819
 820         vlv_punit_get(i915);
 821         err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
 822         vlv_punit_put(i915);
 823
 824         GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
 825                  val, intel_gpu_freq(rps, val));
 826
 827         return err;
 828 }
 829
 830 static int rps_set(struct intel_rps *rps, u8 val, bool update)
 831 {
 832         struct drm_i915_private *i915 = rps_to_i915(rps);
 833         int err;
 834
 835         if (val == rps->last_freq)
 836                 return 0;
 837
 838         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 839                 err = vlv_rps_set(rps, val);
 840         else if (GRAPHICS_VER(i915) >= 6)
 841                 err = gen6_rps_set(rps, val);
 842         else
 843                 err = gen5_rps_set(rps, val);
 844         if (err)
 845                 return err;
 846
 847         if (update && GRAPHICS_VER(i915) >= 6)
 848                 gen6_rps_set_thresholds(rps, val);
 849         rps->last_freq = val;
 850
 851         return 0;
 852 }
 853
 854 void intel_rps_unpark(struct intel_rps *rps)
 855 {
 856         if (!intel_rps_is_enabled(rps))
 857                 return;
 858
 859         GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
 860
 861         /*
 862          * Use the user's desired frequency as a guide, but for better
 863          * performance, jump directly to RPe as our starting frequency.
 864          */
 865         mutex_lock(&rps->lock);
 866
 867         intel_rps_set_active(rps);
 868         intel_rps_set(rps,
 869                       clamp(rps->cur_freq,
 870                             rps->min_freq_softlimit,
 871                             rps->max_freq_softlimit));
 872
 873         mutex_unlock(&rps->lock);
 874
 875         rps->pm_iir = 0;
 876         if (intel_rps_has_interrupts(rps))
 877                 rps_enable_interrupts(rps);
 878         if (intel_rps_uses_timer(rps))
 879                 rps_start_timer(rps);
 880
 881         if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
 882                 gen5_rps_update(rps);
 883 }
 884
 885 void intel_rps_park(struct intel_rps *rps)
 886 {
 887         int adj;
 888
 889         if (!intel_rps_is_enabled(rps))
 890                 return;
 891
 892         if (!intel_rps_clear_active(rps))
 893                 return;
 894
 895         if (intel_rps_uses_timer(rps))
 896                 rps_stop_timer(rps);
 897         if (intel_rps_has_interrupts(rps))
 898                 rps_disable_interrupts(rps);
 899
 900         if (rps->last_freq <= rps->idle_freq)
 901                 return;
 902
 903         /*
 904          * The punit delays the write of the frequency and voltage until it
 905          * determines the GPU is awake. During normal usage we don't want to
 906          * waste power changing the frequency if the GPU is sleeping (rc6).
 907          * However, the GPU and driver is now idle and we do not want to delay
 908          * switching to minimum voltage (reducing power whilst idle) as we do
 909          * not expect to be woken in the near future and so must flush the
 910          * change by waking the device.
 911          *
 912          * We choose to take the media powerwell (either would do to trick the
 913          * punit into committing the voltage change) as that takes a lot less
 914          * power than the render powerwell.
 915          */
 916         intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 917         rps_set(rps, rps->idle_freq, false);
 918         intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 919
 920         /*
 921          * Since we will try and restart from the previously requested
 922          * frequency on unparking, treat this idle point as a downclock
 923          * interrupt and reduce the frequency for resume. If we park/unpark
 924          * more frequently than the rps worker can run, we will not respond
 925          * to any EI and never see a change in frequency.
 926          *
 927          * (Note we accommodate Cherryview's limitation of only using an
 928          * even bin by applying it to all.)
 929          */
 930         adj = rps->last_adj;
 931         if (adj < 0)
 932                 adj *= 2;
 933         else /* CHV needs even encode values */
 934                 adj = -2;
 935         rps->last_adj = adj;
 936         rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
 937         if (rps->cur_freq < rps->efficient_freq) {
 938                 rps->cur_freq = rps->efficient_freq;
 939                 rps->last_adj = 0;
 940         }
 941
 942         GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
 943 }
 944
 945 u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
 946 {
 947         struct intel_guc_slpc *slpc;
 948
 949         if (rps_uses_slpc(rps)) {
 950                 slpc = rps_to_slpc(rps);
 951
 952                 return slpc->boost_freq;
 953         } else {
 954                 return intel_gpu_freq(rps, rps->boost_freq);
 955         }
 956 }
 957
 958 static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
 959 {
 960         bool boost = false;
 961
 962         /* Validate against (static) hardware limits */
 963         val = intel_freq_opcode(rps, val);
 964         if (val < rps->min_freq || val > rps->max_freq)
 965                 return -EINVAL;
 966
 967         mutex_lock(&rps->lock);
 968         if (val != rps->boost_freq) {
 969                 rps->boost_freq = val;
 970                 boost = atomic_read(&rps->num_waiters);
 971         }
 972         mutex_unlock(&rps->lock);
 973         if (boost)
 974                 schedule_work(&rps->work);
 975
 976         return 0;
 977 }
 978
 979 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
 980 {
 981         struct intel_guc_slpc *slpc;
 982
 983         if (rps_uses_slpc(rps)) {
 984                 slpc = rps_to_slpc(rps);
 985
 986                 return intel_guc_slpc_set_boost_freq(slpc, freq);
 987         } else {
 988                 return rps_set_boost_freq(rps, freq);
 989         }
 990 }
 991
 992 void intel_rps_dec_waiters(struct intel_rps *rps)
 993 {
 994         struct intel_guc_slpc *slpc;
 995
 996         if (rps_uses_slpc(rps)) {
 997                 slpc = rps_to_slpc(rps);
 998
 999                 intel_guc_slpc_dec_waiters(slpc);
1000         } else {
1001                 atomic_dec(&rps->num_waiters);
1002         }
1003 }
1004
1005 void intel_rps_boost(struct i915_request *rq)
1006 {
1007         struct intel_guc_slpc *slpc;
1008
1009         if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1010                 return;
1011
1012         /* Serializes with i915_request_retire() */
1013         if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
1014                 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1015
1016                 if (rps_uses_slpc(rps)) {
1017                         slpc = rps_to_slpc(rps);
1018
1019                         /* Return if old value is non zero */
1020                         if (!atomic_fetch_inc(&slpc->num_waiters))
1021                                 schedule_work(&slpc->boost_work);
1022
1023                         return;
1024                 }
1025
1026                 if (atomic_fetch_inc(&rps->num_waiters))
1027                         return;
1028
1029                 if (!intel_rps_is_active(rps))
1030                         return;
1031
1032                 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1033                          rq->fence.context, rq->fence.seqno);
1034
1035                 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1036                         schedule_work(&rps->work);
1037
1038                 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1039         }
1040 }
1041
1042 int intel_rps_set(struct intel_rps *rps, u8 val)
1043 {
1044         int err;
1045
1046         lockdep_assert_held(&rps->lock);
1047         GEM_BUG_ON(val > rps->max_freq);
1048         GEM_BUG_ON(val < rps->min_freq);
1049
1050         if (intel_rps_is_active(rps)) {
1051                 err = rps_set(rps, val, true);
1052                 if (err)
1053                         return err;
1054
1055                 /*
1056                  * Make sure we continue to get interrupts
1057                  * until we hit the minimum or maximum frequencies.
1058                  */
1059                 if (intel_rps_has_interrupts(rps)) {
1060                         struct intel_uncore *uncore = rps_to_uncore(rps);
1061
1062                         set(uncore,
1063                             GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
1064
1065                         set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
1066                 }
1067         }
1068
1069         rps->cur_freq = val;
1070         return 0;
1071 }
1072
1073 static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1074 {
1075         struct drm_i915_private *i915 = rps_to_i915(rps);
1076         struct intel_uncore *uncore = rps_to_uncore(rps);
1077
1078         if (IS_XEHPSDV(i915))
1079                 return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP);
1080         else if (IS_GEN9_LP(i915))
1081                 return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1082         else
1083                 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1084 }
1085
1086 /**
1087  * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1088  * @rps: the intel_rps structure
1089  * @caps: returned freq caps
1090  *
1091  * Returned "caps" frequencies should be converted to MHz using
1092  * intel_gpu_freq()
1093  */
1094 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1095 {
1096         struct drm_i915_private *i915 = rps_to_i915(rps);
1097         u32 rp_state_cap;
1098
1099         rp_state_cap = intel_rps_read_state_cap(rps);
1100
1101         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
1102         if (IS_GEN9_LP(i915)) {
1103                 caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1104                 caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1105                 caps->min_freq = (rp_state_cap >>  0) & 0xff;
1106         } else {
1107                 caps->rp0_freq = (rp_state_cap >>  0) & 0xff;
1108                 caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1109                 caps->min_freq = (rp_state_cap >> 16) & 0xff;
1110         }
1111
1112         if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1113                 /*
1114                  * In this case rp_state_cap register reports frequencies in
1115                  * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1116                  * units of 16.67 MHz
1117                  */
1118                 caps->rp0_freq *= GEN9_FREQ_SCALER;
1119                 caps->rp1_freq *= GEN9_FREQ_SCALER;
1120                 caps->min_freq *= GEN9_FREQ_SCALER;
1121         }
1122 }
1123
1124 static void gen6_rps_init(struct intel_rps *rps)
1125 {
1126         struct drm_i915_private *i915 = rps_to_i915(rps);
1127         struct intel_rps_freq_caps caps;
1128
1129         gen6_rps_get_freq_caps(rps, &caps);
1130         rps->rp0_freq = caps.rp0_freq;
1131         rps->rp1_freq = caps.rp1_freq;
1132         rps->min_freq = caps.min_freq;
1133
1134         /* hw_max = RP0 until we check for overclocking */
1135         rps->max_freq = rps->rp0_freq;
1136
1137         rps->efficient_freq = rps->rp1_freq;
1138         if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1139             IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1140                 u32 ddcc_status = 0;
1141                 u32 mult = 1;
1142
1143                 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1144                         mult = GEN9_FREQ_SCALER;
1145                 if (snb_pcode_read(rps_to_gt(rps)->uncore,
1146                                    HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1147                                    &ddcc_status, NULL) == 0)
1148                         rps->efficient_freq =
1149                                 clamp_t(u32,
1150                                         ((ddcc_status >> 8) & 0xff) * mult,
1151                                         rps->min_freq,
1152                                         rps->max_freq);
1153         }
1154 }
1155
1156 static bool rps_reset(struct intel_rps *rps)
1157 {
1158         struct drm_i915_private *i915 = rps_to_i915(rps);
1159
1160         /* force a reset */
1161         rps->power.mode = -1;
1162         rps->last_freq = -1;
1163
1164         if (rps_set(rps, rps->min_freq, true)) {
1165                 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1166                 return false;
1167         }
1168
1169         rps->cur_freq = rps->min_freq;
1170         return true;
1171 }
1172
1173 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
1174 static bool gen9_rps_enable(struct intel_rps *rps)
1175 {
1176         struct intel_gt *gt = rps_to_gt(rps);
1177         struct intel_uncore *uncore = gt->uncore;
1178
1179         /* Program defaults and thresholds for RPS */
1180         if (GRAPHICS_VER(gt->i915) == 9)
1181                 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1182                                       GEN9_FREQUENCY(rps->rp1_freq));
1183
1184         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1185
1186         rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1187
1188         return rps_reset(rps);
1189 }
1190
1191 static bool gen8_rps_enable(struct intel_rps *rps)
1192 {
1193         struct intel_uncore *uncore = rps_to_uncore(rps);
1194
1195         intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1196                               HSW_FREQUENCY(rps->rp1_freq));
1197
1198         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1199
1200         rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1201
1202         return rps_reset(rps);
1203 }
1204
1205 static bool gen6_rps_enable(struct intel_rps *rps)
1206 {
1207         struct intel_uncore *uncore = rps_to_uncore(rps);
1208
1209         /* Power down if completely idle for over 50ms */
1210         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1211         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1212
1213         rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1214                           GEN6_PM_RP_DOWN_THRESHOLD |
1215                           GEN6_PM_RP_DOWN_TIMEOUT);
1216
1217         return rps_reset(rps);
1218 }
1219
1220 static int chv_rps_max_freq(struct intel_rps *rps)
1221 {
1222         struct drm_i915_private *i915 = rps_to_i915(rps);
1223         struct intel_gt *gt = rps_to_gt(rps);
1224         u32 val;
1225
1226         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1227
1228         switch (gt->info.sseu.eu_total) {
1229         case 8:
1230                 /* (2 * 4) config */
1231                 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1232                 break;
1233         case 12:
1234                 /* (2 * 6) config */
1235                 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1236                 break;
1237         case 16:
1238                 /* (2 * 8) config */
1239         default:
1240                 /* Setting (2 * 8) Min RP0 for any other combination */
1241                 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1242                 break;
1243         }
1244
1245         return val & FB_GFX_FREQ_FUSE_MASK;
1246 }
1247
1248 static int chv_rps_rpe_freq(struct intel_rps *rps)
1249 {
1250         struct drm_i915_private *i915 = rps_to_i915(rps);
1251         u32 val;
1252
1253         val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1254         val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1255
1256         return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1257 }
1258
1259 static int chv_rps_guar_freq(struct intel_rps *rps)
1260 {
1261         struct drm_i915_private *i915 = rps_to_i915(rps);
1262         u32 val;
1263
1264         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1265
1266         return val & FB_GFX_FREQ_FUSE_MASK;
1267 }
1268
1269 static u32 chv_rps_min_freq(struct intel_rps *rps)
1270 {
1271         struct drm_i915_private *i915 = rps_to_i915(rps);
1272         u32 val;
1273
1274         val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1275         val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1276
1277         return val & FB_GFX_FREQ_FUSE_MASK;
1278 }
1279
1280 static bool chv_rps_enable(struct intel_rps *rps)
1281 {
1282         struct intel_uncore *uncore = rps_to_uncore(rps);
1283         struct drm_i915_private *i915 = rps_to_i915(rps);
1284         u32 val;
1285
1286         /* 1: Program defaults and thresholds for RPS*/
1287         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1288         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1289         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1290         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1291         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1292
1293         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1294
1295         /* 2: Enable RPS */
1296         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1297                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1298                               GEN6_RP_MEDIA_IS_GFX |
1299                               GEN6_RP_ENABLE |
1300                               GEN6_RP_UP_BUSY_AVG |
1301                               GEN6_RP_DOWN_IDLE_AVG);
1302
1303         rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1304                           GEN6_PM_RP_DOWN_THRESHOLD |
1305                           GEN6_PM_RP_DOWN_TIMEOUT);
1306
1307         /* Setting Fixed Bias */
1308         vlv_punit_get(i915);
1309
1310         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1311         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1312
1313         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1314
1315         vlv_punit_put(i915);
1316
1317         /* RPS code assumes GPLL is used */
1318         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1319                       "GPLL not enabled\n");
1320
1321         drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1322                 str_yes_no(val & GPLLENABLE));
1323         drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1324
1325         return rps_reset(rps);
1326 }
1327
1328 static int vlv_rps_guar_freq(struct intel_rps *rps)
1329 {
1330         struct drm_i915_private *i915 = rps_to_i915(rps);
1331         u32 val, rp1;
1332
1333         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1334
1335         rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1336         rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1337
1338         return rp1;
1339 }
1340
1341 static int vlv_rps_max_freq(struct intel_rps *rps)
1342 {
1343         struct drm_i915_private *i915 = rps_to_i915(rps);
1344         u32 val, rp0;
1345
1346         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1347
1348         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1349         /* Clamp to max */
1350         rp0 = min_t(u32, rp0, 0xea);
1351
1352         return rp0;
1353 }
1354
1355 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1356 {
1357         struct drm_i915_private *i915 = rps_to_i915(rps);
1358         u32 val, rpe;
1359
1360         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1361         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1362         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1363         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1364
1365         return rpe;
1366 }
1367
1368 static int vlv_rps_min_freq(struct intel_rps *rps)
1369 {
1370         struct drm_i915_private *i915 = rps_to_i915(rps);
1371         u32 val;
1372
1373         val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1374         /*
1375          * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1376          * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1377          * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1378          * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1379          * to make sure it matches what Punit accepts.
1380          */
1381         return max_t(u32, val, 0xc0);
1382 }
1383
1384 static bool vlv_rps_enable(struct intel_rps *rps)
1385 {
1386         struct intel_uncore *uncore = rps_to_uncore(rps);
1387         struct drm_i915_private *i915 = rps_to_i915(rps);
1388         u32 val;
1389
1390         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1391         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1392         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1393         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1394         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1395
1396         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1397
1398         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1399                               GEN6_RP_MEDIA_TURBO |
1400                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1401                               GEN6_RP_MEDIA_IS_GFX |
1402                               GEN6_RP_ENABLE |
1403                               GEN6_RP_UP_BUSY_AVG |
1404                               GEN6_RP_DOWN_IDLE_CONT);
1405
1406         /* WaGsvRC0ResidencyMethod:vlv */
1407         rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1408
1409         vlv_punit_get(i915);
1410
1411         /* Setting Fixed Bias */
1412         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1413         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1414
1415         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1416
1417         vlv_punit_put(i915);
1418
1419         /* RPS code assumes GPLL is used */
1420         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1421                       "GPLL not enabled\n");
1422
1423         drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1424                 str_yes_no(val & GPLLENABLE));
1425         drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1426
1427         return rps_reset(rps);
1428 }
1429
1430 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1431 {
1432         struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1433         struct intel_uncore *uncore = rps_to_uncore(rps);
1434         unsigned int t, state1, state2;
1435         u32 pxvid, ext_v;
1436         u64 corr, corr2;
1437
1438         lockdep_assert_held(&mchdev_lock);
1439
1440         pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1441         pxvid = (pxvid >> 24) & 0x7f;
1442         ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1443
1444         state1 = ext_v;
1445
1446         /* Revel in the empirically derived constants */
1447
1448         /* Correction factor in 1/100000 units */
1449         t = ips_mch_val(uncore);
1450         if (t > 80)
1451                 corr = t * 2349 + 135940;
1452         else if (t >= 50)
1453                 corr = t * 964 + 29317;
1454         else /* < 50 */
1455                 corr = t * 301 + 1004;
1456
1457         corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1458         corr2 = div_u64(corr, 100000) * ips->corr;
1459
1460         state2 = div_u64(corr2 * state1, 10000);
1461         state2 /= 100; /* convert to mW */
1462
1463         __gen5_ips_update(ips);
1464
1465         return ips->gfx_power + state2;
1466 }
1467
1468 static bool has_busy_stats(struct intel_rps *rps)
1469 {
1470         struct intel_engine_cs *engine;
1471         enum intel_engine_id id;
1472
1473         for_each_engine(engine, rps_to_gt(rps), id) {
1474                 if (!intel_engine_supports_stats(engine))
1475                         return false;
1476         }
1477
1478         return true;
1479 }
1480
1481 void intel_rps_enable(struct intel_rps *rps)
1482 {
1483         struct drm_i915_private *i915 = rps_to_i915(rps);
1484         struct intel_uncore *uncore = rps_to_uncore(rps);
1485         bool enabled = false;
1486
1487         if (!HAS_RPS(i915))
1488                 return;
1489
1490         if (rps_uses_slpc(rps))
1491                 return;
1492
1493         intel_gt_check_clock_frequency(rps_to_gt(rps));
1494
1495         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1496         if (rps->max_freq <= rps->min_freq)
1497                 /* leave disabled, no room for dynamic reclocking */;
1498         else if (IS_CHERRYVIEW(i915))
1499                 enabled = chv_rps_enable(rps);
1500         else if (IS_VALLEYVIEW(i915))
1501                 enabled = vlv_rps_enable(rps);
1502         else if (GRAPHICS_VER(i915) >= 9)
1503                 enabled = gen9_rps_enable(rps);
1504         else if (GRAPHICS_VER(i915) >= 8)
1505                 enabled = gen8_rps_enable(rps);
1506         else if (GRAPHICS_VER(i915) >= 6)
1507                 enabled = gen6_rps_enable(rps);
1508         else if (IS_IRONLAKE_M(i915))
1509                 enabled = gen5_rps_enable(rps);
1510         else
1511                 MISSING_CASE(GRAPHICS_VER(i915));
1512         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1513         if (!enabled)
1514                 return;
1515
1516         GT_TRACE(rps_to_gt(rps),
1517                  "min:%x, max:%x, freq:[%d, %d]\n",
1518                  rps->min_freq, rps->max_freq,
1519                  intel_gpu_freq(rps, rps->min_freq),
1520                  intel_gpu_freq(rps, rps->max_freq));
1521
1522         GEM_BUG_ON(rps->max_freq < rps->min_freq);
1523         GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1524
1525         GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1526         GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1527
1528         if (has_busy_stats(rps))
1529                 intel_rps_set_timer(rps);
1530         else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1531                 intel_rps_set_interrupts(rps);
1532         else
1533                 /* Ironlake currently uses intel_ips.ko */ {}
1534
1535         intel_rps_set_enabled(rps);
1536 }
1537
1538 static void gen6_rps_disable(struct intel_rps *rps)
1539 {
1540         set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1541 }
1542
1543 void intel_rps_disable(struct intel_rps *rps)
1544 {
1545         struct drm_i915_private *i915 = rps_to_i915(rps);
1546
1547         intel_rps_clear_enabled(rps);
1548         intel_rps_clear_interrupts(rps);
1549         intel_rps_clear_timer(rps);
1550
1551         if (GRAPHICS_VER(i915) >= 6)
1552                 gen6_rps_disable(rps);
1553         else if (IS_IRONLAKE_M(i915))
1554                 gen5_rps_disable(rps);
1555 }
1556
1557 static int byt_gpu_freq(struct intel_rps *rps, int val)
1558 {
1559         /*
1560          * N = val - 0xb7
1561          * Slow = Fast = GPLL ref * N
1562          */
1563         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1564 }
1565
1566 static int byt_freq_opcode(struct intel_rps *rps, int val)
1567 {
1568         return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1569 }
1570
1571 static int chv_gpu_freq(struct intel_rps *rps, int val)
1572 {
1573         /*
1574          * N = val / 2
1575          * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1576          */
1577         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1578 }
1579
1580 static int chv_freq_opcode(struct intel_rps *rps, int val)
1581 {
1582         /* CHV needs even values */
1583         return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1584 }
1585
1586 int intel_gpu_freq(struct intel_rps *rps, int val)
1587 {
1588         struct drm_i915_private *i915 = rps_to_i915(rps);
1589
1590         if (GRAPHICS_VER(i915) >= 9)
1591                 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1592                                          GEN9_FREQ_SCALER);
1593         else if (IS_CHERRYVIEW(i915))
1594                 return chv_gpu_freq(rps, val);
1595         else if (IS_VALLEYVIEW(i915))
1596                 return byt_gpu_freq(rps, val);
1597         else if (GRAPHICS_VER(i915) >= 6)
1598                 return val * GT_FREQUENCY_MULTIPLIER;
1599         else
1600                 return val;
1601 }
1602
1603 int intel_freq_opcode(struct intel_rps *rps, int val)
1604 {
1605         struct drm_i915_private *i915 = rps_to_i915(rps);
1606
1607         if (GRAPHICS_VER(i915) >= 9)
1608                 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1609                                          GT_FREQUENCY_MULTIPLIER);
1610         else if (IS_CHERRYVIEW(i915))
1611                 return chv_freq_opcode(rps, val);
1612         else if (IS_VALLEYVIEW(i915))
1613                 return byt_freq_opcode(rps, val);
1614         else if (GRAPHICS_VER(i915) >= 6)
1615                 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1616         else
1617                 return val;
1618 }
1619
1620 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1621 {
1622         struct drm_i915_private *i915 = rps_to_i915(rps);
1623
1624         rps->gpll_ref_freq =
1625                 vlv_get_cck_clock(i915, "GPLL ref",
1626                                   CCK_GPLL_CLOCK_CONTROL,
1627                                   i915->czclk_freq);
1628
1629         drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1630                 rps->gpll_ref_freq);
1631 }
1632
1633 static void vlv_rps_init(struct intel_rps *rps)
1634 {
1635         struct drm_i915_private *i915 = rps_to_i915(rps);
1636         u32 val;
1637
1638         vlv_iosf_sb_get(i915,
1639                         BIT(VLV_IOSF_SB_PUNIT) |
1640                         BIT(VLV_IOSF_SB_NC) |
1641                         BIT(VLV_IOSF_SB_CCK));
1642
1643         vlv_init_gpll_ref_freq(rps);
1644
1645         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1646         switch ((val >> 6) & 3) {
1647         case 0:
1648         case 1:
1649                 i915->mem_freq = 800;
1650                 break;
1651         case 2:
1652                 i915->mem_freq = 1066;
1653                 break;
1654         case 3:
1655                 i915->mem_freq = 1333;
1656                 break;
1657         }
1658         drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1659
1660         rps->max_freq = vlv_rps_max_freq(rps);
1661         rps->rp0_freq = rps->max_freq;
1662         drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1663                 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1664
1665         rps->efficient_freq = vlv_rps_rpe_freq(rps);
1666         drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1667                 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1668
1669         rps->rp1_freq = vlv_rps_guar_freq(rps);
1670         drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1671                 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1672
1673         rps->min_freq = vlv_rps_min_freq(rps);
1674         drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1675                 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1676
1677         vlv_iosf_sb_put(i915,
1678                         BIT(VLV_IOSF_SB_PUNIT) |
1679                         BIT(VLV_IOSF_SB_NC) |
1680                         BIT(VLV_IOSF_SB_CCK));
1681 }
1682
1683 static void chv_rps_init(struct intel_rps *rps)
1684 {
1685         struct drm_i915_private *i915 = rps_to_i915(rps);
1686         u32 val;
1687
1688         vlv_iosf_sb_get(i915,
1689                         BIT(VLV_IOSF_SB_PUNIT) |
1690                         BIT(VLV_IOSF_SB_NC) |
1691                         BIT(VLV_IOSF_SB_CCK));
1692
1693         vlv_init_gpll_ref_freq(rps);
1694
1695         val = vlv_cck_read(i915, CCK_FUSE_REG);
1696
1697         switch ((val >> 2) & 0x7) {
1698         case 3:
1699                 i915->mem_freq = 2000;
1700                 break;
1701         default:
1702                 i915->mem_freq = 1600;
1703                 break;
1704         }
1705         drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1706
1707         rps->max_freq = chv_rps_max_freq(rps);
1708         rps->rp0_freq = rps->max_freq;
1709         drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1710                 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1711
1712         rps->efficient_freq = chv_rps_rpe_freq(rps);
1713         drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1714                 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1715
1716         rps->rp1_freq = chv_rps_guar_freq(rps);
1717         drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1718                 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1719
1720         rps->min_freq = chv_rps_min_freq(rps);
1721         drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1722                 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1723
1724         vlv_iosf_sb_put(i915,
1725                         BIT(VLV_IOSF_SB_PUNIT) |
1726                         BIT(VLV_IOSF_SB_NC) |
1727                         BIT(VLV_IOSF_SB_CCK));
1728
1729         drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1730                                    rps->rp1_freq | rps->min_freq) & 1,
1731                       "Odd GPU freq values\n");
1732 }
1733
1734 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1735 {
1736         ei->ktime = ktime_get_raw();
1737         ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1738         ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1739 }
1740
1741 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1742 {
1743         struct intel_uncore *uncore = rps_to_uncore(rps);
1744         const struct intel_rps_ei *prev = &rps->ei;
1745         struct intel_rps_ei now;
1746         u32 events = 0;
1747
1748         if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1749                 return 0;
1750
1751         vlv_c0_read(uncore, &now);
1752
1753         if (prev->ktime) {
1754                 u64 time, c0;
1755                 u32 render, media;
1756
1757                 time = ktime_us_delta(now.ktime, prev->ktime);
1758
1759                 time *= rps_to_i915(rps)->czclk_freq;
1760
1761                 /* Workload can be split between render + media,
1762                  * e.g. SwapBuffers being blitted in X after being rendered in
1763                  * mesa. To account for this we need to combine both engines
1764                  * into our activity counter.
1765                  */
1766                 render = now.render_c0 - prev->render_c0;
1767                 media = now.media_c0 - prev->media_c0;
1768                 c0 = max(render, media);
1769                 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1770
1771                 if (c0 > time * rps->power.up_threshold)
1772                         events = GEN6_PM_RP_UP_THRESHOLD;
1773                 else if (c0 < time * rps->power.down_threshold)
1774                         events = GEN6_PM_RP_DOWN_THRESHOLD;
1775         }
1776
1777         rps->ei = now;
1778         return events;
1779 }
1780
1781 static void rps_work(struct work_struct *work)
1782 {
1783         struct intel_rps *rps = container_of(work, typeof(*rps), work);
1784         struct intel_gt *gt = rps_to_gt(rps);
1785         struct drm_i915_private *i915 = rps_to_i915(rps);
1786         bool client_boost = false;
1787         int new_freq, adj, min, max;
1788         u32 pm_iir = 0;
1789
1790         spin_lock_irq(&gt->irq_lock);
1791         pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1792         client_boost = atomic_read(&rps->num_waiters);
1793         spin_unlock_irq(&gt->irq_lock);
1794
1795         /* Make sure we didn't queue anything we're not going to process. */
1796         if (!pm_iir && !client_boost)
1797                 goto out;
1798
1799         mutex_lock(&rps->lock);
1800         if (!intel_rps_is_active(rps)) {
1801                 mutex_unlock(&rps->lock);
1802                 return;
1803         }
1804
1805         pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1806
1807         adj = rps->last_adj;
1808         new_freq = rps->cur_freq;
1809         min = rps->min_freq_softlimit;
1810         max = rps->max_freq_softlimit;
1811         if (client_boost)
1812                 max = rps->max_freq;
1813
1814         GT_TRACE(gt,
1815                  "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1816                  pm_iir, str_yes_no(client_boost),
1817                  adj, new_freq, min, max);
1818
1819         if (client_boost && new_freq < rps->boost_freq) {
1820                 new_freq = rps->boost_freq;
1821                 adj = 0;
1822         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1823                 if (adj > 0)
1824                         adj *= 2;
1825                 else /* CHV needs even encode values */
1826                         adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1827
1828                 if (new_freq >= rps->max_freq_softlimit)
1829                         adj = 0;
1830         } else if (client_boost) {
1831                 adj = 0;
1832         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1833                 if (rps->cur_freq > rps->efficient_freq)
1834                         new_freq = rps->efficient_freq;
1835                 else if (rps->cur_freq > rps->min_freq_softlimit)
1836                         new_freq = rps->min_freq_softlimit;
1837                 adj = 0;
1838         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1839                 if (adj < 0)
1840                         adj *= 2;
1841                 else /* CHV needs even encode values */
1842                         adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1843
1844                 if (new_freq <= rps->min_freq_softlimit)
1845                         adj = 0;
1846         } else { /* unknown event */
1847                 adj = 0;
1848         }
1849
1850         /*
1851          * sysfs frequency limits may have snuck in while
1852          * servicing the interrupt
1853          */
1854         new_freq += adj;
1855         new_freq = clamp_t(int, new_freq, min, max);
1856
1857         if (intel_rps_set(rps, new_freq)) {
1858                 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1859                 adj = 0;
1860         }
1861         rps->last_adj = adj;
1862
1863         mutex_unlock(&rps->lock);
1864
1865 out:
1866         spin_lock_irq(&gt->irq_lock);
1867         gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1868         spin_unlock_irq(&gt->irq_lock);
1869 }
1870
1871 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1872 {
1873         struct intel_gt *gt = rps_to_gt(rps);
1874         const u32 events = rps->pm_events & pm_iir;
1875
1876         lockdep_assert_held(&gt->irq_lock);
1877
1878         if (unlikely(!events))
1879                 return;
1880
1881         GT_TRACE(gt, "irq events:%x\n", events);
1882
1883         gen6_gt_pm_mask_irq(gt, events);
1884
1885         rps->pm_iir |= events;
1886         schedule_work(&rps->work);
1887 }
1888
1889 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1890 {
1891         struct intel_gt *gt = rps_to_gt(rps);
1892         u32 events;
1893
1894         events = pm_iir & rps->pm_events;
1895         if (events) {
1896                 spin_lock(&gt->irq_lock);
1897
1898                 GT_TRACE(gt, "irq events:%x\n", events);
1899
1900                 gen6_gt_pm_mask_irq(gt, events);
1901                 rps->pm_iir |= events;
1902
1903                 schedule_work(&rps->work);
1904                 spin_unlock(&gt->irq_lock);
1905         }
1906
1907         if (GRAPHICS_VER(gt->i915) >= 8)
1908                 return;
1909
1910         if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1911                 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
1912
1913         if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1914                 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1915 }
1916
1917 void gen5_rps_irq_handler(struct intel_rps *rps)
1918 {
1919         struct intel_uncore *uncore = rps_to_uncore(rps);
1920         u32 busy_up, busy_down, max_avg, min_avg;
1921         u8 new_freq;
1922
1923         spin_lock(&mchdev_lock);
1924
1925         intel_uncore_write16(uncore,
1926                              MEMINTRSTS,
1927                              intel_uncore_read(uncore, MEMINTRSTS));
1928
1929         intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1930         busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1931         busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1932         max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1933         min_avg = intel_uncore_read(uncore, RCBMINAVG);
1934
1935         /* Handle RCS change request from hw */
1936         new_freq = rps->cur_freq;
1937         if (busy_up > max_avg)
1938                 new_freq++;
1939         else if (busy_down < min_avg)
1940                 new_freq--;
1941         new_freq = clamp(new_freq,
1942                          rps->min_freq_softlimit,
1943                          rps->max_freq_softlimit);
1944
1945         if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
1946                 rps->cur_freq = new_freq;
1947
1948         spin_unlock(&mchdev_lock);
1949 }
1950
1951 void intel_rps_init_early(struct intel_rps *rps)
1952 {
1953         mutex_init(&rps->lock);
1954         mutex_init(&rps->power.mutex);
1955
1956         INIT_WORK(&rps->work, rps_work);
1957         timer_setup(&rps->timer, rps_timer, 0);
1958
1959         atomic_set(&rps->num_waiters, 0);
1960 }
1961
1962 void intel_rps_init(struct intel_rps *rps)
1963 {
1964         struct drm_i915_private *i915 = rps_to_i915(rps);
1965
1966         if (rps_uses_slpc(rps))
1967                 return;
1968
1969         if (IS_CHERRYVIEW(i915))
1970                 chv_rps_init(rps);
1971         else if (IS_VALLEYVIEW(i915))
1972                 vlv_rps_init(rps);
1973         else if (GRAPHICS_VER(i915) >= 6)
1974                 gen6_rps_init(rps);
1975         else if (IS_IRONLAKE_M(i915))
1976                 gen5_rps_init(rps);
1977
1978         /* Derive initial user preferences/limits from the hardware limits */
1979         rps->max_freq_softlimit = rps->max_freq;
1980         rps->min_freq_softlimit = rps->min_freq;
1981
1982         /* After setting max-softlimit, find the overclock max freq */
1983         if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
1984                 u32 params = 0;
1985
1986                 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, &params, NULL);
1987                 if (params & BIT(31)) { /* OC supported */
1988                         drm_dbg(&i915->drm,
1989                                 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
1990                                 (rps->max_freq & 0xff) * 50,
1991                                 (params & 0xff) * 50);
1992                         rps->max_freq = params & 0xff;
1993                 }
1994         }
1995
1996         /* Finally allow us to boost to max by default */
1997         rps->boost_freq = rps->max_freq;
1998         rps->idle_freq = rps->min_freq;
1999
2000         /* Start in the middle, from here we will autotune based on workload */
2001         rps->cur_freq = rps->efficient_freq;
2002
2003         rps->pm_intrmsk_mbz = 0;
2004
2005         /*
2006          * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2007          * if GEN6_PM_UP_EI_EXPIRED is masked.
2008          *
2009          * TODO: verify if this can be reproduced on VLV,CHV.
2010          */
2011         if (GRAPHICS_VER(i915) <= 7)
2012                 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2013
2014         if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2015                 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2016
2017         /* GuC needs ARAT expired interrupt unmasked */
2018         if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))
2019                 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2020 }
2021
2022 void intel_rps_sanitize(struct intel_rps *rps)
2023 {
2024         if (rps_uses_slpc(rps))
2025                 return;
2026
2027         if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2028                 rps_disable_interrupts(rps);
2029 }
2030
2031 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2032 {
2033         struct drm_i915_private *i915 = rps_to_i915(rps);
2034         u32 cagf;
2035
2036         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2037                 cagf = (rpstat >> 8) & 0xff;
2038         else if (GRAPHICS_VER(i915) >= 9)
2039                 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
2040         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2041                 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
2042         else if (GRAPHICS_VER(i915) >= 6)
2043                 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
2044         else
2045                 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >>
2046                                         MEMSTAT_PSTATE_SHIFT);
2047
2048         return cagf;
2049 }
2050
2051 static u32 read_cagf(struct intel_rps *rps)
2052 {
2053         struct drm_i915_private *i915 = rps_to_i915(rps);
2054         struct intel_uncore *uncore = rps_to_uncore(rps);
2055         u32 freq;
2056
2057         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2058                 vlv_punit_get(i915);
2059                 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
2060                 vlv_punit_put(i915);
2061         } else if (GRAPHICS_VER(i915) >= 6) {
2062                 freq = intel_uncore_read(uncore, GEN6_RPSTAT1);
2063         } else {
2064                 freq = intel_uncore_read(uncore, MEMSTAT_ILK);
2065         }
2066
2067         return intel_rps_get_cagf(rps, freq);
2068 }
2069
2070 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2071 {
2072         struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2073         intel_wakeref_t wakeref;
2074         u32 freq = 0;
2075
2076         with_intel_runtime_pm_if_in_use(rpm, wakeref)
2077                 freq = intel_gpu_freq(rps, read_cagf(rps));
2078
2079         return freq;
2080 }
2081
2082 u32 intel_rps_read_punit_req(struct intel_rps *rps)
2083 {
2084         struct intel_uncore *uncore = rps_to_uncore(rps);
2085         struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2086         intel_wakeref_t wakeref;
2087         u32 freq = 0;
2088
2089         with_intel_runtime_pm_if_in_use(rpm, wakeref)
2090                 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2091
2092         return freq;
2093 }
2094
2095 static u32 intel_rps_get_req(u32 pureq)
2096 {
2097         u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2098
2099         return req;
2100 }
2101
2102 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2103 {
2104         u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));
2105
2106         return intel_gpu_freq(rps, freq);
2107 }
2108
2109 u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2110 {
2111         if (rps_uses_slpc(rps))
2112                 return intel_rps_read_punit_req_frequency(rps);
2113         else
2114                 return intel_gpu_freq(rps, rps->cur_freq);
2115 }
2116
2117 u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2118 {
2119         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2120
2121         if (rps_uses_slpc(rps))
2122                 return slpc->max_freq_softlimit;
2123         else
2124                 return intel_gpu_freq(rps, rps->max_freq_softlimit);
2125 }
2126
2127 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2128 {
2129         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2130
2131         if (rps_uses_slpc(rps))
2132                 return slpc->rp0_freq;
2133         else
2134                 return intel_gpu_freq(rps, rps->rp0_freq);
2135 }
2136
2137 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2138 {
2139         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2140
2141         if (rps_uses_slpc(rps))
2142                 return slpc->rp1_freq;
2143         else
2144                 return intel_gpu_freq(rps, rps->rp1_freq);
2145 }
2146
2147 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2148 {
2149         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2150
2151         if (rps_uses_slpc(rps))
2152                 return slpc->min_freq;
2153         else
2154                 return intel_gpu_freq(rps, rps->min_freq);
2155 }
2156
2157 static int set_max_freq(struct intel_rps *rps, u32 val)
2158 {
2159         struct drm_i915_private *i915 = rps_to_i915(rps);
2160         int ret = 0;
2161
2162         mutex_lock(&rps->lock);
2163
2164         val = intel_freq_opcode(rps, val);
2165         if (val < rps->min_freq ||
2166             val > rps->max_freq ||
2167             val < rps->min_freq_softlimit) {
2168                 ret = -EINVAL;
2169                 goto unlock;
2170         }
2171
2172         if (val > rps->rp0_freq)
2173                 drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2174                         intel_gpu_freq(rps, val));
2175
2176         rps->max_freq_softlimit = val;
2177
2178         val = clamp_t(int, rps->cur_freq,
2179                       rps->min_freq_softlimit,
2180                       rps->max_freq_softlimit);
2181
2182         /*
2183          * We still need *_set_rps to process the new max_delay and
2184          * update the interrupt limits and PMINTRMSK even though
2185          * frequency request may be unchanged.
2186          */
2187         intel_rps_set(rps, val);
2188
2189 unlock:
2190         mutex_unlock(&rps->lock);
2191
2192         return ret;
2193 }
2194
2195 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2196 {
2197         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2198
2199         if (rps_uses_slpc(rps))
2200                 return intel_guc_slpc_set_max_freq(slpc, val);
2201         else
2202                 return set_max_freq(rps, val);
2203 }
2204
2205 u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2206 {
2207         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2208
2209         if (rps_uses_slpc(rps))
2210                 return slpc->min_freq_softlimit;
2211         else
2212                 return intel_gpu_freq(rps, rps->min_freq_softlimit);
2213 }
2214
2215 static int set_min_freq(struct intel_rps *rps, u32 val)
2216 {
2217         int ret = 0;
2218
2219         mutex_lock(&rps->lock);
2220
2221         val = intel_freq_opcode(rps, val);
2222         if (val < rps->min_freq ||
2223             val > rps->max_freq ||
2224             val > rps->max_freq_softlimit) {
2225                 ret = -EINVAL;
2226                 goto unlock;
2227         }
2228
2229         rps->min_freq_softlimit = val;
2230
2231         val = clamp_t(int, rps->cur_freq,
2232                       rps->min_freq_softlimit,
2233                       rps->max_freq_softlimit);
2234
2235         /*
2236          * We still need *_set_rps to process the new min_delay and
2237          * update the interrupt limits and PMINTRMSK even though
2238          * frequency request may be unchanged.
2239          */
2240         intel_rps_set(rps, val);
2241
2242 unlock:
2243         mutex_unlock(&rps->lock);
2244
2245         return ret;
2246 }
2247
2248 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2249 {
2250         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2251
2252         if (rps_uses_slpc(rps))
2253                 return intel_guc_slpc_set_min_freq(slpc, val);
2254         else
2255                 return set_min_freq(rps, val);
2256 }
2257
2258 static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2259 {
2260         struct intel_uncore *uncore = rps_to_uncore(rps);
2261         u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2262
2263         /* Allow punit to process software requests */
2264         intel_uncore_write(uncore, GEN6_RP_CONTROL, state);
2265 }
2266
2267 void intel_rps_raise_unslice(struct intel_rps *rps)
2268 {
2269         struct intel_uncore *uncore = rps_to_uncore(rps);
2270
2271         mutex_lock(&rps->lock);
2272
2273         if (rps_uses_slpc(rps)) {
2274                 /* RP limits have not been initialized yet for SLPC path */
2275                 struct intel_rps_freq_caps caps;
2276
2277                 gen6_rps_get_freq_caps(rps, &caps);
2278
2279                 intel_rps_set_manual(rps, true);
2280                 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2281                                    ((caps.rp0_freq <<
2282                                    GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2283                                    GEN9_IGNORE_SLICE_RATIO));
2284                 intel_rps_set_manual(rps, false);
2285         } else {
2286                 intel_rps_set(rps, rps->rp0_freq);
2287         }
2288
2289         mutex_unlock(&rps->lock);
2290 }
2291
2292 void intel_rps_lower_unslice(struct intel_rps *rps)
2293 {
2294         struct intel_uncore *uncore = rps_to_uncore(rps);
2295
2296         mutex_lock(&rps->lock);
2297
2298         if (rps_uses_slpc(rps)) {
2299                 /* RP limits have not been initialized yet for SLPC path */
2300                 struct intel_rps_freq_caps caps;
2301
2302                 gen6_rps_get_freq_caps(rps, &caps);
2303
2304                 intel_rps_set_manual(rps, true);
2305                 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2306                                    ((caps.min_freq <<
2307                                    GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2308                                    GEN9_IGNORE_SLICE_RATIO));
2309                 intel_rps_set_manual(rps, false);
2310         } else {
2311                 intel_rps_set(rps, rps->min_freq);
2312         }
2313
2314         mutex_unlock(&rps->lock);
2315 }
2316
2317 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2318 {
2319         struct intel_gt *gt = rps_to_gt(rps);
2320         intel_wakeref_t wakeref;
2321         u32 val;
2322
2323         with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2324                 val = intel_uncore_read(gt->uncore, reg32);
2325
2326         return val;
2327 }
2328
2329 bool rps_read_mask_mmio(struct intel_rps *rps,
2330                         i915_reg_t reg32, u32 mask)
2331 {
2332         return rps_read_mmio(rps, reg32) & mask;
2333 }
2334
2335 /* External interface for intel_ips.ko */
2336
2337 static struct drm_i915_private __rcu *ips_mchdev;
2338
2339 /**
2340  * Tells the intel_ips driver that the i915 driver is now loaded, if
2341  * IPS got loaded first.
2342  *
2343  * This awkward dance is so that neither module has to depend on the
2344  * other in order for IPS to do the appropriate communication of
2345  * GPU turbo limits to i915.
2346  */
2347 static void
2348 ips_ping_for_i915_load(void)
2349 {
2350         void (*link)(void);
2351
2352         link = symbol_get(ips_link_to_i915_driver);
2353         if (link) {
2354                 link();
2355                 symbol_put(ips_link_to_i915_driver);
2356         }
2357 }
2358
2359 void intel_rps_driver_register(struct intel_rps *rps)
2360 {
2361         struct intel_gt *gt = rps_to_gt(rps);
2362
2363         /*
2364          * We only register the i915 ips part with intel-ips once everything is
2365          * set up, to avoid intel-ips sneaking in and reading bogus values.
2366          */
2367         if (GRAPHICS_VER(gt->i915) == 5) {
2368                 GEM_BUG_ON(ips_mchdev);
2369                 rcu_assign_pointer(ips_mchdev, gt->i915);
2370                 ips_ping_for_i915_load();
2371         }
2372 }
2373
2374 void intel_rps_driver_unregister(struct intel_rps *rps)
2375 {
2376         if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2377                 rcu_assign_pointer(ips_mchdev, NULL);
2378 }
2379
2380 static struct drm_i915_private *mchdev_get(void)
2381 {
2382         struct drm_i915_private *i915;
2383
2384         rcu_read_lock();
2385         i915 = rcu_dereference(ips_mchdev);
2386         if (i915 && !kref_get_unless_zero(&i915->drm.ref))
2387                 i915 = NULL;
2388         rcu_read_unlock();
2389
2390         return i915;
2391 }
2392
2393 /**
2394  * i915_read_mch_val - return value for IPS use
2395  *
2396  * Calculate and return a value for the IPS driver to use when deciding whether
2397  * we have thermal and power headroom to increase CPU or GPU power budget.
2398  */
2399 unsigned long i915_read_mch_val(void)
2400 {
2401         struct drm_i915_private *i915;
2402         unsigned long chipset_val = 0;
2403         unsigned long graphics_val = 0;
2404         intel_wakeref_t wakeref;
2405
2406         i915 = mchdev_get();
2407         if (!i915)
2408                 return 0;
2409
2410         with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2411                 struct intel_ips *ips = &to_gt(i915)->rps.ips;
2412
2413                 spin_lock_irq(&mchdev_lock);
2414                 chipset_val = __ips_chipset_val(ips);
2415                 graphics_val = __ips_gfx_val(ips);
2416                 spin_unlock_irq(&mchdev_lock);
2417         }
2418
2419         drm_dev_put(&i915->drm);
2420         return chipset_val + graphics_val;
2421 }
2422 EXPORT_SYMBOL_GPL(i915_read_mch_val);
2423
2424 /**
2425  * i915_gpu_raise - raise GPU frequency limit
2426  *
2427  * Raise the limit; IPS indicates we have thermal headroom.
2428  */
2429 bool i915_gpu_raise(void)
2430 {
2431         struct drm_i915_private *i915;
2432         struct intel_rps *rps;
2433
2434         i915 = mchdev_get();
2435         if (!i915)
2436                 return false;
2437
2438         rps = &to_gt(i915)->rps;
2439
2440         spin_lock_irq(&mchdev_lock);
2441         if (rps->max_freq_softlimit < rps->max_freq)
2442                 rps->max_freq_softlimit++;
2443         spin_unlock_irq(&mchdev_lock);
2444
2445         drm_dev_put(&i915->drm);
2446         return true;
2447 }
2448 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2449
2450 /**
2451  * i915_gpu_lower - lower GPU frequency limit
2452  *
2453  * IPS indicates we're close to a thermal limit, so throttle back the GPU
2454  * frequency maximum.
2455  */
2456 bool i915_gpu_lower(void)
2457 {
2458         struct drm_i915_private *i915;
2459         struct intel_rps *rps;
2460
2461         i915 = mchdev_get();
2462         if (!i915)
2463                 return false;
2464
2465         rps = &to_gt(i915)->rps;
2466
2467         spin_lock_irq(&mchdev_lock);
2468         if (rps->max_freq_softlimit > rps->min_freq)
2469                 rps->max_freq_softlimit--;
2470         spin_unlock_irq(&mchdev_lock);
2471
2472         drm_dev_put(&i915->drm);
2473         return true;
2474 }
2475 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2476
2477 /**
2478  * i915_gpu_busy - indicate GPU business to IPS
2479  *
2480  * Tell the IPS driver whether or not the GPU is busy.
2481  */
2482 bool i915_gpu_busy(void)
2483 {
2484         struct drm_i915_private *i915;
2485         bool ret;
2486
2487         i915 = mchdev_get();
2488         if (!i915)
2489                 return false;
2490
2491         ret = to_gt(i915)->awake;
2492
2493         drm_dev_put(&i915->drm);
2494         return ret;
2495 }
2496 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2497
2498 /**
2499  * i915_gpu_turbo_disable - disable graphics turbo
2500  *
2501  * Disable graphics turbo by resetting the max frequency and setting the
2502  * current frequency to the default.
2503  */
2504 bool i915_gpu_turbo_disable(void)
2505 {
2506         struct drm_i915_private *i915;
2507         struct intel_rps *rps;
2508         bool ret;
2509
2510         i915 = mchdev_get();
2511         if (!i915)
2512                 return false;
2513
2514         rps = &to_gt(i915)->rps;
2515
2516         spin_lock_irq(&mchdev_lock);
2517         rps->max_freq_softlimit = rps->min_freq;
2518         ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);
2519         spin_unlock_irq(&mchdev_lock);
2520
2521         drm_dev_put(&i915->drm);
2522         return ret;
2523 }
2524 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2525
2526 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2527 #include "selftest_rps.c"
2528 #include "selftest_slpc.c"
2529 #endif