drivers/gpu/drm/i915/intel_clock_gating.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include "display/intel_de.h"
  29 #include "display/intel_display.h"
  30 #include "display/intel_display_trace.h"
  31 #include "display/skl_watermark.h"
  32
  33 #include "gt/intel_engine_regs.h"
  34 #include "gt/intel_gt.h"
  35 #include "gt/intel_gt_mcr.h"
  36 #include "gt/intel_gt_regs.h"
  37
  38 #include "i915_drv.h"
  39 #include "i915_reg.h"
  40 #include "intel_clock_gating.h"
  41 #include "intel_mchbar_regs.h"
  42 #include "vlv_sideband.h"
  43
  44 struct drm_i915_clock_gating_funcs {
  45         void (*init_clock_gating)(struct drm_i915_private *i915);
  46 };
  47
  48 static void gen9_init_clock_gating(struct drm_i915_private *i915)
  49 {
  50         if (HAS_LLC(i915)) {
  51                 /*
  52                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  53                  * Display WA #0390: skl,kbl
  54                  *
  55                  * Must match Sampler, Pixel Back End, and Media. See
  56                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  57                  */
  58                 intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
  59         }
  60
  61         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  62         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
  63
  64         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  65         intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
  66
  67         /*
  68          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
  69          * Display WA #0859: skl,bxt,kbl,glk,cfl
  70          */
  71         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
  72 }
  73
  74 static void bxt_init_clock_gating(struct drm_i915_private *i915)
  75 {
  76         gen9_init_clock_gating(i915);
  77
  78         /* WaDisableSDEUnitClockGating:bxt */
  79         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  80
  81         /*
  82          * FIXME:
  83          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
  84          */
  85         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
  86
  87         /*
  88          * Wa: Backlight PWM may stop in the asserted state, causing backlight
  89          * to stay fully on.
  90          */
  91         intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
  92                            intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
  93                            PWM1_GATING_DIS | PWM2_GATING_DIS);
  94
  95         /*
  96          * Lower the display internal timeout.
  97          * This is needed to avoid any hard hangs when DSI port PLL
  98          * is off and a MMIO access is attempted by any privilege
  99          * application, using batch buffers or any other means.
 100          */
 101         intel_uncore_write(&i915->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 102
 103         /*
 104          * WaFbcTurnOffFbcWatermark:bxt
 105          * Display WA #0562: bxt
 106          */
 107         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 108
 109         /*
 110          * WaFbcHighMemBwCorruptionAvoidance:bxt
 111          * Display WA #0883: bxt
 112          */
 113         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 114 }
 115
 116 static void glk_init_clock_gating(struct drm_i915_private *i915)
 117 {
 118         gen9_init_clock_gating(i915);
 119
 120         /*
 121          * WaDisablePWMClockGating:glk
 122          * Backlight PWM may stop in the asserted state, causing backlight
 123          * to stay fully on.
 124          */
 125         intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
 126                            intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
 127                            PWM1_GATING_DIS | PWM2_GATING_DIS);
 128 }
 129
 130 static void ibx_init_clock_gating(struct drm_i915_private *i915)
 131 {
 132         /*
 133          * On Ibex Peak and Cougar Point, we need to disable clock
 134          * gating for the panel power sequencer or it will fail to
 135          * start up when no ports are active.
 136          */
 137         intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
 138 }
 139
 140 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
 141 {
 142         enum pipe pipe;
 143
 144         for_each_pipe(dev_priv, pipe) {
 145                 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
 146
 147                 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
 148                 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
 149         }
 150 }
 151
 152 static void ilk_init_clock_gating(struct drm_i915_private *i915)
 153 {
 154         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 155
 156         /*
 157          * Required for FBC
 158          * WaFbcDisableDpfcClockGating:ilk
 159          */
 160         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
 161                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
 162                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
 163
 164         intel_uncore_write(&i915->uncore, PCH_3DCGDIS0,
 165                            MARIUNIT_CLOCK_GATE_DISABLE |
 166                            SVSMUNIT_CLOCK_GATE_DISABLE);
 167         intel_uncore_write(&i915->uncore, PCH_3DCGDIS1,
 168                            VFMUNIT_CLOCK_GATE_DISABLE);
 169
 170         /*
 171          * According to the spec the following bits should be set in
 172          * order to enable memory self-refresh
 173          * The bit 22/21 of 0x42004
 174          * The bit 5 of 0x42020
 175          * The bit 15 of 0x45000
 176          */
 177         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
 178                            (intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
 179                             ILK_DPARB_GATE | ILK_VSDPFD_FULL));
 180         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
 181         intel_uncore_write(&i915->uncore, DISP_ARB_CTL,
 182                            (intel_uncore_read(&i915->uncore, DISP_ARB_CTL) |
 183                             DISP_FBC_WM_DIS));
 184
 185         /*
 186          * Based on the document from hardware guys the following bits
 187          * should be set unconditionally in order to enable FBC.
 188          * The bit 22 of 0x42000
 189          * The bit 22 of 0x42004
 190          * The bit 7,8,9 of 0x42020.
 191          */
 192         if (IS_IRONLAKE_M(i915)) {
 193                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
 194                 intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 195                 intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
 196         }
 197
 198         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 199
 200         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 201
 202         g4x_disable_trickle_feed(i915);
 203
 204         ibx_init_clock_gating(i915);
 205 }
 206
 207 static void cpt_init_clock_gating(struct drm_i915_private *i915)
 208 {
 209         enum pipe pipe;
 210         u32 val;
 211
 212         /*
 213          * On Ibex Peak and Cougar Point, we need to disable clock
 214          * gating for the panel power sequencer or it will fail to
 215          * start up when no ports are active.
 216          */
 217         intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
 218                            PCH_DPLUNIT_CLOCK_GATE_DISABLE |
 219                            PCH_CPUNIT_CLOCK_GATE_DISABLE);
 220         intel_uncore_rmw(&i915->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
 221         /* The below fixes the weird display corruption, a few pixels shifted
 222          * downward, on (only) LVDS of some HP laptops with IVY.
 223          */
 224         for_each_pipe(i915, pipe) {
 225                 val = intel_uncore_read(&i915->uncore, TRANS_CHICKEN2(pipe));
 226                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 227                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 228                 if (i915->display.vbt.fdi_rx_polarity_inverted)
 229                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 230                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
 231                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
 232                 intel_uncore_write(&i915->uncore, TRANS_CHICKEN2(pipe), val);
 233         }
 234         /* WADP0ClockGatingDisable */
 235         for_each_pipe(i915, pipe) {
 236                 intel_uncore_write(&i915->uncore, TRANS_CHICKEN1(pipe),
 237                                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 238         }
 239 }
 240
 241 static void gen6_check_mch_setup(struct drm_i915_private *i915)
 242 {
 243         u32 tmp;
 244
 245         tmp = intel_uncore_read(&i915->uncore, MCH_SSKPD);
 246         if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
 247                 drm_dbg_kms(&i915->drm,
 248                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
 249                             tmp);
 250 }
 251
 252 static void gen6_init_clock_gating(struct drm_i915_private *i915)
 253 {
 254         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 255
 256         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 257
 258         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 259
 260         intel_uncore_write(&i915->uncore, GEN6_UCGCTL1,
 261                            intel_uncore_read(&i915->uncore, GEN6_UCGCTL1) |
 262                            GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
 263                            GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 264
 265         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
 266          * gating disable must be set.  Failure to set it results in
 267          * flickering pixels due to Z write ordering failures after
 268          * some amount of runtime in the Mesa "fire" demo, and Unigine
 269          * Sanctuary and Tropics, and apparently anything else with
 270          * alpha test or pixel discard.
 271          *
 272          * According to the spec, bit 11 (RCCUNIT) must also be set,
 273          * but we didn't debug actual testcases to find it out.
 274          *
 275          * WaDisableRCCUnitClockGating:snb
 276          * WaDisableRCPBUnitClockGating:snb
 277          */
 278         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 279                            GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
 280                            GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
 281
 282         /*
 283          * According to the spec the following bits should be
 284          * set in order to enable memory self-refresh and fbc:
 285          * The bit21 and bit22 of 0x42000
 286          * The bit21 and bit22 of 0x42004
 287          * The bit5 and bit7 of 0x42020
 288          * The bit14 of 0x70180
 289          * The bit14 of 0x71180
 290          *
 291          * WaFbcAsynchFlipDisableFbcQueue:snb
 292          */
 293         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN1,
 294                            intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN1) |
 295                            ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
 296         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
 297                            intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
 298                            ILK_DPARB_GATE | ILK_VSDPFD_FULL);
 299         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D,
 300                            intel_uncore_read(&i915->uncore, ILK_DSPCLK_GATE_D) |
 301                            ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
 302                            ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
 303
 304         g4x_disable_trickle_feed(i915);
 305
 306         cpt_init_clock_gating(i915);
 307
 308         gen6_check_mch_setup(i915);
 309 }
 310
 311 static void lpt_init_clock_gating(struct drm_i915_private *i915)
 312 {
 313         /*
 314          * TODO: this bit should only be enabled when really needed, then
 315          * disabled when not needed anymore in order to save power.
 316          */
 317         if (HAS_PCH_LPT_LP(i915))
 318                 intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D,
 319                                  0, PCH_LP_PARTITION_LEVEL_DISABLE);
 320
 321         /* WADPOClockGatingDisable:hsw */
 322         intel_uncore_rmw(&i915->uncore, TRANS_CHICKEN1(PIPE_A),
 323                          0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 324 }
 325
 326 static void gen8_set_l3sqc_credits(struct drm_i915_private *i915,
 327                                    int general_prio_credits,
 328                                    int high_prio_credits)
 329 {
 330         u32 misccpctl;
 331         u32 val;
 332
 333         /* WaTempDisableDOPClkGating:bdw */
 334         misccpctl = intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 335                                      GEN7_DOP_CLOCK_GATE_ENABLE, 0);
 336
 337         val = intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
 338         val &= ~L3_PRIO_CREDITS_MASK;
 339         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
 340         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
 341         intel_gt_mcr_multicast_write(to_gt(i915), GEN8_L3SQCREG1, val);
 342
 343         /*
 344          * Wait at least 100 clocks before re-enabling clock gating.
 345          * See the definition of L3SQCREG1 in BSpec.
 346          */
 347         intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
 348         udelay(1);
 349         intel_uncore_write(&i915->uncore, GEN7_MISCCPCTL, misccpctl);
 350 }
 351
 352 static void icl_init_clock_gating(struct drm_i915_private *i915)
 353 {
 354         /* Wa_1409120013:icl,ehl */
 355         intel_uncore_write(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 356                            DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 357
 358         /*Wa_14010594013:icl, ehl */
 359         intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1,
 360                          0, ICL_DELAY_PMRSP);
 361 }
 362
 363 static void gen12lp_init_clock_gating(struct drm_i915_private *i915)
 364 {
 365         /* Wa_1409120013 */
 366         if (DISPLAY_VER(i915) == 12)
 367                 intel_uncore_write(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 368                                    DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 369
 370         /* Wa_14013723622:tgl,rkl,dg1,adl-s */
 371         if (DISPLAY_VER(i915) == 12)
 372                 intel_uncore_rmw(&i915->uncore, CLKREQ_POLICY,
 373                                  CLKREQ_POLICY_MEM_UP_OVRD, 0);
 374 }
 375
 376 static void adlp_init_clock_gating(struct drm_i915_private *i915)
 377 {
 378         gen12lp_init_clock_gating(i915);
 379
 380         /* Wa_22011091694:adlp */
 381         intel_de_rmw(i915, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
 382
 383         /* Bspec/49189 Initialize Sequence */
 384         intel_de_rmw(i915, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
 385 }
 386
 387 static void xehpsdv_init_clock_gating(struct drm_i915_private *i915)
 388 {
 389         /* Wa_22010146351:xehpsdv */
 390         if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
 391                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
 392 }
 393
 394 static void dg2_init_clock_gating(struct drm_i915_private *i915)
 395 {
 396         /* Wa_22010954014:dg2 */
 397         intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
 398                          SGSI_SIDECLK_DIS);
 399
 400         /*
 401          * Wa_14010733611:dg2_g10
 402          * Wa_22010146351:dg2_g10
 403          */
 404         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
 405                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
 406                                  SGR_DIS | SGGI_DIS);
 407 }
 408
 409 static void pvc_init_clock_gating(struct drm_i915_private *i915)
 410 {
 411         /* Wa_14012385139:pvc */
 412         if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
 413                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
 414
 415         /* Wa_22010954014:pvc */
 416         if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
 417                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
 418 }
 419
 420 static void cnp_init_clock_gating(struct drm_i915_private *i915)
 421 {
 422         if (!HAS_PCH_CNP(i915))
 423                 return;
 424
 425         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
 426         intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
 427 }
 428
 429 static void cfl_init_clock_gating(struct drm_i915_private *i915)
 430 {
 431         cnp_init_clock_gating(i915);
 432         gen9_init_clock_gating(i915);
 433
 434         /* WAC6entrylatency:cfl */
 435         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 436
 437         /*
 438          * WaFbcTurnOffFbcWatermark:cfl
 439          * Display WA #0562: cfl
 440          */
 441         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 442
 443         /*
 444          * WaFbcNukeOnHostModify:cfl
 445          * Display WA #0873: cfl
 446          */
 447         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 448                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 449 }
 450
 451 static void kbl_init_clock_gating(struct drm_i915_private *i915)
 452 {
 453         gen9_init_clock_gating(i915);
 454
 455         /* WAC6entrylatency:kbl */
 456         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 457
 458         /* WaDisableSDEUnitClockGating:kbl */
 459         if (IS_KBL_GRAPHICS_STEP(i915, 0, STEP_C0))
 460                 intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6,
 461                                  0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 462
 463         /* WaDisableGamClockGating:kbl */
 464         if (IS_KBL_GRAPHICS_STEP(i915, 0, STEP_C0))
 465                 intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1,
 466                                  0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
 467
 468         /*
 469          * WaFbcTurnOffFbcWatermark:kbl
 470          * Display WA #0562: kbl
 471          */
 472         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 473
 474         /*
 475          * WaFbcNukeOnHostModify:kbl
 476          * Display WA #0873: kbl
 477          */
 478         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 479                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 480 }
 481
 482 static void skl_init_clock_gating(struct drm_i915_private *i915)
 483 {
 484         gen9_init_clock_gating(i915);
 485
 486         /* WaDisableDopClockGating:skl */
 487         intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 488                          GEN7_DOP_CLOCK_GATE_ENABLE, 0);
 489
 490         /* WAC6entrylatency:skl */
 491         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 492
 493         /*
 494          * WaFbcTurnOffFbcWatermark:skl
 495          * Display WA #0562: skl
 496          */
 497         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 498
 499         /*
 500          * WaFbcNukeOnHostModify:skl
 501          * Display WA #0873: skl
 502          */
 503         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 504                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 505
 506         /*
 507          * WaFbcHighMemBwCorruptionAvoidance:skl
 508          * Display WA #0883: skl
 509          */
 510         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 511 }
 512
 513 static void bdw_init_clock_gating(struct drm_i915_private *i915)
 514 {
 515         enum pipe pipe;
 516
 517         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 518         intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 519
 520         /* WaSwitchSolVfFArbitrationPriority:bdw */
 521         intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 522
 523         /* WaPsrDPAMaskVBlankInSRD:bdw */
 524         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
 525
 526         for_each_pipe(i915, pipe) {
 527                 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
 528                 intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
 529                                  0, BDW_UNMASK_VBL_TO_REGS_IN_SRD);
 530         }
 531
 532         /* WaVSRefCountFullforceMissDisable:bdw */
 533         /* WaDSRefCountFullforceMissDisable:bdw */
 534         intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
 535                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 536
 537         intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
 538                            _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 539
 540         /* WaDisableSDEUnitClockGating:bdw */
 541         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 542
 543         /* WaProgramL3SqcReg1Default:bdw */
 544         gen8_set_l3sqc_credits(i915, 30, 2);
 545
 546         /* WaKVMNotificationOnConfigChange:bdw */
 547         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR2_1,
 548                          0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
 549
 550         lpt_init_clock_gating(i915);
 551
 552         /* WaDisableDopClockGating:bdw
 553          *
 554          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
 555          * clock gating.
 556          */
 557         intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
 558 }
 559
 560 static void hsw_init_clock_gating(struct drm_i915_private *i915)
 561 {
 562         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 563         intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 564
 565         /* This is required by WaCatErrorRejectionIssue:hsw */
 566         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 567                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 568
 569         /* WaSwitchSolVfFArbitrationPriority:hsw */
 570         intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 571
 572         lpt_init_clock_gating(i915);
 573 }
 574
 575 static void ivb_init_clock_gating(struct drm_i915_private *i915)
 576 {
 577         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
 578
 579         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
 580         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 581
 582         /* WaDisableBackToBackFlipFix:ivb */
 583         intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
 584                            CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 585                            CHICKEN3_DGMG_DONE_FIX_DISABLE);
 586
 587         if (IS_IVB_GT1(i915))
 588                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 589                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 590         else {
 591                 /* must write both registers */
 592                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 593                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 594                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2_GT2,
 595                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 596         }
 597
 598         /*
 599          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 600          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
 601          */
 602         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 603                            GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 604
 605         /* This is required by WaCatErrorRejectionIssue:ivb */
 606         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 607                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 608
 609         g4x_disable_trickle_feed(i915);
 610
 611         intel_uncore_rmw(&i915->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
 612                          GEN6_MBC_SNPCR_MED);
 613
 614         if (!HAS_PCH_NOP(i915))
 615                 cpt_init_clock_gating(i915);
 616
 617         gen6_check_mch_setup(i915);
 618 }
 619
 620 static void vlv_init_clock_gating(struct drm_i915_private *i915)
 621 {
 622         /* WaDisableBackToBackFlipFix:vlv */
 623         intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
 624                            CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 625                            CHICKEN3_DGMG_DONE_FIX_DISABLE);
 626
 627         /* WaDisableDopClockGating:vlv */
 628         intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 629                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 630
 631         /* This is required by WaCatErrorRejectionIssue:vlv */
 632         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 633                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 634
 635         /*
 636          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 637          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
 638          */
 639         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 640                            GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 641
 642         /* WaDisableL3Bank2xClockGate:vlv
 643          * Disabling L3 clock gating- MMIO 940c[25] = 1
 644          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
 645         intel_uncore_rmw(&i915->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
 646
 647         /*
 648          * WaDisableVLVClockGating_VBIIssue:vlv
 649          * Disable clock gating on th GCFG unit to prevent a delay
 650          * in the reporting of vblank events.
 651          */
 652         intel_uncore_write(&i915->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
 653 }
 654
 655 static void chv_init_clock_gating(struct drm_i915_private *i915)
 656 {
 657         /* WaVSRefCountFullforceMissDisable:chv */
 658         /* WaDSRefCountFullforceMissDisable:chv */
 659         intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
 660                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 661
 662         /* WaDisableSemaphoreAndSyncFlipWait:chv */
 663         intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
 664                            _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 665
 666         /* WaDisableCSUnitClockGating:chv */
 667         intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 668
 669         /* WaDisableSDEUnitClockGating:chv */
 670         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 671
 672         /*
 673          * WaProgramL3SqcReg1Default:chv
 674          * See gfxspecs/Related Documents/Performance Guide/
 675          * LSQC Setting Recommendations.
 676          */
 677         gen8_set_l3sqc_credits(i915, 38, 2);
 678 }
 679
 680 static void g4x_init_clock_gating(struct drm_i915_private *i915)
 681 {
 682         u32 dspclk_gate;
 683
 684         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, 0);
 685         intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
 686                            GS_UNIT_CLOCK_GATE_DISABLE |
 687                            CL_UNIT_CLOCK_GATE_DISABLE);
 688         intel_uncore_write(&i915->uncore, RAMCLK_GATE_D, 0);
 689         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
 690                 OVRUNIT_CLOCK_GATE_DISABLE |
 691                 OVCUNIT_CLOCK_GATE_DISABLE;
 692         if (IS_GM45(i915))
 693                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
 694         intel_uncore_write(&i915->uncore, DSPCLK_GATE_D(i915), dspclk_gate);
 695
 696         g4x_disable_trickle_feed(i915);
 697 }
 698
 699 static void i965gm_init_clock_gating(struct drm_i915_private *i915)
 700 {
 701         struct intel_uncore *uncore = &i915->uncore;
 702
 703         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
 704         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
 705         intel_uncore_write(uncore, DSPCLK_GATE_D(i915), 0);
 706         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
 707         intel_uncore_write16(uncore, DEUC, 0);
 708         intel_uncore_write(uncore,
 709                            MI_ARB_STATE,
 710                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 711 }
 712
 713 static void i965g_init_clock_gating(struct drm_i915_private *i915)
 714 {
 715         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
 716                            I965_RCC_CLOCK_GATE_DISABLE |
 717                            I965_RCPB_CLOCK_GATE_DISABLE |
 718                            I965_ISC_CLOCK_GATE_DISABLE |
 719                            I965_FBC_CLOCK_GATE_DISABLE);
 720         intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, 0);
 721         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 722                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 723 }
 724
 725 static void gen3_init_clock_gating(struct drm_i915_private *i915)
 726 {
 727         u32 dstate = intel_uncore_read(&i915->uncore, D_STATE);
 728
 729         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
 730                 DSTATE_DOT_CLOCK_GATING;
 731         intel_uncore_write(&i915->uncore, D_STATE, dstate);
 732
 733         if (IS_PINEVIEW(i915))
 734                 intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
 735                                    _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
 736
 737         /* IIR "flip pending" means done if this bit is set */
 738         intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
 739                            _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
 740
 741         /* interrupts should cause a wake up from C3 */
 742         intel_uncore_write(&i915->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
 743
 744         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
 745         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 746                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
 747
 748         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 749                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 750 }
 751
 752 static void i85x_init_clock_gating(struct drm_i915_private *i915)
 753 {
 754         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
 755
 756         /* interrupts should cause a wake up from C3 */
 757         intel_uncore_write(&i915->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
 758                            _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
 759
 760         intel_uncore_write(&i915->uncore, MEM_MODE,
 761                            _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
 762
 763         /*
 764          * Have FBC ignore 3D activity since we use software
 765          * render tracking, and otherwise a pure 3D workload
 766          * (even if it just renders a single frame and then does
 767          * abosultely nothing) would not allow FBC to recompress
 768          * until a 2D blit occurs.
 769          */
 770         intel_uncore_write(&i915->uncore, SCPD0,
 771                            _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
 772 }
 773
 774 static void i830_init_clock_gating(struct drm_i915_private *i915)
 775 {
 776         intel_uncore_write(&i915->uncore, MEM_MODE,
 777                            _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
 778                            _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
 779 }
 780
 781 void intel_clock_gating_init(struct drm_i915_private *i915)
 782 {
 783         i915->clock_gating_funcs->init_clock_gating(i915);
 784 }
 785
 786 static void nop_init_clock_gating(struct drm_i915_private *i915)
 787 {
 788         drm_dbg_kms(&i915->drm,
 789                     "No clock gating settings or workarounds applied.\n");
 790 }
 791
 792 #define CG_FUNCS(platform)                                              \
 793 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
 794         .init_clock_gating = platform##_init_clock_gating,              \
 795 }
 796
 797 CG_FUNCS(pvc);
 798 CG_FUNCS(dg2);
 799 CG_FUNCS(xehpsdv);
 800 CG_FUNCS(adlp);
 801 CG_FUNCS(gen12lp);
 802 CG_FUNCS(icl);
 803 CG_FUNCS(cfl);
 804 CG_FUNCS(skl);
 805 CG_FUNCS(kbl);
 806 CG_FUNCS(bxt);
 807 CG_FUNCS(glk);
 808 CG_FUNCS(bdw);
 809 CG_FUNCS(chv);
 810 CG_FUNCS(hsw);
 811 CG_FUNCS(ivb);
 812 CG_FUNCS(vlv);
 813 CG_FUNCS(gen6);
 814 CG_FUNCS(ilk);
 815 CG_FUNCS(g4x);
 816 CG_FUNCS(i965gm);
 817 CG_FUNCS(i965g);
 818 CG_FUNCS(gen3);
 819 CG_FUNCS(i85x);
 820 CG_FUNCS(i830);
 821 CG_FUNCS(nop);
 822 #undef CG_FUNCS
 823
 824 /**
 825  * intel_clock_gating_hooks_init - setup the clock gating hooks
 826  * @i915: device private
 827  *
 828  * Setup the hooks that configure which clocks of a given platform can be
 829  * gated and also apply various GT and display specific workarounds for these
 830  * platforms. Note that some GT specific workarounds are applied separately
 831  * when GPU contexts or batchbuffers start their execution.
 832  */
 833 void intel_clock_gating_hooks_init(struct drm_i915_private *i915)
 834 {
 835         if (IS_METEORLAKE(i915))
 836                 i915->clock_gating_funcs = &nop_clock_gating_funcs;
 837         else if (IS_PONTEVECCHIO(i915))
 838                 i915->clock_gating_funcs = &pvc_clock_gating_funcs;
 839         else if (IS_DG2(i915))
 840                 i915->clock_gating_funcs = &dg2_clock_gating_funcs;
 841         else if (IS_XEHPSDV(i915))
 842                 i915->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
 843         else if (IS_ALDERLAKE_P(i915))
 844                 i915->clock_gating_funcs = &adlp_clock_gating_funcs;
 845         else if (GRAPHICS_VER(i915) == 12)
 846                 i915->clock_gating_funcs = &gen12lp_clock_gating_funcs;
 847         else if (GRAPHICS_VER(i915) == 11)
 848                 i915->clock_gating_funcs = &icl_clock_gating_funcs;
 849         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 850                 i915->clock_gating_funcs = &cfl_clock_gating_funcs;
 851         else if (IS_SKYLAKE(i915))
 852                 i915->clock_gating_funcs = &skl_clock_gating_funcs;
 853         else if (IS_KABYLAKE(i915))
 854                 i915->clock_gating_funcs = &kbl_clock_gating_funcs;
 855         else if (IS_BROXTON(i915))
 856                 i915->clock_gating_funcs = &bxt_clock_gating_funcs;
 857         else if (IS_GEMINILAKE(i915))
 858                 i915->clock_gating_funcs = &glk_clock_gating_funcs;
 859         else if (IS_BROADWELL(i915))
 860                 i915->clock_gating_funcs = &bdw_clock_gating_funcs;
 861         else if (IS_CHERRYVIEW(i915))
 862                 i915->clock_gating_funcs = &chv_clock_gating_funcs;
 863         else if (IS_HASWELL(i915))
 864                 i915->clock_gating_funcs = &hsw_clock_gating_funcs;
 865         else if (IS_IVYBRIDGE(i915))
 866                 i915->clock_gating_funcs = &ivb_clock_gating_funcs;
 867         else if (IS_VALLEYVIEW(i915))
 868                 i915->clock_gating_funcs = &vlv_clock_gating_funcs;
 869         else if (GRAPHICS_VER(i915) == 6)
 870                 i915->clock_gating_funcs = &gen6_clock_gating_funcs;
 871         else if (GRAPHICS_VER(i915) == 5)
 872                 i915->clock_gating_funcs = &ilk_clock_gating_funcs;
 873         else if (IS_G4X(i915))
 874                 i915->clock_gating_funcs = &g4x_clock_gating_funcs;
 875         else if (IS_I965GM(i915))
 876                 i915->clock_gating_funcs = &i965gm_clock_gating_funcs;
 877         else if (IS_I965G(i915))
 878                 i915->clock_gating_funcs = &i965g_clock_gating_funcs;
 879         else if (GRAPHICS_VER(i915) == 3)
 880                 i915->clock_gating_funcs = &gen3_clock_gating_funcs;
 881         else if (IS_I85X(i915) || IS_I865G(i915))
 882                 i915->clock_gating_funcs = &i85x_clock_gating_funcs;
 883         else if (GRAPHICS_VER(i915) == 2)
 884                 i915->clock_gating_funcs = &i830_clock_gating_funcs;
 885         else {
 886                 MISSING_CASE(INTEL_DEVID(i915));
 887                 i915->clock_gating_funcs = &nop_clock_gating_funcs;
 888         }
 889 }