drivers/gpu/drm/i915/intel_clock_gating.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include "display/intel_de.h"
  29 #include "display/intel_display.h"
  30 #include "display/intel_display_trace.h"
  31 #include "display/skl_watermark.h"
  32
  33 #include "gt/intel_engine_regs.h"
  34 #include "gt/intel_gt.h"
  35 #include "gt/intel_gt_mcr.h"
  36 #include "gt/intel_gt_regs.h"
  37
  38 #include "i915_drv.h"
  39 #include "intel_clock_gating.h"
  40 #include "intel_mchbar_regs.h"
  41 #include "vlv_sideband.h"
  42
  43 struct drm_i915_clock_gating_funcs {
  44         void (*init_clock_gating)(struct drm_i915_private *i915);
  45 };
  46
  47 static void gen9_init_clock_gating(struct drm_i915_private *i915)
  48 {
  49         if (HAS_LLC(i915)) {
  50                 /*
  51                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  52                  * Display WA #0390: skl,kbl
  53                  *
  54                  * Must match Sampler, Pixel Back End, and Media. See
  55                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  56                  */
  57                 intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
  58         }
  59
  60         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  61         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
  62
  63         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  64         intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
  65
  66         /*
  67          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
  68          * Display WA #0859: skl,bxt,kbl,glk,cfl
  69          */
  70         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
  71 }
  72
  73 static void bxt_init_clock_gating(struct drm_i915_private *i915)
  74 {
  75         gen9_init_clock_gating(i915);
  76
  77         /* WaDisableSDEUnitClockGating:bxt */
  78         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  79
  80         /*
  81          * FIXME:
  82          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
  83          */
  84         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
  85
  86         /*
  87          * Wa: Backlight PWM may stop in the asserted state, causing backlight
  88          * to stay fully on.
  89          */
  90         intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
  91                            intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
  92                            PWM1_GATING_DIS | PWM2_GATING_DIS);
  93
  94         /*
  95          * Lower the display internal timeout.
  96          * This is needed to avoid any hard hangs when DSI port PLL
  97          * is off and a MMIO access is attempted by any privilege
  98          * application, using batch buffers or any other means.
  99          */
 100         intel_uncore_write(&i915->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 101
 102         /*
 103          * WaFbcTurnOffFbcWatermark:bxt
 104          * Display WA #0562: bxt
 105          */
 106         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 107
 108         /*
 109          * WaFbcHighMemBwCorruptionAvoidance:bxt
 110          * Display WA #0883: bxt
 111          */
 112         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 113 }
 114
 115 static void glk_init_clock_gating(struct drm_i915_private *i915)
 116 {
 117         gen9_init_clock_gating(i915);
 118
 119         /*
 120          * WaDisablePWMClockGating:glk
 121          * Backlight PWM may stop in the asserted state, causing backlight
 122          * to stay fully on.
 123          */
 124         intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
 125                            intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
 126                            PWM1_GATING_DIS | PWM2_GATING_DIS);
 127 }
 128
 129 static void ibx_init_clock_gating(struct drm_i915_private *i915)
 130 {
 131         /*
 132          * On Ibex Peak and Cougar Point, we need to disable clock
 133          * gating for the panel power sequencer or it will fail to
 134          * start up when no ports are active.
 135          */
 136         intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
 137 }
 138
 139 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
 140 {
 141         enum pipe pipe;
 142
 143         for_each_pipe(dev_priv, pipe) {
 144                 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
 145
 146                 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
 147                 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
 148         }
 149 }
 150
 151 static void ilk_init_clock_gating(struct drm_i915_private *i915)
 152 {
 153         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 154
 155         /*
 156          * Required for FBC
 157          * WaFbcDisableDpfcClockGating:ilk
 158          */
 159         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
 160                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
 161                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
 162
 163         intel_uncore_write(&i915->uncore, PCH_3DCGDIS0,
 164                            MARIUNIT_CLOCK_GATE_DISABLE |
 165                            SVSMUNIT_CLOCK_GATE_DISABLE);
 166         intel_uncore_write(&i915->uncore, PCH_3DCGDIS1,
 167                            VFMUNIT_CLOCK_GATE_DISABLE);
 168
 169         /*
 170          * According to the spec the following bits should be set in
 171          * order to enable memory self-refresh
 172          * The bit 22/21 of 0x42004
 173          * The bit 5 of 0x42020
 174          * The bit 15 of 0x45000
 175          */
 176         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
 177                            (intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
 178                             ILK_DPARB_GATE | ILK_VSDPFD_FULL));
 179         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
 180         intel_uncore_write(&i915->uncore, DISP_ARB_CTL,
 181                            (intel_uncore_read(&i915->uncore, DISP_ARB_CTL) |
 182                             DISP_FBC_WM_DIS));
 183
 184         /*
 185          * Based on the document from hardware guys the following bits
 186          * should be set unconditionally in order to enable FBC.
 187          * The bit 22 of 0x42000
 188          * The bit 22 of 0x42004
 189          * The bit 7,8,9 of 0x42020.
 190          */
 191         if (IS_IRONLAKE_M(i915)) {
 192                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
 193                 intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 194                 intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
 195         }
 196
 197         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 198
 199         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 200
 201         g4x_disable_trickle_feed(i915);
 202
 203         ibx_init_clock_gating(i915);
 204 }
 205
 206 static void cpt_init_clock_gating(struct drm_i915_private *i915)
 207 {
 208         enum pipe pipe;
 209         u32 val;
 210
 211         /*
 212          * On Ibex Peak and Cougar Point, we need to disable clock
 213          * gating for the panel power sequencer or it will fail to
 214          * start up when no ports are active.
 215          */
 216         intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
 217                            PCH_DPLUNIT_CLOCK_GATE_DISABLE |
 218                            PCH_CPUNIT_CLOCK_GATE_DISABLE);
 219         intel_uncore_rmw(&i915->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
 220         /* The below fixes the weird display corruption, a few pixels shifted
 221          * downward, on (only) LVDS of some HP laptops with IVY.
 222          */
 223         for_each_pipe(i915, pipe) {
 224                 val = intel_uncore_read(&i915->uncore, TRANS_CHICKEN2(pipe));
 225                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 226                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 227                 if (i915->display.vbt.fdi_rx_polarity_inverted)
 228                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 229                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
 230                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
 231                 intel_uncore_write(&i915->uncore, TRANS_CHICKEN2(pipe), val);
 232         }
 233         /* WADP0ClockGatingDisable */
 234         for_each_pipe(i915, pipe) {
 235                 intel_uncore_write(&i915->uncore, TRANS_CHICKEN1(pipe),
 236                                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 237         }
 238 }
 239
 240 static void gen6_check_mch_setup(struct drm_i915_private *i915)
 241 {
 242         u32 tmp;
 243
 244         tmp = intel_uncore_read(&i915->uncore, MCH_SSKPD);
 245         if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
 246                 drm_dbg_kms(&i915->drm,
 247                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
 248                             tmp);
 249 }
 250
 251 static void gen6_init_clock_gating(struct drm_i915_private *i915)
 252 {
 253         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 254
 255         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 256
 257         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 258
 259         intel_uncore_write(&i915->uncore, GEN6_UCGCTL1,
 260                            intel_uncore_read(&i915->uncore, GEN6_UCGCTL1) |
 261                            GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
 262                            GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 263
 264         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
 265          * gating disable must be set.  Failure to set it results in
 266          * flickering pixels due to Z write ordering failures after
 267          * some amount of runtime in the Mesa "fire" demo, and Unigine
 268          * Sanctuary and Tropics, and apparently anything else with
 269          * alpha test or pixel discard.
 270          *
 271          * According to the spec, bit 11 (RCCUNIT) must also be set,
 272          * but we didn't debug actual testcases to find it out.
 273          *
 274          * WaDisableRCCUnitClockGating:snb
 275          * WaDisableRCPBUnitClockGating:snb
 276          */
 277         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 278                            GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
 279                            GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
 280
 281         /*
 282          * According to the spec the following bits should be
 283          * set in order to enable memory self-refresh and fbc:
 284          * The bit21 and bit22 of 0x42000
 285          * The bit21 and bit22 of 0x42004
 286          * The bit5 and bit7 of 0x42020
 287          * The bit14 of 0x70180
 288          * The bit14 of 0x71180
 289          *
 290          * WaFbcAsynchFlipDisableFbcQueue:snb
 291          */
 292         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN1,
 293                            intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN1) |
 294                            ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
 295         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
 296                            intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
 297                            ILK_DPARB_GATE | ILK_VSDPFD_FULL);
 298         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D,
 299                            intel_uncore_read(&i915->uncore, ILK_DSPCLK_GATE_D) |
 300                            ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
 301                            ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
 302
 303         g4x_disable_trickle_feed(i915);
 304
 305         cpt_init_clock_gating(i915);
 306
 307         gen6_check_mch_setup(i915);
 308 }
 309
 310 static void lpt_init_clock_gating(struct drm_i915_private *i915)
 311 {
 312         /*
 313          * TODO: this bit should only be enabled when really needed, then
 314          * disabled when not needed anymore in order to save power.
 315          */
 316         if (HAS_PCH_LPT_LP(i915))
 317                 intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D,
 318                                  0, PCH_LP_PARTITION_LEVEL_DISABLE);
 319
 320         /* WADPOClockGatingDisable:hsw */
 321         intel_uncore_rmw(&i915->uncore, TRANS_CHICKEN1(PIPE_A),
 322                          0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 323 }
 324
 325 static void gen8_set_l3sqc_credits(struct drm_i915_private *i915,
 326                                    int general_prio_credits,
 327                                    int high_prio_credits)
 328 {
 329         u32 misccpctl;
 330         u32 val;
 331
 332         /* WaTempDisableDOPClkGating:bdw */
 333         misccpctl = intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 334                                      GEN7_DOP_CLOCK_GATE_ENABLE, 0);
 335
 336         val = intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
 337         val &= ~L3_PRIO_CREDITS_MASK;
 338         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
 339         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
 340         intel_gt_mcr_multicast_write(to_gt(i915), GEN8_L3SQCREG1, val);
 341
 342         /*
 343          * Wait at least 100 clocks before re-enabling clock gating.
 344          * See the definition of L3SQCREG1 in BSpec.
 345          */
 346         intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
 347         udelay(1);
 348         intel_uncore_write(&i915->uncore, GEN7_MISCCPCTL, misccpctl);
 349 }
 350
 351 static void icl_init_clock_gating(struct drm_i915_private *i915)
 352 {
 353         /* Wa_1409120013:icl,ehl */
 354         intel_uncore_write(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 355                            DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 356
 357         /*Wa_14010594013:icl, ehl */
 358         intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1,
 359                          0, ICL_DELAY_PMRSP);
 360 }
 361
 362 static void gen12lp_init_clock_gating(struct drm_i915_private *i915)
 363 {
 364         /* Wa_1409120013 */
 365         if (DISPLAY_VER(i915) == 12)
 366                 intel_uncore_write(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 367                                    DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 368
 369         /* Wa_14013723622:tgl,rkl,dg1,adl-s */
 370         if (DISPLAY_VER(i915) == 12)
 371                 intel_uncore_rmw(&i915->uncore, CLKREQ_POLICY,
 372                                  CLKREQ_POLICY_MEM_UP_OVRD, 0);
 373 }
 374
 375 static void adlp_init_clock_gating(struct drm_i915_private *i915)
 376 {
 377         gen12lp_init_clock_gating(i915);
 378
 379         /* Wa_22011091694:adlp */
 380         intel_de_rmw(i915, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
 381
 382         /* Bspec/49189 Initialize Sequence */
 383         intel_de_rmw(i915, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
 384 }
 385
 386 static void xehpsdv_init_clock_gating(struct drm_i915_private *i915)
 387 {
 388         /* Wa_22010146351:xehpsdv */
 389         if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
 390                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
 391 }
 392
 393 static void dg2_init_clock_gating(struct drm_i915_private *i915)
 394 {
 395         /* Wa_22010954014:dg2 */
 396         intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
 397                          SGSI_SIDECLK_DIS);
 398
 399         /*
 400          * Wa_14010733611:dg2_g10
 401          * Wa_22010146351:dg2_g10
 402          */
 403         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
 404                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
 405                                  SGR_DIS | SGGI_DIS);
 406 }
 407
 408 static void pvc_init_clock_gating(struct drm_i915_private *i915)
 409 {
 410         /* Wa_14012385139:pvc */
 411         if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
 412                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
 413
 414         /* Wa_22010954014:pvc */
 415         if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
 416                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
 417 }
 418
 419 static void cnp_init_clock_gating(struct drm_i915_private *i915)
 420 {
 421         if (!HAS_PCH_CNP(i915))
 422                 return;
 423
 424         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
 425         intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
 426 }
 427
 428 static void cfl_init_clock_gating(struct drm_i915_private *i915)
 429 {
 430         cnp_init_clock_gating(i915);
 431         gen9_init_clock_gating(i915);
 432
 433         /* WAC6entrylatency:cfl */
 434         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 435
 436         /*
 437          * WaFbcTurnOffFbcWatermark:cfl
 438          * Display WA #0562: cfl
 439          */
 440         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 441
 442         /*
 443          * WaFbcNukeOnHostModify:cfl
 444          * Display WA #0873: cfl
 445          */
 446         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 447                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 448 }
 449
 450 static void kbl_init_clock_gating(struct drm_i915_private *i915)
 451 {
 452         gen9_init_clock_gating(i915);
 453
 454         /* WAC6entrylatency:kbl */
 455         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 456
 457         /* WaDisableSDEUnitClockGating:kbl */
 458         if (IS_KBL_GRAPHICS_STEP(i915, 0, STEP_C0))
 459                 intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6,
 460                                  0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 461
 462         /* WaDisableGamClockGating:kbl */
 463         if (IS_KBL_GRAPHICS_STEP(i915, 0, STEP_C0))
 464                 intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1,
 465                                  0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
 466
 467         /*
 468          * WaFbcTurnOffFbcWatermark:kbl
 469          * Display WA #0562: kbl
 470          */
 471         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 472
 473         /*
 474          * WaFbcNukeOnHostModify:kbl
 475          * Display WA #0873: kbl
 476          */
 477         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 478                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 479 }
 480
 481 static void skl_init_clock_gating(struct drm_i915_private *i915)
 482 {
 483         gen9_init_clock_gating(i915);
 484
 485         /* WaDisableDopClockGating:skl */
 486         intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 487                          GEN7_DOP_CLOCK_GATE_ENABLE, 0);
 488
 489         /* WAC6entrylatency:skl */
 490         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 491
 492         /*
 493          * WaFbcTurnOffFbcWatermark:skl
 494          * Display WA #0562: skl
 495          */
 496         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 497
 498         /*
 499          * WaFbcNukeOnHostModify:skl
 500          * Display WA #0873: skl
 501          */
 502         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 503                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 504
 505         /*
 506          * WaFbcHighMemBwCorruptionAvoidance:skl
 507          * Display WA #0883: skl
 508          */
 509         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 510 }
 511
 512 static void bdw_init_clock_gating(struct drm_i915_private *i915)
 513 {
 514         enum pipe pipe;
 515
 516         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 517         intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 518
 519         /* WaSwitchSolVfFArbitrationPriority:bdw */
 520         intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 521
 522         /* WaPsrDPAMaskVBlankInSRD:bdw */
 523         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, DPA_MASK_VBLANK_SRD);
 524
 525         for_each_pipe(i915, pipe) {
 526                 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
 527                 intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
 528                                  0, BDW_DPRS_MASK_VBLANK_SRD);
 529         }
 530
 531         /* WaVSRefCountFullforceMissDisable:bdw */
 532         /* WaDSRefCountFullforceMissDisable:bdw */
 533         intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
 534                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 535
 536         intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
 537                            _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 538
 539         /* WaDisableSDEUnitClockGating:bdw */
 540         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 541
 542         /* WaProgramL3SqcReg1Default:bdw */
 543         gen8_set_l3sqc_credits(i915, 30, 2);
 544
 545         /* WaKVMNotificationOnConfigChange:bdw */
 546         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR2_1,
 547                          0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
 548
 549         lpt_init_clock_gating(i915);
 550
 551         /* WaDisableDopClockGating:bdw
 552          *
 553          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
 554          * clock gating.
 555          */
 556         intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
 557 }
 558
 559 static void hsw_init_clock_gating(struct drm_i915_private *i915)
 560 {
 561         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 562         intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 563
 564         /* This is required by WaCatErrorRejectionIssue:hsw */
 565         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 566                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 567
 568         /* WaSwitchSolVfFArbitrationPriority:hsw */
 569         intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 570
 571         lpt_init_clock_gating(i915);
 572 }
 573
 574 static void ivb_init_clock_gating(struct drm_i915_private *i915)
 575 {
 576         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
 577
 578         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
 579         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 580
 581         /* WaDisableBackToBackFlipFix:ivb */
 582         intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
 583                            CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 584                            CHICKEN3_DGMG_DONE_FIX_DISABLE);
 585
 586         if (IS_IVB_GT1(i915))
 587                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 588                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 589         else {
 590                 /* must write both registers */
 591                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 592                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 593                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2_GT2,
 594                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 595         }
 596
 597         /*
 598          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 599          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
 600          */
 601         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 602                            GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 603
 604         /* This is required by WaCatErrorRejectionIssue:ivb */
 605         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 606                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 607
 608         g4x_disable_trickle_feed(i915);
 609
 610         intel_uncore_rmw(&i915->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
 611                          GEN6_MBC_SNPCR_MED);
 612
 613         if (!HAS_PCH_NOP(i915))
 614                 cpt_init_clock_gating(i915);
 615
 616         gen6_check_mch_setup(i915);
 617 }
 618
 619 static void vlv_init_clock_gating(struct drm_i915_private *i915)
 620 {
 621         /* WaDisableBackToBackFlipFix:vlv */
 622         intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
 623                            CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 624                            CHICKEN3_DGMG_DONE_FIX_DISABLE);
 625
 626         /* WaDisableDopClockGating:vlv */
 627         intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 628                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 629
 630         /* This is required by WaCatErrorRejectionIssue:vlv */
 631         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 632                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 633
 634         /*
 635          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 636          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
 637          */
 638         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 639                            GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 640
 641         /* WaDisableL3Bank2xClockGate:vlv
 642          * Disabling L3 clock gating- MMIO 940c[25] = 1
 643          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
 644         intel_uncore_rmw(&i915->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
 645
 646         /*
 647          * WaDisableVLVClockGating_VBIIssue:vlv
 648          * Disable clock gating on th GCFG unit to prevent a delay
 649          * in the reporting of vblank events.
 650          */
 651         intel_uncore_write(&i915->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
 652 }
 653
 654 static void chv_init_clock_gating(struct drm_i915_private *i915)
 655 {
 656         /* WaVSRefCountFullforceMissDisable:chv */
 657         /* WaDSRefCountFullforceMissDisable:chv */
 658         intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
 659                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 660
 661         /* WaDisableSemaphoreAndSyncFlipWait:chv */
 662         intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
 663                            _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 664
 665         /* WaDisableCSUnitClockGating:chv */
 666         intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 667
 668         /* WaDisableSDEUnitClockGating:chv */
 669         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 670
 671         /*
 672          * WaProgramL3SqcReg1Default:chv
 673          * See gfxspecs/Related Documents/Performance Guide/
 674          * LSQC Setting Recommendations.
 675          */
 676         gen8_set_l3sqc_credits(i915, 38, 2);
 677 }
 678
 679 static void g4x_init_clock_gating(struct drm_i915_private *i915)
 680 {
 681         u32 dspclk_gate;
 682
 683         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, 0);
 684         intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
 685                            GS_UNIT_CLOCK_GATE_DISABLE |
 686                            CL_UNIT_CLOCK_GATE_DISABLE);
 687         intel_uncore_write(&i915->uncore, RAMCLK_GATE_D, 0);
 688         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
 689                 OVRUNIT_CLOCK_GATE_DISABLE |
 690                 OVCUNIT_CLOCK_GATE_DISABLE;
 691         if (IS_GM45(i915))
 692                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
 693         intel_uncore_write(&i915->uncore, DSPCLK_GATE_D(i915), dspclk_gate);
 694
 695         g4x_disable_trickle_feed(i915);
 696 }
 697
 698 static void i965gm_init_clock_gating(struct drm_i915_private *i915)
 699 {
 700         struct intel_uncore *uncore = &i915->uncore;
 701
 702         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
 703         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
 704         intel_uncore_write(uncore, DSPCLK_GATE_D(i915), 0);
 705         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
 706         intel_uncore_write16(uncore, DEUC, 0);
 707         intel_uncore_write(uncore,
 708                            MI_ARB_STATE,
 709                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 710 }
 711
 712 static void i965g_init_clock_gating(struct drm_i915_private *i915)
 713 {
 714         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
 715                            I965_RCC_CLOCK_GATE_DISABLE |
 716                            I965_RCPB_CLOCK_GATE_DISABLE |
 717                            I965_ISC_CLOCK_GATE_DISABLE |
 718                            I965_FBC_CLOCK_GATE_DISABLE);
 719         intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, 0);
 720         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 721                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 722 }
 723
 724 static void gen3_init_clock_gating(struct drm_i915_private *i915)
 725 {
 726         u32 dstate = intel_uncore_read(&i915->uncore, D_STATE);
 727
 728         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
 729                 DSTATE_DOT_CLOCK_GATING;
 730         intel_uncore_write(&i915->uncore, D_STATE, dstate);
 731
 732         if (IS_PINEVIEW(i915))
 733                 intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
 734                                    _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
 735
 736         /* IIR "flip pending" means done if this bit is set */
 737         intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
 738                            _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
 739
 740         /* interrupts should cause a wake up from C3 */
 741         intel_uncore_write(&i915->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
 742
 743         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
 744         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 745                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
 746
 747         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 748                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 749 }
 750
 751 static void i85x_init_clock_gating(struct drm_i915_private *i915)
 752 {
 753         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
 754
 755         /* interrupts should cause a wake up from C3 */
 756         intel_uncore_write(&i915->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
 757                            _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
 758
 759         intel_uncore_write(&i915->uncore, MEM_MODE,
 760                            _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
 761
 762         /*
 763          * Have FBC ignore 3D activity since we use software
 764          * render tracking, and otherwise a pure 3D workload
 765          * (even if it just renders a single frame and then does
 766          * abosultely nothing) would not allow FBC to recompress
 767          * until a 2D blit occurs.
 768          */
 769         intel_uncore_write(&i915->uncore, SCPD0,
 770                            _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
 771 }
 772
 773 static void i830_init_clock_gating(struct drm_i915_private *i915)
 774 {
 775         intel_uncore_write(&i915->uncore, MEM_MODE,
 776                            _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
 777                            _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
 778 }
 779
 780 void intel_clock_gating_init(struct drm_i915_private *i915)
 781 {
 782         i915->clock_gating_funcs->init_clock_gating(i915);
 783 }
 784
 785 static void nop_init_clock_gating(struct drm_i915_private *i915)
 786 {
 787         drm_dbg_kms(&i915->drm,
 788                     "No clock gating settings or workarounds applied.\n");
 789 }
 790
 791 #define CG_FUNCS(platform)                                              \
 792 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
 793         .init_clock_gating = platform##_init_clock_gating,              \
 794 }
 795
 796 CG_FUNCS(pvc);
 797 CG_FUNCS(dg2);
 798 CG_FUNCS(xehpsdv);
 799 CG_FUNCS(adlp);
 800 CG_FUNCS(gen12lp);
 801 CG_FUNCS(icl);
 802 CG_FUNCS(cfl);
 803 CG_FUNCS(skl);
 804 CG_FUNCS(kbl);
 805 CG_FUNCS(bxt);
 806 CG_FUNCS(glk);
 807 CG_FUNCS(bdw);
 808 CG_FUNCS(chv);
 809 CG_FUNCS(hsw);
 810 CG_FUNCS(ivb);
 811 CG_FUNCS(vlv);
 812 CG_FUNCS(gen6);
 813 CG_FUNCS(ilk);
 814 CG_FUNCS(g4x);
 815 CG_FUNCS(i965gm);
 816 CG_FUNCS(i965g);
 817 CG_FUNCS(gen3);
 818 CG_FUNCS(i85x);
 819 CG_FUNCS(i830);
 820 CG_FUNCS(nop);
 821 #undef CG_FUNCS
 822
 823 /**
 824  * intel_clock_gating_hooks_init - setup the clock gating hooks
 825  * @i915: device private
 826  *
 827  * Setup the hooks that configure which clocks of a given platform can be
 828  * gated and also apply various GT and display specific workarounds for these
 829  * platforms. Note that some GT specific workarounds are applied separately
 830  * when GPU contexts or batchbuffers start their execution.
 831  */
 832 void intel_clock_gating_hooks_init(struct drm_i915_private *i915)
 833 {
 834         if (IS_METEORLAKE(i915))
 835                 i915->clock_gating_funcs = &nop_clock_gating_funcs;
 836         else if (IS_PONTEVECCHIO(i915))
 837                 i915->clock_gating_funcs = &pvc_clock_gating_funcs;
 838         else if (IS_DG2(i915))
 839                 i915->clock_gating_funcs = &dg2_clock_gating_funcs;
 840         else if (IS_XEHPSDV(i915))
 841                 i915->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
 842         else if (IS_ALDERLAKE_P(i915))
 843                 i915->clock_gating_funcs = &adlp_clock_gating_funcs;
 844         else if (GRAPHICS_VER(i915) == 12)
 845                 i915->clock_gating_funcs = &gen12lp_clock_gating_funcs;
 846         else if (GRAPHICS_VER(i915) == 11)
 847                 i915->clock_gating_funcs = &icl_clock_gating_funcs;
 848         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 849                 i915->clock_gating_funcs = &cfl_clock_gating_funcs;
 850         else if (IS_SKYLAKE(i915))
 851                 i915->clock_gating_funcs = &skl_clock_gating_funcs;
 852         else if (IS_KABYLAKE(i915))
 853                 i915->clock_gating_funcs = &kbl_clock_gating_funcs;
 854         else if (IS_BROXTON(i915))
 855                 i915->clock_gating_funcs = &bxt_clock_gating_funcs;
 856         else if (IS_GEMINILAKE(i915))
 857                 i915->clock_gating_funcs = &glk_clock_gating_funcs;
 858         else if (IS_BROADWELL(i915))
 859                 i915->clock_gating_funcs = &bdw_clock_gating_funcs;
 860         else if (IS_CHERRYVIEW(i915))
 861                 i915->clock_gating_funcs = &chv_clock_gating_funcs;
 862         else if (IS_HASWELL(i915))
 863                 i915->clock_gating_funcs = &hsw_clock_gating_funcs;
 864         else if (IS_IVYBRIDGE(i915))
 865                 i915->clock_gating_funcs = &ivb_clock_gating_funcs;
 866         else if (IS_VALLEYVIEW(i915))
 867                 i915->clock_gating_funcs = &vlv_clock_gating_funcs;
 868         else if (GRAPHICS_VER(i915) == 6)
 869                 i915->clock_gating_funcs = &gen6_clock_gating_funcs;
 870         else if (GRAPHICS_VER(i915) == 5)
 871                 i915->clock_gating_funcs = &ilk_clock_gating_funcs;
 872         else if (IS_G4X(i915))
 873                 i915->clock_gating_funcs = &g4x_clock_gating_funcs;
 874         else if (IS_I965GM(i915))
 875                 i915->clock_gating_funcs = &i965gm_clock_gating_funcs;
 876         else if (IS_I965G(i915))
 877                 i915->clock_gating_funcs = &i965g_clock_gating_funcs;
 878         else if (GRAPHICS_VER(i915) == 3)
 879                 i915->clock_gating_funcs = &gen3_clock_gating_funcs;
 880         else if (IS_I85X(i915) || IS_I865G(i915))
 881                 i915->clock_gating_funcs = &i85x_clock_gating_funcs;
 882         else if (GRAPHICS_VER(i915) == 2)
 883                 i915->clock_gating_funcs = &i830_clock_gating_funcs;
 884         else {
 885                 MISSING_CASE(INTEL_DEVID(i915));
 886                 i915->clock_gating_funcs = &nop_clock_gating_funcs;
 887         }
 888 }