2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
28 #include <linux/module.h>
29 #include <linux/pm_runtime.h>
31 #include <drm/drm_atomic_helper.h>
32 #include <drm/drm_fourcc.h>
33 #include <drm/drm_plane_helper.h>
35 #include "display/intel_atomic.h"
36 #include "display/intel_atomic_plane.h"
37 #include "display/intel_bw.h"
38 #include "display/intel_de.h"
39 #include "display/intel_display_types.h"
40 #include "display/intel_fbc.h"
41 #include "display/intel_sprite.h"
42 #include "display/skl_universal_plane.h"
44 #include "gt/intel_llc.h"
47 #include "i915_fixed.h"
49 #include "i915_trace.h"
50 #include "intel_pcode.h"
52 #include "vlv_sideband.h"
53 #include "../../../platform/x86/intel_ips.h"
55 /* Stores plane specific WM parameters */
56 struct skl_wm_params {
57 bool x_tiled, y_tiled;
64 u32 plane_bytes_per_line;
65 uint_fixed_16_16_t plane_blocks_per_line;
66 uint_fixed_16_16_t y_tile_minimum;
71 /* used in computing the new watermarks state */
72 struct intel_wm_config {
73 unsigned int num_pipes_active;
78 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
82 if (HAS_LLC(dev_priv)) {
84 * WaCompressedResourceDisplayNewHashMode:skl,kbl
85 * Display WA #0390: skl,kbl
87 * Must match Sampler, Pixel Back End, and Media. See
88 * WaCompressedResourceSamplerPbeMediaNewHashMode.
90 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
91 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
92 SKL_DE_COMPRESSED_HASH_MODE);
95 for_each_pipe(dev_priv, pipe) {
97 * "Plane N strech max must be programmed to 11b (x1)
98 * when Async flips are enabled on that plane."
100 if (!IS_GEMINILAKE(dev_priv) && intel_vtd_active())
101 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
102 SKL_PLANE1_STRETCH_MAX_MASK, SKL_PLANE1_STRETCH_MAX_X1);
105 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
106 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
107 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
109 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
110 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
111 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
114 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
115 * Display WA #0859: skl,bxt,kbl,glk,cfl
117 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
118 DISP_FBC_MEMORY_WAKE);
121 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
123 gen9_init_clock_gating(dev_priv);
125 /* WaDisableSDEUnitClockGating:bxt */
126 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
127 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
131 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
133 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
134 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
137 * Wa: Backlight PWM may stop in the asserted state, causing backlight
140 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
141 PWM1_GATING_DIS | PWM2_GATING_DIS);
144 * Lower the display internal timeout.
145 * This is needed to avoid any hard hangs when DSI port PLL
146 * is off and a MMIO access is attempted by any privilege
147 * application, using batch buffers or any other means.
149 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
152 * WaFbcTurnOffFbcWatermark:bxt
153 * Display WA #0562: bxt
155 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
159 * WaFbcHighMemBwCorruptionAvoidance:bxt
160 * Display WA #0883: bxt
162 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
163 ILK_DPFC_DISABLE_DUMMY0);
166 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
168 gen9_init_clock_gating(dev_priv);
171 * WaDisablePWMClockGating:glk
172 * Backlight PWM may stop in the asserted state, causing backlight
175 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
176 PWM1_GATING_DIS | PWM2_GATING_DIS);
179 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
183 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
185 switch (tmp & CLKCFG_FSB_MASK) {
187 dev_priv->fsb_freq = 533; /* 133*4 */
190 dev_priv->fsb_freq = 800; /* 200*4 */
193 dev_priv->fsb_freq = 667; /* 167*4 */
196 dev_priv->fsb_freq = 400; /* 100*4 */
200 switch (tmp & CLKCFG_MEM_MASK) {
202 dev_priv->mem_freq = 533;
205 dev_priv->mem_freq = 667;
208 dev_priv->mem_freq = 800;
212 /* detect pineview DDR3 setting */
213 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
214 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
217 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
221 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
222 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
224 switch (ddrpll & 0xff) {
226 dev_priv->mem_freq = 800;
229 dev_priv->mem_freq = 1066;
232 dev_priv->mem_freq = 1333;
235 dev_priv->mem_freq = 1600;
238 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
240 dev_priv->mem_freq = 0;
244 switch (csipll & 0x3ff) {
246 dev_priv->fsb_freq = 3200;
249 dev_priv->fsb_freq = 3733;
252 dev_priv->fsb_freq = 4266;
255 dev_priv->fsb_freq = 4800;
258 dev_priv->fsb_freq = 5333;
261 dev_priv->fsb_freq = 5866;
264 dev_priv->fsb_freq = 6400;
267 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
269 dev_priv->fsb_freq = 0;
274 static const struct cxsr_latency cxsr_latency_table[] = {
275 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
276 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
277 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
278 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
279 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
281 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
282 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
283 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
284 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
285 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
287 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
288 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
289 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
290 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
291 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
293 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
294 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
295 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
296 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
297 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
299 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
300 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
301 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
302 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
303 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
305 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
306 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
307 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
308 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
309 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
312 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
317 const struct cxsr_latency *latency;
320 if (fsb == 0 || mem == 0)
323 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
324 latency = &cxsr_latency_table[i];
325 if (is_desktop == latency->is_desktop &&
326 is_ddr3 == latency->is_ddr3 &&
327 fsb == latency->fsb_freq && mem == latency->mem_freq)
331 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
336 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
340 vlv_punit_get(dev_priv);
342 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
344 val &= ~FORCE_DDR_HIGH_FREQ;
346 val |= FORCE_DDR_HIGH_FREQ;
347 val &= ~FORCE_DDR_LOW_FREQ;
348 val |= FORCE_DDR_FREQ_REQ_ACK;
349 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
351 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
352 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
353 drm_err(&dev_priv->drm,
354 "timed out waiting for Punit DDR DVFS request\n");
356 vlv_punit_put(dev_priv);
359 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
363 vlv_punit_get(dev_priv);
365 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
367 val |= DSP_MAXFIFO_PM5_ENABLE;
369 val &= ~DSP_MAXFIFO_PM5_ENABLE;
370 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
372 vlv_punit_put(dev_priv);
375 #define FW_WM(value, plane) \
376 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
378 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
383 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
384 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
385 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
386 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
387 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
388 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
389 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
390 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
391 } else if (IS_PINEVIEW(dev_priv)) {
392 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
393 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
395 val |= PINEVIEW_SELF_REFRESH_EN;
397 val &= ~PINEVIEW_SELF_REFRESH_EN;
398 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
399 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
400 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
401 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
402 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
403 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
404 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
405 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
406 } else if (IS_I915GM(dev_priv)) {
408 * FIXME can't find a bit like this for 915G, and
409 * and yet it does have the related watermark in
410 * FW_BLC_SELF. What's going on?
412 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
413 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
414 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
415 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
416 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
421 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
423 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
424 enableddisabled(enable),
425 enableddisabled(was_enabled));
431 * intel_set_memory_cxsr - Configure CxSR state
432 * @dev_priv: i915 device
433 * @enable: Allow vs. disallow CxSR
435 * Allow or disallow the system to enter a special CxSR
436 * (C-state self refresh) state. What typically happens in CxSR mode
437 * is that several display FIFOs may get combined into a single larger
438 * FIFO for a particular plane (so called max FIFO mode) to allow the
439 * system to defer memory fetches longer, and the memory will enter
442 * Note that enabling CxSR does not guarantee that the system enter
443 * this special mode, nor does it guarantee that the system stays
444 * in that mode once entered. So this just allows/disallows the system
445 * to autonomously utilize the CxSR mode. Other factors such as core
446 * C-states will affect when/if the system actually enters/exits the
449 * Note that on VLV/CHV this actually only controls the max FIFO mode,
450 * and the system is free to enter/exit memory self refresh at any time
451 * even when the use of CxSR has been disallowed.
453 * While the system is actually in the CxSR/max FIFO mode, some plane
454 * control registers will not get latched on vblank. Thus in order to
455 * guarantee the system will respond to changes in the plane registers
456 * we must always disallow CxSR prior to making changes to those registers.
457 * Unfortunately the system will re-evaluate the CxSR conditions at
458 * frame start which happens after vblank start (which is when the plane
459 * registers would get latched), so we can't proceed with the plane update
460 * during the same frame where we disallowed CxSR.
462 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
463 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
464 * the hardware w.r.t. HPLL SR when writing to plane registers.
465 * Disallowing just CxSR is sufficient.
467 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
471 mutex_lock(&dev_priv->wm.wm_mutex);
472 ret = _intel_set_memory_cxsr(dev_priv, enable);
473 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
474 dev_priv->wm.vlv.cxsr = enable;
475 else if (IS_G4X(dev_priv))
476 dev_priv->wm.g4x.cxsr = enable;
477 mutex_unlock(&dev_priv->wm.wm_mutex);
483 * Latency for FIFO fetches is dependent on several factors:
484 * - memory configuration (speed, channels)
486 * - current MCH state
487 * It can be fairly high in some situations, so here we assume a fairly
488 * pessimal value. It's a tradeoff between extra memory fetches (if we
489 * set this value too high, the FIFO will fetch frequently to stay full)
490 * and power consumption (set it too low to save power and we might see
491 * FIFO underruns and display "flicker").
493 * A value of 5us seems to be a good balance; safe for very low end
494 * platforms but not overly aggressive on lower latency configs.
496 static const int pessimal_latency_ns = 5000;
498 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
499 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
501 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
503 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
504 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
505 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
506 enum pipe pipe = crtc->pipe;
507 int sprite0_start, sprite1_start;
508 u32 dsparb, dsparb2, dsparb3;
512 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
513 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
514 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
515 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
518 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
519 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
520 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
521 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
524 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
525 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
526 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
527 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
534 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
535 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
536 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
537 fifo_state->plane[PLANE_CURSOR] = 63;
540 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
541 enum i9xx_plane_id i9xx_plane)
543 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
546 size = dsparb & 0x7f;
547 if (i9xx_plane == PLANE_B)
548 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
550 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
551 dsparb, plane_name(i9xx_plane), size);
556 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
557 enum i9xx_plane_id i9xx_plane)
559 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
562 size = dsparb & 0x1ff;
563 if (i9xx_plane == PLANE_B)
564 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
565 size >>= 1; /* Convert to cachelines */
567 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
568 dsparb, plane_name(i9xx_plane), size);
573 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
574 enum i9xx_plane_id i9xx_plane)
576 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
579 size = dsparb & 0x7f;
580 size >>= 2; /* Convert to cachelines */
582 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
583 dsparb, plane_name(i9xx_plane), size);
588 /* Pineview has different values for various configs */
589 static const struct intel_watermark_params pnv_display_wm = {
590 .fifo_size = PINEVIEW_DISPLAY_FIFO,
591 .max_wm = PINEVIEW_MAX_WM,
592 .default_wm = PINEVIEW_DFT_WM,
593 .guard_size = PINEVIEW_GUARD_WM,
594 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
597 static const struct intel_watermark_params pnv_display_hplloff_wm = {
598 .fifo_size = PINEVIEW_DISPLAY_FIFO,
599 .max_wm = PINEVIEW_MAX_WM,
600 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
601 .guard_size = PINEVIEW_GUARD_WM,
602 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
605 static const struct intel_watermark_params pnv_cursor_wm = {
606 .fifo_size = PINEVIEW_CURSOR_FIFO,
607 .max_wm = PINEVIEW_CURSOR_MAX_WM,
608 .default_wm = PINEVIEW_CURSOR_DFT_WM,
609 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
610 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
613 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
614 .fifo_size = PINEVIEW_CURSOR_FIFO,
615 .max_wm = PINEVIEW_CURSOR_MAX_WM,
616 .default_wm = PINEVIEW_CURSOR_DFT_WM,
617 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
618 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
621 static const struct intel_watermark_params i965_cursor_wm_info = {
622 .fifo_size = I965_CURSOR_FIFO,
623 .max_wm = I965_CURSOR_MAX_WM,
624 .default_wm = I965_CURSOR_DFT_WM,
626 .cacheline_size = I915_FIFO_LINE_SIZE,
629 static const struct intel_watermark_params i945_wm_info = {
630 .fifo_size = I945_FIFO_SIZE,
631 .max_wm = I915_MAX_WM,
634 .cacheline_size = I915_FIFO_LINE_SIZE,
637 static const struct intel_watermark_params i915_wm_info = {
638 .fifo_size = I915_FIFO_SIZE,
639 .max_wm = I915_MAX_WM,
642 .cacheline_size = I915_FIFO_LINE_SIZE,
645 static const struct intel_watermark_params i830_a_wm_info = {
646 .fifo_size = I855GM_FIFO_SIZE,
647 .max_wm = I915_MAX_WM,
650 .cacheline_size = I830_FIFO_LINE_SIZE,
653 static const struct intel_watermark_params i830_bc_wm_info = {
654 .fifo_size = I855GM_FIFO_SIZE,
655 .max_wm = I915_MAX_WM/2,
658 .cacheline_size = I830_FIFO_LINE_SIZE,
661 static const struct intel_watermark_params i845_wm_info = {
662 .fifo_size = I830_FIFO_SIZE,
663 .max_wm = I915_MAX_WM,
666 .cacheline_size = I830_FIFO_LINE_SIZE,
670 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
671 * @pixel_rate: Pipe pixel rate in kHz
672 * @cpp: Plane bytes per pixel
673 * @latency: Memory wakeup latency in 0.1us units
675 * Compute the watermark using the method 1 or "small buffer"
676 * formula. The caller may additonally add extra cachelines
677 * to account for TLB misses and clock crossings.
679 * This method is concerned with the short term drain rate
680 * of the FIFO, ie. it does not account for blanking periods
681 * which would effectively reduce the average drain rate across
682 * a longer period. The name "small" refers to the fact the
683 * FIFO is relatively small compared to the amount of data
686 * The FIFO level vs. time graph might look something like:
690 * __---__---__ (- plane active, _ blanking)
693 * or perhaps like this:
696 * __----__----__ (- plane active, _ blanking)
700 * The watermark in bytes
702 static unsigned int intel_wm_method1(unsigned int pixel_rate,
704 unsigned int latency)
708 ret = mul_u32_u32(pixel_rate, cpp * latency);
709 ret = DIV_ROUND_UP_ULL(ret, 10000);
715 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
716 * @pixel_rate: Pipe pixel rate in kHz
717 * @htotal: Pipe horizontal total
718 * @width: Plane width in pixels
719 * @cpp: Plane bytes per pixel
720 * @latency: Memory wakeup latency in 0.1us units
722 * Compute the watermark using the method 2 or "large buffer"
723 * formula. The caller may additonally add extra cachelines
724 * to account for TLB misses and clock crossings.
726 * This method is concerned with the long term drain rate
727 * of the FIFO, ie. it does account for blanking periods
728 * which effectively reduce the average drain rate across
729 * a longer period. The name "large" refers to the fact the
730 * FIFO is relatively large compared to the amount of data
733 * The FIFO level vs. time graph might look something like:
738 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
742 * The watermark in bytes
744 static unsigned int intel_wm_method2(unsigned int pixel_rate,
748 unsigned int latency)
753 * FIXME remove once all users are computing
754 * watermarks in the correct place.
756 if (WARN_ON_ONCE(htotal == 0))
759 ret = (latency * pixel_rate) / (htotal * 10000);
760 ret = (ret + 1) * width * cpp;
766 * intel_calculate_wm - calculate watermark level
767 * @pixel_rate: pixel clock
768 * @wm: chip FIFO params
769 * @fifo_size: size of the FIFO buffer
770 * @cpp: bytes per pixel
771 * @latency_ns: memory latency for the platform
773 * Calculate the watermark level (the level at which the display plane will
774 * start fetching from memory again). Each chip has a different display
775 * FIFO size and allocation, so the caller needs to figure that out and pass
776 * in the correct intel_watermark_params structure.
778 * As the pixel clock runs, the FIFO will be drained at a rate that depends
779 * on the pixel size. When it reaches the watermark level, it'll start
780 * fetching FIFO line sized based chunks from memory until the FIFO fills
781 * past the watermark point. If the FIFO drains completely, a FIFO underrun
782 * will occur, and a display engine hang could result.
784 static unsigned int intel_calculate_wm(int pixel_rate,
785 const struct intel_watermark_params *wm,
786 int fifo_size, int cpp,
787 unsigned int latency_ns)
789 int entries, wm_size;
792 * Note: we need to make sure we don't overflow for various clock &
794 * clocks go from a few thousand to several hundred thousand.
795 * latency is usually a few thousand
797 entries = intel_wm_method1(pixel_rate, cpp,
799 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
801 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
803 wm_size = fifo_size - entries;
804 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
806 /* Don't promote wm_size to unsigned... */
807 if (wm_size > wm->max_wm)
808 wm_size = wm->max_wm;
810 wm_size = wm->default_wm;
813 * Bspec seems to indicate that the value shouldn't be lower than
814 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
815 * Lets go for 8 which is the burst size since certain platforms
816 * already use a hardcoded 8 (which is what the spec says should be
825 static bool is_disabling(int old, int new, int threshold)
827 return old >= threshold && new < threshold;
830 static bool is_enabling(int old, int new, int threshold)
832 return old < threshold && new >= threshold;
835 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
837 return dev_priv->wm.max_level + 1;
840 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
841 const struct intel_plane_state *plane_state)
843 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
845 /* FIXME check the 'enable' instead */
846 if (!crtc_state->hw.active)
850 * Treat cursor with fb as always visible since cursor updates
851 * can happen faster than the vrefresh rate, and the current
852 * watermark code doesn't handle that correctly. Cursor updates
853 * which set/clear the fb or change the cursor size are going
854 * to get throttled by intel_legacy_cursor_update() to work
855 * around this problem with the watermark code.
857 if (plane->id == PLANE_CURSOR)
858 return plane_state->hw.fb != NULL;
860 return plane_state->uapi.visible;
863 static bool intel_crtc_active(struct intel_crtc *crtc)
865 /* Be paranoid as we can arrive here with only partial
866 * state retrieved from the hardware during setup.
868 * We can ditch the adjusted_mode.crtc_clock check as soon
869 * as Haswell has gained clock readout/fastboot support.
871 * We can ditch the crtc->primary->state->fb check as soon as we can
872 * properly reconstruct framebuffers.
874 * FIXME: The intel_crtc->active here should be switched to
875 * crtc->state->active once we have proper CRTC states wired up
878 return crtc->active && crtc->base.primary->state->fb &&
879 crtc->config->hw.adjusted_mode.crtc_clock;
882 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
884 struct intel_crtc *crtc, *enabled = NULL;
886 for_each_intel_crtc(&dev_priv->drm, crtc) {
887 if (intel_crtc_active(crtc)) {
897 static void pnv_update_wm(struct drm_i915_private *dev_priv)
899 struct intel_crtc *crtc;
900 const struct cxsr_latency *latency;
904 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
909 drm_dbg_kms(&dev_priv->drm,
910 "Unknown FSB/MEM found, disable CxSR\n");
911 intel_set_memory_cxsr(dev_priv, false);
915 crtc = single_enabled_crtc(dev_priv);
917 const struct drm_display_mode *pipe_mode =
918 &crtc->config->hw.pipe_mode;
919 const struct drm_framebuffer *fb =
920 crtc->base.primary->state->fb;
921 int cpp = fb->format->cpp[0];
922 int clock = pipe_mode->crtc_clock;
925 wm = intel_calculate_wm(clock, &pnv_display_wm,
926 pnv_display_wm.fifo_size,
927 cpp, latency->display_sr);
928 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
929 reg &= ~DSPFW_SR_MASK;
930 reg |= FW_WM(wm, SR);
931 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
932 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
935 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
936 pnv_display_wm.fifo_size,
937 4, latency->cursor_sr);
938 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
939 reg &= ~DSPFW_CURSOR_SR_MASK;
940 reg |= FW_WM(wm, CURSOR_SR);
941 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
943 /* Display HPLL off SR */
944 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
945 pnv_display_hplloff_wm.fifo_size,
946 cpp, latency->display_hpll_disable);
947 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
948 reg &= ~DSPFW_HPLL_SR_MASK;
949 reg |= FW_WM(wm, HPLL_SR);
950 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
952 /* cursor HPLL off SR */
953 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
954 pnv_display_hplloff_wm.fifo_size,
955 4, latency->cursor_hpll_disable);
956 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
957 reg &= ~DSPFW_HPLL_CURSOR_MASK;
958 reg |= FW_WM(wm, HPLL_CURSOR);
959 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
960 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
962 intel_set_memory_cxsr(dev_priv, true);
964 intel_set_memory_cxsr(dev_priv, false);
969 * Documentation says:
970 * "If the line size is small, the TLB fetches can get in the way of the
971 * data fetches, causing some lag in the pixel data return which is not
972 * accounted for in the above formulas. The following adjustment only
973 * needs to be applied if eight whole lines fit in the buffer at once.
974 * The WM is adjusted upwards by the difference between the FIFO size
975 * and the size of 8 whole lines. This adjustment is always performed
976 * in the actual pixel depth regardless of whether FBC is enabled or not."
978 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
980 int tlb_miss = fifo_size * 64 - width * cpp * 8;
982 return max(0, tlb_miss);
985 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
986 const struct g4x_wm_values *wm)
990 for_each_pipe(dev_priv, pipe)
991 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
993 intel_uncore_write(&dev_priv->uncore, DSPFW1,
994 FW_WM(wm->sr.plane, SR) |
995 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
996 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
997 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
998 intel_uncore_write(&dev_priv->uncore, DSPFW2,
999 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
1000 FW_WM(wm->sr.fbc, FBC_SR) |
1001 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
1002 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
1003 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1004 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1005 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1006 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
1007 FW_WM(wm->sr.cursor, CURSOR_SR) |
1008 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1009 FW_WM(wm->hpll.plane, HPLL_SR));
1011 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1014 #define FW_WM_VLV(value, plane) \
1015 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1017 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1018 const struct vlv_wm_values *wm)
1022 for_each_pipe(dev_priv, pipe) {
1023 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1025 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1026 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1027 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1028 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1029 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1033 * Zero the (unused) WM1 watermarks, and also clear all the
1034 * high order bits so that there are no out of bounds values
1035 * present in the registers during the reprogramming.
1037 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1038 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1039 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1040 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1041 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1043 intel_uncore_write(&dev_priv->uncore, DSPFW1,
1044 FW_WM(wm->sr.plane, SR) |
1045 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1046 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1047 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1048 intel_uncore_write(&dev_priv->uncore, DSPFW2,
1049 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1050 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1051 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1052 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1053 FW_WM(wm->sr.cursor, CURSOR_SR));
1055 if (IS_CHERRYVIEW(dev_priv)) {
1056 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1057 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1058 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1059 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1060 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1061 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1062 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1063 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1064 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1065 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1066 FW_WM(wm->sr.plane >> 9, SR_HI) |
1067 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1068 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1069 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1070 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1071 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1072 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1073 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1074 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1075 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1077 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1078 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1079 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1080 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1081 FW_WM(wm->sr.plane >> 9, SR_HI) |
1082 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1083 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1084 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1085 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1086 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1087 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1090 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1095 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1097 /* all latencies in usec */
1098 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1099 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1100 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1102 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1105 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1108 * DSPCNTR[13] supposedly controls whether the
1109 * primary plane can use the FIFO space otherwise
1110 * reserved for the sprite plane. It's not 100% clear
1111 * what the actual FIFO size is, but it looks like we
1112 * can happily set both primary and sprite watermarks
1113 * up to 127 cachelines. So that would seem to mean
1114 * that either DSPCNTR[13] doesn't do anything, or that
1115 * the total FIFO is >= 256 cachelines in size. Either
1116 * way, we don't seem to have to worry about this
1117 * repartitioning as the maximum watermark value the
1118 * register can hold for each plane is lower than the
1119 * minimum FIFO size.
1125 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1127 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1129 MISSING_CASE(plane_id);
1134 static int g4x_fbc_fifo_size(int level)
1137 case G4X_WM_LEVEL_SR:
1139 case G4X_WM_LEVEL_HPLL:
1142 MISSING_CASE(level);
1147 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1148 const struct intel_plane_state *plane_state,
1151 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1152 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1153 const struct drm_display_mode *pipe_mode =
1154 &crtc_state->hw.pipe_mode;
1155 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1156 unsigned int clock, htotal, cpp, width, wm;
1161 if (!intel_wm_plane_visible(crtc_state, plane_state))
1164 cpp = plane_state->hw.fb->format->cpp[0];
1167 * WaUse32BppForSRWM:ctg,elk
1169 * The spec fails to list this restriction for the
1170 * HPLL watermark, which seems a little strange.
1171 * Let's use 32bpp for the HPLL watermark as well.
1173 if (plane->id == PLANE_PRIMARY &&
1174 level != G4X_WM_LEVEL_NORMAL)
1177 clock = pipe_mode->crtc_clock;
1178 htotal = pipe_mode->crtc_htotal;
1180 width = drm_rect_width(&plane_state->uapi.dst);
1182 if (plane->id == PLANE_CURSOR) {
1183 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1184 } else if (plane->id == PLANE_PRIMARY &&
1185 level == G4X_WM_LEVEL_NORMAL) {
1186 wm = intel_wm_method1(clock, cpp, latency);
1188 unsigned int small, large;
1190 small = intel_wm_method1(clock, cpp, latency);
1191 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1193 wm = min(small, large);
1196 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1199 wm = DIV_ROUND_UP(wm, 64) + 2;
1201 return min_t(unsigned int, wm, USHRT_MAX);
1204 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1205 int level, enum plane_id plane_id, u16 value)
1207 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1210 for (; level < intel_wm_num_levels(dev_priv); level++) {
1211 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1213 dirty |= raw->plane[plane_id] != value;
1214 raw->plane[plane_id] = value;
1220 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1221 int level, u16 value)
1223 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1226 /* NORMAL level doesn't have an FBC watermark */
1227 level = max(level, G4X_WM_LEVEL_SR);
1229 for (; level < intel_wm_num_levels(dev_priv); level++) {
1230 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1232 dirty |= raw->fbc != value;
1239 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1240 const struct intel_plane_state *plane_state,
1243 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1244 const struct intel_plane_state *plane_state)
1246 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1247 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1248 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1249 enum plane_id plane_id = plane->id;
1253 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1254 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1255 if (plane_id == PLANE_PRIMARY)
1256 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1260 for (level = 0; level < num_levels; level++) {
1261 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1264 wm = g4x_compute_wm(crtc_state, plane_state, level);
1265 max_wm = g4x_plane_fifo_size(plane_id, level);
1270 dirty |= raw->plane[plane_id] != wm;
1271 raw->plane[plane_id] = wm;
1273 if (plane_id != PLANE_PRIMARY ||
1274 level == G4X_WM_LEVEL_NORMAL)
1277 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1278 raw->plane[plane_id]);
1279 max_wm = g4x_fbc_fifo_size(level);
1282 * FBC wm is not mandatory as we
1283 * can always just disable its use.
1288 dirty |= raw->fbc != wm;
1292 /* mark watermarks as invalid */
1293 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1295 if (plane_id == PLANE_PRIMARY)
1296 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1300 drm_dbg_kms(&dev_priv->drm,
1301 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1303 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1304 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1305 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1307 if (plane_id == PLANE_PRIMARY)
1308 drm_dbg_kms(&dev_priv->drm,
1309 "FBC watermarks: SR=%d, HPLL=%d\n",
1310 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1311 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1317 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1318 enum plane_id plane_id, int level)
1320 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1322 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1325 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1328 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1330 if (level > dev_priv->wm.max_level)
1333 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1334 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1335 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1338 /* mark all levels starting from 'level' as invalid */
1339 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1340 struct g4x_wm_state *wm_state, int level)
1342 if (level <= G4X_WM_LEVEL_NORMAL) {
1343 enum plane_id plane_id;
1345 for_each_plane_id_on_crtc(crtc, plane_id)
1346 wm_state->wm.plane[plane_id] = USHRT_MAX;
1349 if (level <= G4X_WM_LEVEL_SR) {
1350 wm_state->cxsr = false;
1351 wm_state->sr.cursor = USHRT_MAX;
1352 wm_state->sr.plane = USHRT_MAX;
1353 wm_state->sr.fbc = USHRT_MAX;
1356 if (level <= G4X_WM_LEVEL_HPLL) {
1357 wm_state->hpll_en = false;
1358 wm_state->hpll.cursor = USHRT_MAX;
1359 wm_state->hpll.plane = USHRT_MAX;
1360 wm_state->hpll.fbc = USHRT_MAX;
1364 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1367 if (level < G4X_WM_LEVEL_SR)
1370 if (level >= G4X_WM_LEVEL_SR &&
1371 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1374 if (level >= G4X_WM_LEVEL_HPLL &&
1375 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1381 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1382 struct intel_crtc *crtc)
1384 struct intel_crtc_state *crtc_state =
1385 intel_atomic_get_new_crtc_state(state, crtc);
1386 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1387 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1388 const struct g4x_pipe_wm *raw;
1389 const struct intel_plane_state *old_plane_state;
1390 const struct intel_plane_state *new_plane_state;
1391 struct intel_plane *plane;
1392 enum plane_id plane_id;
1394 unsigned int dirty = 0;
1396 for_each_oldnew_intel_plane_in_state(state, plane,
1398 new_plane_state, i) {
1399 if (new_plane_state->hw.crtc != &crtc->base &&
1400 old_plane_state->hw.crtc != &crtc->base)
1403 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1404 dirty |= BIT(plane->id);
1410 level = G4X_WM_LEVEL_NORMAL;
1411 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1414 raw = &crtc_state->wm.g4x.raw[level];
1415 for_each_plane_id_on_crtc(crtc, plane_id)
1416 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1418 level = G4X_WM_LEVEL_SR;
1419 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1422 raw = &crtc_state->wm.g4x.raw[level];
1423 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1424 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1425 wm_state->sr.fbc = raw->fbc;
1427 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1429 level = G4X_WM_LEVEL_HPLL;
1430 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1433 raw = &crtc_state->wm.g4x.raw[level];
1434 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1435 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1436 wm_state->hpll.fbc = raw->fbc;
1438 wm_state->hpll_en = wm_state->cxsr;
1443 if (level == G4X_WM_LEVEL_NORMAL)
1446 /* invalidate the higher levels */
1447 g4x_invalidate_wms(crtc, wm_state, level);
1450 * Determine if the FBC watermark(s) can be used. IF
1451 * this isn't the case we prefer to disable the FBC
1452 * watermark(s) rather than disable the SR/HPLL
1453 * level(s) entirely. 'level-1' is the highest valid
1456 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1461 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1462 struct intel_crtc *crtc)
1464 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1465 struct intel_crtc_state *new_crtc_state =
1466 intel_atomic_get_new_crtc_state(state, crtc);
1467 const struct intel_crtc_state *old_crtc_state =
1468 intel_atomic_get_old_crtc_state(state, crtc);
1469 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1470 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1471 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1472 enum plane_id plane_id;
1474 if (!new_crtc_state->hw.active ||
1475 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1476 *intermediate = *optimal;
1478 intermediate->cxsr = false;
1479 intermediate->hpll_en = false;
1483 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1484 !new_crtc_state->disable_cxsr;
1485 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1486 !new_crtc_state->disable_cxsr;
1487 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1489 for_each_plane_id_on_crtc(crtc, plane_id) {
1490 intermediate->wm.plane[plane_id] =
1491 max(optimal->wm.plane[plane_id],
1492 active->wm.plane[plane_id]);
1494 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1495 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1498 intermediate->sr.plane = max(optimal->sr.plane,
1500 intermediate->sr.cursor = max(optimal->sr.cursor,
1502 intermediate->sr.fbc = max(optimal->sr.fbc,
1505 intermediate->hpll.plane = max(optimal->hpll.plane,
1506 active->hpll.plane);
1507 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1508 active->hpll.cursor);
1509 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1512 drm_WARN_ON(&dev_priv->drm,
1513 (intermediate->sr.plane >
1514 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1515 intermediate->sr.cursor >
1516 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1517 intermediate->cxsr);
1518 drm_WARN_ON(&dev_priv->drm,
1519 (intermediate->sr.plane >
1520 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1521 intermediate->sr.cursor >
1522 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1523 intermediate->hpll_en);
1525 drm_WARN_ON(&dev_priv->drm,
1526 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1527 intermediate->fbc_en && intermediate->cxsr);
1528 drm_WARN_ON(&dev_priv->drm,
1529 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1530 intermediate->fbc_en && intermediate->hpll_en);
1534 * If our intermediate WM are identical to the final WM, then we can
1535 * omit the post-vblank programming; only update if it's different.
1537 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1538 new_crtc_state->wm.need_postvbl_update = true;
1543 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1544 struct g4x_wm_values *wm)
1546 struct intel_crtc *crtc;
1547 int num_active_pipes = 0;
1553 for_each_intel_crtc(&dev_priv->drm, crtc) {
1554 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1559 if (!wm_state->cxsr)
1561 if (!wm_state->hpll_en)
1562 wm->hpll_en = false;
1563 if (!wm_state->fbc_en)
1569 if (num_active_pipes != 1) {
1571 wm->hpll_en = false;
1575 for_each_intel_crtc(&dev_priv->drm, crtc) {
1576 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1577 enum pipe pipe = crtc->pipe;
1579 wm->pipe[pipe] = wm_state->wm;
1580 if (crtc->active && wm->cxsr)
1581 wm->sr = wm_state->sr;
1582 if (crtc->active && wm->hpll_en)
1583 wm->hpll = wm_state->hpll;
1587 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1589 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1590 struct g4x_wm_values new_wm = {};
1592 g4x_merge_wm(dev_priv, &new_wm);
1594 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1597 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1598 _intel_set_memory_cxsr(dev_priv, false);
1600 g4x_write_wm_values(dev_priv, &new_wm);
1602 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1603 _intel_set_memory_cxsr(dev_priv, true);
1608 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1609 struct intel_crtc *crtc)
1611 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1612 const struct intel_crtc_state *crtc_state =
1613 intel_atomic_get_new_crtc_state(state, crtc);
1615 mutex_lock(&dev_priv->wm.wm_mutex);
1616 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1617 g4x_program_watermarks(dev_priv);
1618 mutex_unlock(&dev_priv->wm.wm_mutex);
1621 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1622 struct intel_crtc *crtc)
1624 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1625 const struct intel_crtc_state *crtc_state =
1626 intel_atomic_get_new_crtc_state(state, crtc);
1628 if (!crtc_state->wm.need_postvbl_update)
1631 mutex_lock(&dev_priv->wm.wm_mutex);
1632 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1633 g4x_program_watermarks(dev_priv);
1634 mutex_unlock(&dev_priv->wm.wm_mutex);
1637 /* latency must be in 0.1us units. */
1638 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1639 unsigned int htotal,
1642 unsigned int latency)
1646 ret = intel_wm_method2(pixel_rate, htotal,
1647 width, cpp, latency);
1648 ret = DIV_ROUND_UP(ret, 64);
1653 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1655 /* all latencies in usec */
1656 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1658 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1660 if (IS_CHERRYVIEW(dev_priv)) {
1661 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1662 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1664 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1668 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1669 const struct intel_plane_state *plane_state,
1672 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1673 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1674 const struct drm_display_mode *pipe_mode =
1675 &crtc_state->hw.pipe_mode;
1676 unsigned int clock, htotal, cpp, width, wm;
1678 if (dev_priv->wm.pri_latency[level] == 0)
1681 if (!intel_wm_plane_visible(crtc_state, plane_state))
1684 cpp = plane_state->hw.fb->format->cpp[0];
1685 clock = pipe_mode->crtc_clock;
1686 htotal = pipe_mode->crtc_htotal;
1687 width = crtc_state->pipe_src_w;
1689 if (plane->id == PLANE_CURSOR) {
1691 * FIXME the formula gives values that are
1692 * too big for the cursor FIFO, and hence we
1693 * would never be able to use cursors. For
1694 * now just hardcode the watermark.
1698 wm = vlv_wm_method2(clock, htotal, width, cpp,
1699 dev_priv->wm.pri_latency[level] * 10);
1702 return min_t(unsigned int, wm, USHRT_MAX);
1705 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1707 return (active_planes & (BIT(PLANE_SPRITE0) |
1708 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1711 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1713 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1714 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1715 const struct g4x_pipe_wm *raw =
1716 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1717 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1718 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1719 int num_active_planes = hweight8(active_planes);
1720 const int fifo_size = 511;
1721 int fifo_extra, fifo_left = fifo_size;
1722 int sprite0_fifo_extra = 0;
1723 unsigned int total_rate;
1724 enum plane_id plane_id;
1727 * When enabling sprite0 after sprite1 has already been enabled
1728 * we tend to get an underrun unless sprite0 already has some
1729 * FIFO space allcoated. Hence we always allocate at least one
1730 * cacheline for sprite0 whenever sprite1 is enabled.
1732 * All other plane enable sequences appear immune to this problem.
1734 if (vlv_need_sprite0_fifo_workaround(active_planes))
1735 sprite0_fifo_extra = 1;
1737 total_rate = raw->plane[PLANE_PRIMARY] +
1738 raw->plane[PLANE_SPRITE0] +
1739 raw->plane[PLANE_SPRITE1] +
1742 if (total_rate > fifo_size)
1745 if (total_rate == 0)
1748 for_each_plane_id_on_crtc(crtc, plane_id) {
1751 if ((active_planes & BIT(plane_id)) == 0) {
1752 fifo_state->plane[plane_id] = 0;
1756 rate = raw->plane[plane_id];
1757 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1758 fifo_left -= fifo_state->plane[plane_id];
1761 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1762 fifo_left -= sprite0_fifo_extra;
1764 fifo_state->plane[PLANE_CURSOR] = 63;
1766 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1768 /* spread the remainder evenly */
1769 for_each_plane_id_on_crtc(crtc, plane_id) {
1775 if ((active_planes & BIT(plane_id)) == 0)
1778 plane_extra = min(fifo_extra, fifo_left);
1779 fifo_state->plane[plane_id] += plane_extra;
1780 fifo_left -= plane_extra;
1783 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1785 /* give it all to the first plane if none are active */
1786 if (active_planes == 0) {
1787 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1788 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1794 /* mark all levels starting from 'level' as invalid */
1795 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1796 struct vlv_wm_state *wm_state, int level)
1798 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1800 for (; level < intel_wm_num_levels(dev_priv); level++) {
1801 enum plane_id plane_id;
1803 for_each_plane_id_on_crtc(crtc, plane_id)
1804 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1806 wm_state->sr[level].cursor = USHRT_MAX;
1807 wm_state->sr[level].plane = USHRT_MAX;
1811 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1816 return fifo_size - wm;
1820 * Starting from 'level' set all higher
1821 * levels to 'value' in the "raw" watermarks.
1823 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1824 int level, enum plane_id plane_id, u16 value)
1826 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1827 int num_levels = intel_wm_num_levels(dev_priv);
1830 for (; level < num_levels; level++) {
1831 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1833 dirty |= raw->plane[plane_id] != value;
1834 raw->plane[plane_id] = value;
1840 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1841 const struct intel_plane_state *plane_state)
1843 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1844 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1845 enum plane_id plane_id = plane->id;
1846 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1850 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1851 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1855 for (level = 0; level < num_levels; level++) {
1856 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1857 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1858 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1863 dirty |= raw->plane[plane_id] != wm;
1864 raw->plane[plane_id] = wm;
1867 /* mark all higher levels as invalid */
1868 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1872 drm_dbg_kms(&dev_priv->drm,
1873 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1875 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1876 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1877 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1882 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1883 enum plane_id plane_id, int level)
1885 const struct g4x_pipe_wm *raw =
1886 &crtc_state->wm.vlv.raw[level];
1887 const struct vlv_fifo_state *fifo_state =
1888 &crtc_state->wm.vlv.fifo_state;
1890 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1893 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1895 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1896 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1897 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1898 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1901 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1902 struct intel_crtc *crtc)
1904 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1905 struct intel_crtc_state *crtc_state =
1906 intel_atomic_get_new_crtc_state(state, crtc);
1907 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1908 const struct vlv_fifo_state *fifo_state =
1909 &crtc_state->wm.vlv.fifo_state;
1910 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1911 int num_active_planes = hweight8(active_planes);
1912 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1913 const struct intel_plane_state *old_plane_state;
1914 const struct intel_plane_state *new_plane_state;
1915 struct intel_plane *plane;
1916 enum plane_id plane_id;
1918 unsigned int dirty = 0;
1920 for_each_oldnew_intel_plane_in_state(state, plane,
1922 new_plane_state, i) {
1923 if (new_plane_state->hw.crtc != &crtc->base &&
1924 old_plane_state->hw.crtc != &crtc->base)
1927 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1928 dirty |= BIT(plane->id);
1932 * DSPARB registers may have been reset due to the
1933 * power well being turned off. Make sure we restore
1934 * them to a consistent state even if no primary/sprite
1935 * planes are initially active.
1938 crtc_state->fifo_changed = true;
1943 /* cursor changes don't warrant a FIFO recompute */
1944 if (dirty & ~BIT(PLANE_CURSOR)) {
1945 const struct intel_crtc_state *old_crtc_state =
1946 intel_atomic_get_old_crtc_state(state, crtc);
1947 const struct vlv_fifo_state *old_fifo_state =
1948 &old_crtc_state->wm.vlv.fifo_state;
1950 ret = vlv_compute_fifo(crtc_state);
1954 if (needs_modeset ||
1955 memcmp(old_fifo_state, fifo_state,
1956 sizeof(*fifo_state)) != 0)
1957 crtc_state->fifo_changed = true;
1960 /* initially allow all levels */
1961 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1963 * Note that enabling cxsr with no primary/sprite planes
1964 * enabled can wedge the pipe. Hence we only allow cxsr
1965 * with exactly one enabled primary/sprite plane.
1967 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1969 for (level = 0; level < wm_state->num_levels; level++) {
1970 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1971 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1973 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1976 for_each_plane_id_on_crtc(crtc, plane_id) {
1977 wm_state->wm[level].plane[plane_id] =
1978 vlv_invert_wm_value(raw->plane[plane_id],
1979 fifo_state->plane[plane_id]);
1982 wm_state->sr[level].plane =
1983 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1984 raw->plane[PLANE_SPRITE0],
1985 raw->plane[PLANE_SPRITE1]),
1988 wm_state->sr[level].cursor =
1989 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1996 /* limit to only levels we can actually handle */
1997 wm_state->num_levels = level;
1999 /* invalidate the higher levels */
2000 vlv_invalidate_wms(crtc, wm_state, level);
2005 #define VLV_FIFO(plane, value) \
2006 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2008 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2009 struct intel_crtc *crtc)
2011 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2012 struct intel_uncore *uncore = &dev_priv->uncore;
2013 const struct intel_crtc_state *crtc_state =
2014 intel_atomic_get_new_crtc_state(state, crtc);
2015 const struct vlv_fifo_state *fifo_state =
2016 &crtc_state->wm.vlv.fifo_state;
2017 int sprite0_start, sprite1_start, fifo_size;
2018 u32 dsparb, dsparb2, dsparb3;
2020 if (!crtc_state->fifo_changed)
2023 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2024 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2025 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2027 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2028 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2030 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2033 * uncore.lock serves a double purpose here. It allows us to
2034 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2035 * it protects the DSPARB registers from getting clobbered by
2036 * parallel updates from multiple pipes.
2038 * intel_pipe_update_start() has already disabled interrupts
2039 * for us, so a plain spin_lock() is sufficient here.
2041 spin_lock(&uncore->lock);
2043 switch (crtc->pipe) {
2045 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2046 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2048 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2049 VLV_FIFO(SPRITEB, 0xff));
2050 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2051 VLV_FIFO(SPRITEB, sprite1_start));
2053 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2054 VLV_FIFO(SPRITEB_HI, 0x1));
2055 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2056 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2058 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2059 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2062 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2063 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2065 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2066 VLV_FIFO(SPRITED, 0xff));
2067 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2068 VLV_FIFO(SPRITED, sprite1_start));
2070 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2071 VLV_FIFO(SPRITED_HI, 0xff));
2072 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2073 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2075 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2076 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2079 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2080 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2082 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2083 VLV_FIFO(SPRITEF, 0xff));
2084 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2085 VLV_FIFO(SPRITEF, sprite1_start));
2087 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2088 VLV_FIFO(SPRITEF_HI, 0xff));
2089 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2090 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2092 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2093 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2099 intel_uncore_posting_read_fw(uncore, DSPARB);
2101 spin_unlock(&uncore->lock);
2106 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2107 struct intel_crtc *crtc)
2109 struct intel_crtc_state *new_crtc_state =
2110 intel_atomic_get_new_crtc_state(state, crtc);
2111 const struct intel_crtc_state *old_crtc_state =
2112 intel_atomic_get_old_crtc_state(state, crtc);
2113 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2114 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2115 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2118 if (!new_crtc_state->hw.active ||
2119 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2120 *intermediate = *optimal;
2122 intermediate->cxsr = false;
2126 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2127 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2128 !new_crtc_state->disable_cxsr;
2130 for (level = 0; level < intermediate->num_levels; level++) {
2131 enum plane_id plane_id;
2133 for_each_plane_id_on_crtc(crtc, plane_id) {
2134 intermediate->wm[level].plane[plane_id] =
2135 min(optimal->wm[level].plane[plane_id],
2136 active->wm[level].plane[plane_id]);
2139 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2140 active->sr[level].plane);
2141 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2142 active->sr[level].cursor);
2145 vlv_invalidate_wms(crtc, intermediate, level);
2149 * If our intermediate WM are identical to the final WM, then we can
2150 * omit the post-vblank programming; only update if it's different.
2152 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2153 new_crtc_state->wm.need_postvbl_update = true;
2158 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2159 struct vlv_wm_values *wm)
2161 struct intel_crtc *crtc;
2162 int num_active_pipes = 0;
2164 wm->level = dev_priv->wm.max_level;
2167 for_each_intel_crtc(&dev_priv->drm, crtc) {
2168 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2173 if (!wm_state->cxsr)
2177 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2180 if (num_active_pipes != 1)
2183 if (num_active_pipes > 1)
2184 wm->level = VLV_WM_LEVEL_PM2;
2186 for_each_intel_crtc(&dev_priv->drm, crtc) {
2187 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2188 enum pipe pipe = crtc->pipe;
2190 wm->pipe[pipe] = wm_state->wm[wm->level];
2191 if (crtc->active && wm->cxsr)
2192 wm->sr = wm_state->sr[wm->level];
2194 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2195 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2196 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2197 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2201 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2203 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2204 struct vlv_wm_values new_wm = {};
2206 vlv_merge_wm(dev_priv, &new_wm);
2208 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2211 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2212 chv_set_memory_dvfs(dev_priv, false);
2214 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2215 chv_set_memory_pm5(dev_priv, false);
2217 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2218 _intel_set_memory_cxsr(dev_priv, false);
2220 vlv_write_wm_values(dev_priv, &new_wm);
2222 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2223 _intel_set_memory_cxsr(dev_priv, true);
2225 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2226 chv_set_memory_pm5(dev_priv, true);
2228 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2229 chv_set_memory_dvfs(dev_priv, true);
2234 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2235 struct intel_crtc *crtc)
2237 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2238 const struct intel_crtc_state *crtc_state =
2239 intel_atomic_get_new_crtc_state(state, crtc);
2241 mutex_lock(&dev_priv->wm.wm_mutex);
2242 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2243 vlv_program_watermarks(dev_priv);
2244 mutex_unlock(&dev_priv->wm.wm_mutex);
2247 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2248 struct intel_crtc *crtc)
2250 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2251 const struct intel_crtc_state *crtc_state =
2252 intel_atomic_get_new_crtc_state(state, crtc);
2254 if (!crtc_state->wm.need_postvbl_update)
2257 mutex_lock(&dev_priv->wm.wm_mutex);
2258 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2259 vlv_program_watermarks(dev_priv);
2260 mutex_unlock(&dev_priv->wm.wm_mutex);
2263 static void i965_update_wm(struct drm_i915_private *dev_priv)
2265 struct intel_crtc *crtc;
2270 /* Calc sr entries for one plane configs */
2271 crtc = single_enabled_crtc(dev_priv);
2273 /* self-refresh has much higher latency */
2274 static const int sr_latency_ns = 12000;
2275 const struct drm_display_mode *pipe_mode =
2276 &crtc->config->hw.pipe_mode;
2277 const struct drm_framebuffer *fb =
2278 crtc->base.primary->state->fb;
2279 int clock = pipe_mode->crtc_clock;
2280 int htotal = pipe_mode->crtc_htotal;
2281 int hdisplay = crtc->config->pipe_src_w;
2282 int cpp = fb->format->cpp[0];
2285 entries = intel_wm_method2(clock, htotal,
2286 hdisplay, cpp, sr_latency_ns / 100);
2287 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2288 srwm = I965_FIFO_SIZE - entries;
2292 drm_dbg_kms(&dev_priv->drm,
2293 "self-refresh entries: %d, wm: %d\n",
2296 entries = intel_wm_method2(clock, htotal,
2297 crtc->base.cursor->state->crtc_w, 4,
2298 sr_latency_ns / 100);
2299 entries = DIV_ROUND_UP(entries,
2300 i965_cursor_wm_info.cacheline_size) +
2301 i965_cursor_wm_info.guard_size;
2303 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2304 if (cursor_sr > i965_cursor_wm_info.max_wm)
2305 cursor_sr = i965_cursor_wm_info.max_wm;
2307 drm_dbg_kms(&dev_priv->drm,
2308 "self-refresh watermark: display plane %d "
2309 "cursor %d\n", srwm, cursor_sr);
2311 cxsr_enabled = true;
2313 cxsr_enabled = false;
2314 /* Turn off self refresh if both pipes are enabled */
2315 intel_set_memory_cxsr(dev_priv, false);
2318 drm_dbg_kms(&dev_priv->drm,
2319 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2322 /* 965 has limitations... */
2323 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2327 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2328 FW_WM(8, PLANEC_OLD));
2329 /* update cursor SR watermark */
2330 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2333 intel_set_memory_cxsr(dev_priv, true);
2338 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2340 const struct intel_watermark_params *wm_info;
2345 int planea_wm, planeb_wm;
2346 struct intel_crtc *crtc, *enabled = NULL;
2348 if (IS_I945GM(dev_priv))
2349 wm_info = &i945_wm_info;
2350 else if (DISPLAY_VER(dev_priv) != 2)
2351 wm_info = &i915_wm_info;
2353 wm_info = &i830_a_wm_info;
2355 if (DISPLAY_VER(dev_priv) == 2)
2356 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2358 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2359 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2360 if (intel_crtc_active(crtc)) {
2361 const struct drm_display_mode *pipe_mode =
2362 &crtc->config->hw.pipe_mode;
2363 const struct drm_framebuffer *fb =
2364 crtc->base.primary->state->fb;
2367 if (DISPLAY_VER(dev_priv) == 2)
2370 cpp = fb->format->cpp[0];
2372 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2373 wm_info, fifo_size, cpp,
2374 pessimal_latency_ns);
2377 planea_wm = fifo_size - wm_info->guard_size;
2378 if (planea_wm > (long)wm_info->max_wm)
2379 planea_wm = wm_info->max_wm;
2382 if (DISPLAY_VER(dev_priv) == 2)
2383 wm_info = &i830_bc_wm_info;
2385 if (DISPLAY_VER(dev_priv) == 2)
2386 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2388 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2389 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2390 if (intel_crtc_active(crtc)) {
2391 const struct drm_display_mode *pipe_mode =
2392 &crtc->config->hw.pipe_mode;
2393 const struct drm_framebuffer *fb =
2394 crtc->base.primary->state->fb;
2397 if (DISPLAY_VER(dev_priv) == 2)
2400 cpp = fb->format->cpp[0];
2402 planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2403 wm_info, fifo_size, cpp,
2404 pessimal_latency_ns);
2405 if (enabled == NULL)
2410 planeb_wm = fifo_size - wm_info->guard_size;
2411 if (planeb_wm > (long)wm_info->max_wm)
2412 planeb_wm = wm_info->max_wm;
2415 drm_dbg_kms(&dev_priv->drm,
2416 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2418 if (IS_I915GM(dev_priv) && enabled) {
2419 struct drm_i915_gem_object *obj;
2421 obj = intel_fb_obj(enabled->base.primary->state->fb);
2423 /* self-refresh seems busted with untiled */
2424 if (!i915_gem_object_is_tiled(obj))
2429 * Overlay gets an aggressive default since video jitter is bad.
2433 /* Play safe and disable self-refresh before adjusting watermarks. */
2434 intel_set_memory_cxsr(dev_priv, false);
2436 /* Calc sr entries for one plane configs */
2437 if (HAS_FW_BLC(dev_priv) && enabled) {
2438 /* self-refresh has much higher latency */
2439 static const int sr_latency_ns = 6000;
2440 const struct drm_display_mode *pipe_mode =
2441 &enabled->config->hw.pipe_mode;
2442 const struct drm_framebuffer *fb =
2443 enabled->base.primary->state->fb;
2444 int clock = pipe_mode->crtc_clock;
2445 int htotal = pipe_mode->crtc_htotal;
2446 int hdisplay = enabled->config->pipe_src_w;
2450 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2453 cpp = fb->format->cpp[0];
2455 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2456 sr_latency_ns / 100);
2457 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2458 drm_dbg_kms(&dev_priv->drm,
2459 "self-refresh entries: %d\n", entries);
2460 srwm = wm_info->fifo_size - entries;
2464 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2465 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2466 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2468 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2471 drm_dbg_kms(&dev_priv->drm,
2472 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2473 planea_wm, planeb_wm, cwm, srwm);
2475 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2476 fwater_hi = (cwm & 0x1f);
2478 /* Set request length to 8 cachelines per fetch */
2479 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2480 fwater_hi = fwater_hi | (1 << 8);
2482 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2483 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2486 intel_set_memory_cxsr(dev_priv, true);
2489 static void i845_update_wm(struct drm_i915_private *dev_priv)
2491 struct intel_crtc *crtc;
2492 const struct drm_display_mode *pipe_mode;
2496 crtc = single_enabled_crtc(dev_priv);
2500 pipe_mode = &crtc->config->hw.pipe_mode;
2501 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2503 i845_get_fifo_size(dev_priv, PLANE_A),
2504 4, pessimal_latency_ns);
2505 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2506 fwater_lo |= (3<<8) | planea_wm;
2508 drm_dbg_kms(&dev_priv->drm,
2509 "Setting FIFO watermarks - A: %d\n", planea_wm);
2511 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2514 /* latency must be in 0.1us units. */
2515 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2517 unsigned int latency)
2521 ret = intel_wm_method1(pixel_rate, cpp, latency);
2522 ret = DIV_ROUND_UP(ret, 64) + 2;
2527 /* latency must be in 0.1us units. */
2528 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2529 unsigned int htotal,
2532 unsigned int latency)
2536 ret = intel_wm_method2(pixel_rate, htotal,
2537 width, cpp, latency);
2538 ret = DIV_ROUND_UP(ret, 64) + 2;
2543 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2546 * Neither of these should be possible since this function shouldn't be
2547 * called if the CRTC is off or the plane is invisible. But let's be
2548 * extra paranoid to avoid a potential divide-by-zero if we screw up
2549 * elsewhere in the driver.
2553 if (WARN_ON(!horiz_pixels))
2556 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2559 struct ilk_wm_maximums {
2567 * For both WM_PIPE and WM_LP.
2568 * mem_value must be in 0.1us units.
2570 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2571 const struct intel_plane_state *plane_state,
2572 u32 mem_value, bool is_lp)
2574 u32 method1, method2;
2580 if (!intel_wm_plane_visible(crtc_state, plane_state))
2583 cpp = plane_state->hw.fb->format->cpp[0];
2585 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2590 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2591 crtc_state->hw.pipe_mode.crtc_htotal,
2592 drm_rect_width(&plane_state->uapi.dst),
2595 return min(method1, method2);
2599 * For both WM_PIPE and WM_LP.
2600 * mem_value must be in 0.1us units.
2602 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2603 const struct intel_plane_state *plane_state,
2606 u32 method1, method2;
2612 if (!intel_wm_plane_visible(crtc_state, plane_state))
2615 cpp = plane_state->hw.fb->format->cpp[0];
2617 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2618 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2619 crtc_state->hw.pipe_mode.crtc_htotal,
2620 drm_rect_width(&plane_state->uapi.dst),
2622 return min(method1, method2);
2626 * For both WM_PIPE and WM_LP.
2627 * mem_value must be in 0.1us units.
2629 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2630 const struct intel_plane_state *plane_state,
2638 if (!intel_wm_plane_visible(crtc_state, plane_state))
2641 cpp = plane_state->hw.fb->format->cpp[0];
2643 return ilk_wm_method2(crtc_state->pixel_rate,
2644 crtc_state->hw.pipe_mode.crtc_htotal,
2645 drm_rect_width(&plane_state->uapi.dst),
2649 /* Only for WM_LP. */
2650 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2651 const struct intel_plane_state *plane_state,
2656 if (!intel_wm_plane_visible(crtc_state, plane_state))
2659 cpp = plane_state->hw.fb->format->cpp[0];
2661 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2666 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2668 if (DISPLAY_VER(dev_priv) >= 8)
2670 else if (DISPLAY_VER(dev_priv) >= 7)
2677 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2678 int level, bool is_sprite)
2680 if (DISPLAY_VER(dev_priv) >= 8)
2681 /* BDW primary/sprite plane watermarks */
2682 return level == 0 ? 255 : 2047;
2683 else if (DISPLAY_VER(dev_priv) >= 7)
2684 /* IVB/HSW primary/sprite plane watermarks */
2685 return level == 0 ? 127 : 1023;
2686 else if (!is_sprite)
2687 /* ILK/SNB primary plane watermarks */
2688 return level == 0 ? 127 : 511;
2690 /* ILK/SNB sprite plane watermarks */
2691 return level == 0 ? 63 : 255;
2695 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2697 if (DISPLAY_VER(dev_priv) >= 7)
2698 return level == 0 ? 63 : 255;
2700 return level == 0 ? 31 : 63;
2703 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2705 if (DISPLAY_VER(dev_priv) >= 8)
2711 /* Calculate the maximum primary/sprite plane watermark */
2712 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2714 const struct intel_wm_config *config,
2715 enum intel_ddb_partitioning ddb_partitioning,
2718 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2720 /* if sprites aren't enabled, sprites get nothing */
2721 if (is_sprite && !config->sprites_enabled)
2724 /* HSW allows LP1+ watermarks even with multiple pipes */
2725 if (level == 0 || config->num_pipes_active > 1) {
2726 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2729 * For some reason the non self refresh
2730 * FIFO size is only half of the self
2731 * refresh FIFO size on ILK/SNB.
2733 if (DISPLAY_VER(dev_priv) <= 6)
2737 if (config->sprites_enabled) {
2738 /* level 0 is always calculated with 1:1 split */
2739 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2748 /* clamp to max that the registers can hold */
2749 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2752 /* Calculate the maximum cursor plane watermark */
2753 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2755 const struct intel_wm_config *config)
2757 /* HSW LP1+ watermarks w/ multiple pipes */
2758 if (level > 0 && config->num_pipes_active > 1)
2761 /* otherwise just report max that registers can hold */
2762 return ilk_cursor_wm_reg_max(dev_priv, level);
2765 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2767 const struct intel_wm_config *config,
2768 enum intel_ddb_partitioning ddb_partitioning,
2769 struct ilk_wm_maximums *max)
2771 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2772 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2773 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2774 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2777 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2779 struct ilk_wm_maximums *max)
2781 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2782 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2783 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2784 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2787 static bool ilk_validate_wm_level(int level,
2788 const struct ilk_wm_maximums *max,
2789 struct intel_wm_level *result)
2793 /* already determined to be invalid? */
2794 if (!result->enable)
2797 result->enable = result->pri_val <= max->pri &&
2798 result->spr_val <= max->spr &&
2799 result->cur_val <= max->cur;
2801 ret = result->enable;
2804 * HACK until we can pre-compute everything,
2805 * and thus fail gracefully if LP0 watermarks
2808 if (level == 0 && !result->enable) {
2809 if (result->pri_val > max->pri)
2810 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2811 level, result->pri_val, max->pri);
2812 if (result->spr_val > max->spr)
2813 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2814 level, result->spr_val, max->spr);
2815 if (result->cur_val > max->cur)
2816 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2817 level, result->cur_val, max->cur);
2819 result->pri_val = min_t(u32, result->pri_val, max->pri);
2820 result->spr_val = min_t(u32, result->spr_val, max->spr);
2821 result->cur_val = min_t(u32, result->cur_val, max->cur);
2822 result->enable = true;
2828 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2829 const struct intel_crtc *crtc,
2831 struct intel_crtc_state *crtc_state,
2832 const struct intel_plane_state *pristate,
2833 const struct intel_plane_state *sprstate,
2834 const struct intel_plane_state *curstate,
2835 struct intel_wm_level *result)
2837 u16 pri_latency = dev_priv->wm.pri_latency[level];
2838 u16 spr_latency = dev_priv->wm.spr_latency[level];
2839 u16 cur_latency = dev_priv->wm.cur_latency[level];
2841 /* WM1+ latency values stored in 0.5us units */
2849 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2850 pri_latency, level);
2851 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2855 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2858 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2860 result->enable = true;
2863 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2866 struct intel_uncore *uncore = &dev_priv->uncore;
2868 if (DISPLAY_VER(dev_priv) >= 9) {
2871 int level, max_level = ilk_wm_max_level(dev_priv);
2872 int mult = IS_DG2(dev_priv) ? 2 : 1;
2874 /* read the first set of memory latencies[0:3] */
2875 val = 0; /* data0 to be programmed to 0 for first set */
2876 ret = sandybridge_pcode_read(dev_priv,
2877 GEN9_PCODE_READ_MEM_LATENCY,
2881 drm_err(&dev_priv->drm,
2882 "SKL Mailbox read error = %d\n", ret);
2886 wm[0] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2887 wm[1] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2888 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2889 wm[2] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2890 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2891 wm[3] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2892 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2894 /* read the second set of memory latencies[4:7] */
2895 val = 1; /* data0 to be programmed to 1 for second set */
2896 ret = sandybridge_pcode_read(dev_priv,
2897 GEN9_PCODE_READ_MEM_LATENCY,
2900 drm_err(&dev_priv->drm,
2901 "SKL Mailbox read error = %d\n", ret);
2905 wm[4] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2906 wm[5] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2907 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2908 wm[6] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2909 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2910 wm[7] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2911 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2914 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2915 * need to be disabled. We make sure to sanitize the values out
2916 * of the punit to satisfy this requirement.
2918 for (level = 1; level <= max_level; level++) {
2919 if (wm[level] == 0) {
2920 for (i = level + 1; i <= max_level; i++)
2923 max_level = level - 1;
2930 * WaWmMemoryReadLatency
2932 * punit doesn't take into account the read latency so we need
2933 * to add proper adjustement to each valid level we retrieve
2934 * from the punit when level 0 response data is 0us.
2937 u8 adjust = DISPLAY_VER(dev_priv) >= 12 ? 3 : 2;
2939 for (level = 0; level <= max_level; level++)
2940 wm[level] += adjust;
2944 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2945 * If we could not get dimm info enable this WA to prevent from
2946 * any underrun. If not able to get Dimm info assume 16GB dimm
2947 * to avoid any underrun.
2949 if (dev_priv->dram_info.wm_lv_0_adjust_needed)
2951 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2952 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2954 wm[0] = (sskpd >> 56) & 0xFF;
2956 wm[0] = sskpd & 0xF;
2957 wm[1] = (sskpd >> 4) & 0xFF;
2958 wm[2] = (sskpd >> 12) & 0xFF;
2959 wm[3] = (sskpd >> 20) & 0x1FF;
2960 wm[4] = (sskpd >> 32) & 0x1FF;
2961 } else if (DISPLAY_VER(dev_priv) >= 6) {
2962 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2964 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2965 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2966 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2967 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2968 } else if (DISPLAY_VER(dev_priv) >= 5) {
2969 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2971 /* ILK primary LP0 latency is 700 ns */
2973 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2974 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2976 MISSING_CASE(INTEL_DEVID(dev_priv));
2980 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2983 /* ILK sprite LP0 latency is 1300 ns */
2984 if (DISPLAY_VER(dev_priv) == 5)
2988 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2991 /* ILK cursor LP0 latency is 1300 ns */
2992 if (DISPLAY_VER(dev_priv) == 5)
2996 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2998 /* how many WM levels are we expecting */
2999 if (HAS_HW_SAGV_WM(dev_priv))
3001 else if (DISPLAY_VER(dev_priv) >= 9)
3003 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3005 else if (DISPLAY_VER(dev_priv) >= 6)
3011 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3015 int level, max_level = ilk_wm_max_level(dev_priv);
3017 for (level = 0; level <= max_level; level++) {
3018 unsigned int latency = wm[level];
3021 drm_dbg_kms(&dev_priv->drm,
3022 "%s WM%d latency not provided\n",
3028 * - latencies are in us on gen9.
3029 * - before then, WM1+ latency values are in 0.5us units
3031 if (DISPLAY_VER(dev_priv) >= 9)
3036 drm_dbg_kms(&dev_priv->drm,
3037 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3038 wm[level], latency / 10, latency % 10);
3042 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3045 int level, max_level = ilk_wm_max_level(dev_priv);
3050 wm[0] = max(wm[0], min);
3051 for (level = 1; level <= max_level; level++)
3052 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3057 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3062 * The BIOS provided WM memory latency values are often
3063 * inadequate for high resolution displays. Adjust them.
3065 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3066 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3067 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3072 drm_dbg_kms(&dev_priv->drm,
3073 "WM latency values increased to avoid potential underruns\n");
3074 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3075 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3076 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3079 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3082 * On some SNB machines (Thinkpad X220 Tablet at least)
3083 * LP3 usage can cause vblank interrupts to be lost.
3084 * The DEIIR bit will go high but it looks like the CPU
3085 * never gets interrupted.
3087 * It's not clear whether other interrupt source could
3088 * be affected or if this is somehow limited to vblank
3089 * interrupts only. To play it safe we disable LP3
3090 * watermarks entirely.
3092 if (dev_priv->wm.pri_latency[3] == 0 &&
3093 dev_priv->wm.spr_latency[3] == 0 &&
3094 dev_priv->wm.cur_latency[3] == 0)
3097 dev_priv->wm.pri_latency[3] = 0;
3098 dev_priv->wm.spr_latency[3] = 0;
3099 dev_priv->wm.cur_latency[3] = 0;
3101 drm_dbg_kms(&dev_priv->drm,
3102 "LP3 watermarks disabled due to potential for lost interrupts\n");
3103 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3104 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3105 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3108 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3110 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3112 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3113 sizeof(dev_priv->wm.pri_latency));
3114 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3115 sizeof(dev_priv->wm.pri_latency));
3117 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3118 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3120 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3121 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3122 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3124 if (DISPLAY_VER(dev_priv) == 6) {
3125 snb_wm_latency_quirk(dev_priv);
3126 snb_wm_lp3_irq_quirk(dev_priv);
3130 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3132 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3133 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3136 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3137 struct intel_pipe_wm *pipe_wm)
3139 /* LP0 watermark maximums depend on this pipe alone */
3140 const struct intel_wm_config config = {
3141 .num_pipes_active = 1,
3142 .sprites_enabled = pipe_wm->sprites_enabled,
3143 .sprites_scaled = pipe_wm->sprites_scaled,
3145 struct ilk_wm_maximums max;
3147 /* LP0 watermarks always use 1/2 DDB partitioning */
3148 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3150 /* At least LP0 must be valid */
3151 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3152 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3159 /* Compute new watermarks for the pipe */
3160 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3161 struct intel_crtc *crtc)
3163 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3164 struct intel_crtc_state *crtc_state =
3165 intel_atomic_get_new_crtc_state(state, crtc);
3166 struct intel_pipe_wm *pipe_wm;
3167 struct intel_plane *plane;
3168 const struct intel_plane_state *plane_state;
3169 const struct intel_plane_state *pristate = NULL;
3170 const struct intel_plane_state *sprstate = NULL;
3171 const struct intel_plane_state *curstate = NULL;
3172 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3173 struct ilk_wm_maximums max;
3175 pipe_wm = &crtc_state->wm.ilk.optimal;
3177 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3178 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3179 pristate = plane_state;
3180 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3181 sprstate = plane_state;
3182 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3183 curstate = plane_state;
3186 pipe_wm->pipe_enabled = crtc_state->hw.active;
3188 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3189 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3190 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3191 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3194 usable_level = max_level;
3196 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3197 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3200 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3201 if (pipe_wm->sprites_scaled)
3204 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3205 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3206 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3208 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3211 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3213 for (level = 1; level <= usable_level; level++) {
3214 struct intel_wm_level *wm = &pipe_wm->wm[level];
3216 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3217 pristate, sprstate, curstate, wm);
3220 * Disable any watermark level that exceeds the
3221 * register maximums since such watermarks are
3224 if (!ilk_validate_wm_level(level, &max, wm)) {
3225 memset(wm, 0, sizeof(*wm));
3234 * Build a set of 'intermediate' watermark values that satisfy both the old
3235 * state and the new state. These can be programmed to the hardware
3238 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3239 struct intel_crtc *crtc)
3241 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3242 struct intel_crtc_state *new_crtc_state =
3243 intel_atomic_get_new_crtc_state(state, crtc);
3244 const struct intel_crtc_state *old_crtc_state =
3245 intel_atomic_get_old_crtc_state(state, crtc);
3246 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3247 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3248 int level, max_level = ilk_wm_max_level(dev_priv);
3251 * Start with the final, target watermarks, then combine with the
3252 * currently active watermarks to get values that are safe both before
3253 * and after the vblank.
3255 *a = new_crtc_state->wm.ilk.optimal;
3256 if (!new_crtc_state->hw.active ||
3257 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) ||
3258 state->skip_intermediate_wm)
3261 a->pipe_enabled |= b->pipe_enabled;
3262 a->sprites_enabled |= b->sprites_enabled;
3263 a->sprites_scaled |= b->sprites_scaled;
3265 for (level = 0; level <= max_level; level++) {
3266 struct intel_wm_level *a_wm = &a->wm[level];
3267 const struct intel_wm_level *b_wm = &b->wm[level];
3269 a_wm->enable &= b_wm->enable;
3270 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3271 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3272 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3273 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3277 * We need to make sure that these merged watermark values are
3278 * actually a valid configuration themselves. If they're not,
3279 * there's no safe way to transition from the old state to
3280 * the new state, so we need to fail the atomic transaction.
3282 if (!ilk_validate_pipe_wm(dev_priv, a))
3286 * If our intermediate WM are identical to the final WM, then we can
3287 * omit the post-vblank programming; only update if it's different.
3289 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3290 new_crtc_state->wm.need_postvbl_update = true;
3296 * Merge the watermarks from all active pipes for a specific level.
3298 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3300 struct intel_wm_level *ret_wm)
3302 const struct intel_crtc *crtc;
3304 ret_wm->enable = true;
3306 for_each_intel_crtc(&dev_priv->drm, crtc) {
3307 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3308 const struct intel_wm_level *wm = &active->wm[level];
3310 if (!active->pipe_enabled)
3314 * The watermark values may have been used in the past,
3315 * so we must maintain them in the registers for some
3316 * time even if the level is now disabled.
3319 ret_wm->enable = false;
3321 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3322 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3323 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3324 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3329 * Merge all low power watermarks for all active pipes.
3331 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3332 const struct intel_wm_config *config,
3333 const struct ilk_wm_maximums *max,
3334 struct intel_pipe_wm *merged)
3336 int level, max_level = ilk_wm_max_level(dev_priv);
3337 int last_enabled_level = max_level;
3339 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3340 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3341 config->num_pipes_active > 1)
3342 last_enabled_level = 0;
3344 /* ILK: FBC WM must be disabled always */
3345 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3347 /* merge each WM1+ level */
3348 for (level = 1; level <= max_level; level++) {
3349 struct intel_wm_level *wm = &merged->wm[level];
3351 ilk_merge_wm_level(dev_priv, level, wm);
3353 if (level > last_enabled_level)
3355 else if (!ilk_validate_wm_level(level, max, wm))
3356 /* make sure all following levels get disabled */
3357 last_enabled_level = level - 1;
3360 * The spec says it is preferred to disable
3361 * FBC WMs instead of disabling a WM level.
3363 if (wm->fbc_val > max->fbc) {
3365 merged->fbc_wm_enabled = false;
3370 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3372 * FIXME this is racy. FBC might get enabled later.
3373 * What we should check here is whether FBC can be
3374 * enabled sometime later.
3376 if (DISPLAY_VER(dev_priv) == 5 && !merged->fbc_wm_enabled &&
3377 intel_fbc_is_active(dev_priv)) {
3378 for (level = 2; level <= max_level; level++) {
3379 struct intel_wm_level *wm = &merged->wm[level];
3386 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3388 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3389 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3392 /* The value we need to program into the WM_LPx latency field */
3393 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3396 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3399 return dev_priv->wm.pri_latency[level];
3402 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3403 const struct intel_pipe_wm *merged,
3404 enum intel_ddb_partitioning partitioning,
3405 struct ilk_wm_values *results)
3407 struct intel_crtc *crtc;
3410 results->enable_fbc_wm = merged->fbc_wm_enabled;
3411 results->partitioning = partitioning;
3413 /* LP1+ register values */
3414 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3415 const struct intel_wm_level *r;
3417 level = ilk_wm_lp_to_level(wm_lp, merged);
3419 r = &merged->wm[level];
3422 * Maintain the watermark values even if the level is
3423 * disabled. Doing otherwise could cause underruns.
3425 results->wm_lp[wm_lp - 1] =
3426 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3427 (r->pri_val << WM1_LP_SR_SHIFT) |
3431 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3433 if (DISPLAY_VER(dev_priv) >= 8)
3434 results->wm_lp[wm_lp - 1] |=
3435 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3437 results->wm_lp[wm_lp - 1] |=
3438 r->fbc_val << WM1_LP_FBC_SHIFT;
3441 * Always set WM1S_LP_EN when spr_val != 0, even if the
3442 * level is disabled. Doing otherwise could cause underruns.
3444 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3445 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3446 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3448 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3451 /* LP0 register values */
3452 for_each_intel_crtc(&dev_priv->drm, crtc) {
3453 enum pipe pipe = crtc->pipe;
3454 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3455 const struct intel_wm_level *r = &pipe_wm->wm[0];
3457 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3460 results->wm_pipe[pipe] =
3461 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3462 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3467 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3468 * case both are at the same level. Prefer r1 in case they're the same. */
3469 static struct intel_pipe_wm *
3470 ilk_find_best_result(struct drm_i915_private *dev_priv,
3471 struct intel_pipe_wm *r1,
3472 struct intel_pipe_wm *r2)
3474 int level, max_level = ilk_wm_max_level(dev_priv);
3475 int level1 = 0, level2 = 0;
3477 for (level = 1; level <= max_level; level++) {
3478 if (r1->wm[level].enable)
3480 if (r2->wm[level].enable)
3484 if (level1 == level2) {
3485 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3489 } else if (level1 > level2) {
3496 /* dirty bits used to track which watermarks need changes */
3497 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3498 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3499 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3500 #define WM_DIRTY_FBC (1 << 24)
3501 #define WM_DIRTY_DDB (1 << 25)
3503 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3504 const struct ilk_wm_values *old,
3505 const struct ilk_wm_values *new)
3507 unsigned int dirty = 0;
3511 for_each_pipe(dev_priv, pipe) {
3512 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3513 dirty |= WM_DIRTY_PIPE(pipe);
3514 /* Must disable LP1+ watermarks too */
3515 dirty |= WM_DIRTY_LP_ALL;
3519 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3520 dirty |= WM_DIRTY_FBC;
3521 /* Must disable LP1+ watermarks too */
3522 dirty |= WM_DIRTY_LP_ALL;
3525 if (old->partitioning != new->partitioning) {
3526 dirty |= WM_DIRTY_DDB;
3527 /* Must disable LP1+ watermarks too */
3528 dirty |= WM_DIRTY_LP_ALL;
3531 /* LP1+ watermarks already deemed dirty, no need to continue */
3532 if (dirty & WM_DIRTY_LP_ALL)
3535 /* Find the lowest numbered LP1+ watermark in need of an update... */
3536 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3537 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3538 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3542 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3543 for (; wm_lp <= 3; wm_lp++)
3544 dirty |= WM_DIRTY_LP(wm_lp);
3549 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3552 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3553 bool changed = false;
3555 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3556 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3557 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3560 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3561 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3562 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3565 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3566 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3567 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3572 * Don't touch WM1S_LP_EN here.
3573 * Doing so could cause underruns.
3580 * The spec says we shouldn't write when we don't need, because every write
3581 * causes WMs to be re-evaluated, expending some power.
3583 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3584 struct ilk_wm_values *results)
3586 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3590 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3594 _ilk_disable_lp_wm(dev_priv, dirty);
3596 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3597 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3598 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3599 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3600 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3601 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3603 if (dirty & WM_DIRTY_DDB) {
3604 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3605 val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3606 if (results->partitioning == INTEL_DDB_PART_1_2)
3607 val &= ~WM_MISC_DATA_PARTITION_5_6;
3609 val |= WM_MISC_DATA_PARTITION_5_6;
3610 intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3612 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3613 if (results->partitioning == INTEL_DDB_PART_1_2)
3614 val &= ~DISP_DATA_PARTITION_5_6;
3616 val |= DISP_DATA_PARTITION_5_6;
3617 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3621 if (dirty & WM_DIRTY_FBC) {
3622 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3623 if (results->enable_fbc_wm)
3624 val &= ~DISP_FBC_WM_DIS;
3626 val |= DISP_FBC_WM_DIS;
3627 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3630 if (dirty & WM_DIRTY_LP(1) &&
3631 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3632 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3634 if (DISPLAY_VER(dev_priv) >= 7) {
3635 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3636 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3637 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3638 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3641 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3642 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3643 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3644 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3645 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3646 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3648 dev_priv->wm.hw = *results;
3651 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3653 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3656 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3658 u8 enabled_slices = 0;
3659 enum dbuf_slice slice;
3661 for_each_dbuf_slice(dev_priv, slice) {
3662 if (intel_uncore_read(&dev_priv->uncore,
3663 DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
3664 enabled_slices |= BIT(slice);
3667 return enabled_slices;
3671 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3672 * so assume we'll always need it in order to avoid underruns.
3674 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3676 return DISPLAY_VER(dev_priv) == 9;
3680 intel_has_sagv(struct drm_i915_private *dev_priv)
3682 return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) &&
3683 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3687 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3689 if (DISPLAY_VER(dev_priv) >= 12) {
3693 ret = sandybridge_pcode_read(dev_priv,
3694 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3697 dev_priv->sagv_block_time_us = val;
3701 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3702 } else if (DISPLAY_VER(dev_priv) == 11) {
3703 dev_priv->sagv_block_time_us = 10;
3705 } else if (DISPLAY_VER(dev_priv) == 10) {
3706 dev_priv->sagv_block_time_us = 20;
3708 } else if (DISPLAY_VER(dev_priv) == 9) {
3709 dev_priv->sagv_block_time_us = 30;
3712 MISSING_CASE(DISPLAY_VER(dev_priv));
3715 /* Default to an unusable block time */
3716 dev_priv->sagv_block_time_us = -1;
3720 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3721 * depending on power and performance requirements. The display engine access
3722 * to system memory is blocked during the adjustment time. Because of the
3723 * blocking time, having this enabled can cause full system hangs and/or pipe
3724 * underruns if we don't meet all of the following requirements:
3726 * - <= 1 pipe enabled
3727 * - All planes can enable watermarks for latencies >= SAGV engine block time
3728 * - We're not using an interlaced display configuration
3731 intel_enable_sagv(struct drm_i915_private *dev_priv)
3735 if (!intel_has_sagv(dev_priv))
3738 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3741 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3742 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3745 /* We don't need to wait for SAGV when enabling */
3748 * Some skl systems, pre-release machines in particular,
3749 * don't actually have SAGV.
3751 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3752 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3753 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3755 } else if (ret < 0) {
3756 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3760 dev_priv->sagv_status = I915_SAGV_ENABLED;
3765 intel_disable_sagv(struct drm_i915_private *dev_priv)
3769 if (!intel_has_sagv(dev_priv))
3772 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3775 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3776 /* bspec says to keep retrying for at least 1 ms */
3777 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3779 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3782 * Some skl systems, pre-release machines in particular,
3783 * don't actually have SAGV.
3785 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3786 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3787 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3789 } else if (ret < 0) {
3790 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3794 dev_priv->sagv_status = I915_SAGV_DISABLED;
3798 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3800 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3801 const struct intel_bw_state *new_bw_state;
3802 const struct intel_bw_state *old_bw_state;
3806 * Just return if we can't control SAGV or don't have it.
3807 * This is different from situation when we have SAGV but just can't
3808 * afford it due to DBuf limitation - in case if SAGV is completely
3809 * disabled in a BIOS, we are not even allowed to send a PCode request,
3810 * as it will throw an error. So have to check it here.
3812 if (!intel_has_sagv(dev_priv))
3815 new_bw_state = intel_atomic_get_new_bw_state(state);
3819 if (DISPLAY_VER(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3820 intel_disable_sagv(dev_priv);
3824 old_bw_state = intel_atomic_get_old_bw_state(state);
3828 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3831 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3834 * If new mask is zero - means there is nothing to mask,
3835 * we can only unmask, which should be done in unmask.
3841 * Restrict required qgv points before updating the configuration.
3842 * According to BSpec we can't mask and unmask qgv points at the same
3843 * time. Also masking should be done before updating the configuration
3844 * and unmasking afterwards.
3846 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3849 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3851 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3852 const struct intel_bw_state *new_bw_state;
3853 const struct intel_bw_state *old_bw_state;
3857 * Just return if we can't control SAGV or don't have it.
3858 * This is different from situation when we have SAGV but just can't
3859 * afford it due to DBuf limitation - in case if SAGV is completely
3860 * disabled in a BIOS, we are not even allowed to send a PCode request,
3861 * as it will throw an error. So have to check it here.
3863 if (!intel_has_sagv(dev_priv))
3866 new_bw_state = intel_atomic_get_new_bw_state(state);
3870 if (DISPLAY_VER(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3871 intel_enable_sagv(dev_priv);
3875 old_bw_state = intel_atomic_get_old_bw_state(state);
3879 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3882 new_mask = new_bw_state->qgv_points_mask;
3885 * Allow required qgv points after updating the configuration.
3886 * According to BSpec we can't mask and unmask qgv points at the same
3887 * time. Also masking should be done before updating the configuration
3888 * and unmasking afterwards.
3890 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3893 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3895 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3896 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3897 enum plane_id plane_id;
3898 int max_level = INT_MAX;
3900 if (!intel_has_sagv(dev_priv))
3903 if (!crtc_state->hw.active)
3906 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3909 for_each_plane_id_on_crtc(crtc, plane_id) {
3910 const struct skl_plane_wm *wm =
3911 &crtc_state->wm.skl.optimal.planes[plane_id];
3914 /* Skip this plane if it's not enabled */
3915 if (!wm->wm[0].enable)
3918 /* Find the highest enabled wm level for this plane */
3919 for (level = ilk_wm_max_level(dev_priv);
3920 !wm->wm[level].enable; --level)
3923 /* Highest common enabled wm level for all planes */
3924 max_level = min(level, max_level);
3927 /* No enabled planes? */
3928 if (max_level == INT_MAX)
3931 for_each_plane_id_on_crtc(crtc, plane_id) {
3932 const struct skl_plane_wm *wm =
3933 &crtc_state->wm.skl.optimal.planes[plane_id];
3936 * All enabled planes must have enabled a common wm level that
3937 * can tolerate memory latencies higher than sagv_block_time_us
3939 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
3946 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3948 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3949 enum plane_id plane_id;
3951 if (!crtc_state->hw.active)
3954 for_each_plane_id_on_crtc(crtc, plane_id) {
3955 const struct skl_plane_wm *wm =
3956 &crtc_state->wm.skl.optimal.planes[plane_id];
3958 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
3965 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3967 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3968 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3970 if (DISPLAY_VER(dev_priv) >= 12)
3971 return tgl_crtc_can_enable_sagv(crtc_state);
3973 return skl_crtc_can_enable_sagv(crtc_state);
3976 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3977 const struct intel_bw_state *bw_state)
3979 if (DISPLAY_VER(dev_priv) < 11 &&
3980 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3983 return bw_state->pipe_sagv_reject == 0;
3986 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3988 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3990 struct intel_crtc *crtc;
3991 struct intel_crtc_state *new_crtc_state;
3992 struct intel_bw_state *new_bw_state = NULL;
3993 const struct intel_bw_state *old_bw_state = NULL;
3996 for_each_new_intel_crtc_in_state(state, crtc,
3997 new_crtc_state, i) {
3998 new_bw_state = intel_atomic_get_bw_state(state);
3999 if (IS_ERR(new_bw_state))
4000 return PTR_ERR(new_bw_state);
4002 old_bw_state = intel_atomic_get_old_bw_state(state);
4004 if (intel_crtc_can_enable_sagv(new_crtc_state))
4005 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
4007 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
4013 new_bw_state->active_pipes =
4014 intel_calc_active_pipes(state, old_bw_state->active_pipes);
4016 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4017 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4022 for_each_new_intel_crtc_in_state(state, crtc,
4023 new_crtc_state, i) {
4024 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4027 * We store use_sagv_wm in the crtc state rather than relying on
4028 * that bw state since we have no convenient way to get at the
4029 * latter from the plane commit hooks (especially in the legacy
4032 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) &&
4033 DISPLAY_VER(dev_priv) >= 12 &&
4034 intel_can_enable_sagv(dev_priv, new_bw_state);
4037 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4038 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4039 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4042 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4043 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4051 static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4053 return INTEL_INFO(dev_priv)->dbuf.size /
4054 hweight8(INTEL_INFO(dev_priv)->dbuf.slice_mask);
4058 skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4059 struct skl_ddb_entry *ddb)
4061 int slice_size = intel_dbuf_slice_size(dev_priv);
4069 ddb->start = (ffs(slice_mask) - 1) * slice_size;
4070 ddb->end = fls(slice_mask) * slice_size;
4072 WARN_ON(ddb->start >= ddb->end);
4073 WARN_ON(ddb->end > INTEL_INFO(dev_priv)->dbuf.size);
4076 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
4078 struct skl_ddb_entry ddb;
4080 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
4081 slice_mask = BIT(DBUF_S1);
4082 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
4083 slice_mask = BIT(DBUF_S3);
4085 skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
4090 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4091 const struct skl_ddb_entry *entry)
4093 int slice_size = intel_dbuf_slice_size(dev_priv);
4094 enum dbuf_slice start_slice, end_slice;
4097 if (!skl_ddb_entry_size(entry))
4100 start_slice = entry->start / slice_size;
4101 end_slice = (entry->end - 1) / slice_size;
4104 * Per plane DDB entry can in a really worst case be on multiple slices
4105 * but single entry is anyway contigious.
4107 while (start_slice <= end_slice) {
4108 slice_mask |= BIT(start_slice);
4115 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4117 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4118 int hdisplay, vdisplay;
4120 if (!crtc_state->hw.active)
4124 * Watermark/ddb requirement highly depends upon width of the
4125 * framebuffer, So instead of allocating DDB equally among pipes
4126 * distribute DDB based on resolution/width of the display.
4128 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4133 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4135 unsigned int *weight_start,
4136 unsigned int *weight_end,
4137 unsigned int *weight_total)
4139 struct drm_i915_private *dev_priv =
4140 to_i915(dbuf_state->base.state->base.dev);
4147 for_each_pipe(dev_priv, pipe) {
4148 int weight = dbuf_state->weight[pipe];
4151 * Do not account pipes using other slice sets
4152 * luckily as of current BSpec slice sets do not partially
4153 * intersect(pipes share either same one slice or same slice set
4154 * i.e no partial intersection), so it is enough to check for
4157 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4160 *weight_total += weight;
4161 if (pipe < for_pipe) {
4162 *weight_start += weight;
4163 *weight_end += weight;
4164 } else if (pipe == for_pipe) {
4165 *weight_end += weight;
4171 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4173 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4174 unsigned int weight_total, weight_start, weight_end;
4175 const struct intel_dbuf_state *old_dbuf_state =
4176 intel_atomic_get_old_dbuf_state(state);
4177 struct intel_dbuf_state *new_dbuf_state =
4178 intel_atomic_get_new_dbuf_state(state);
4179 struct intel_crtc_state *crtc_state;
4180 struct skl_ddb_entry ddb_slices;
4181 enum pipe pipe = crtc->pipe;
4182 unsigned int mbus_offset = 0;
4184 u32 dbuf_slice_mask;
4188 if (new_dbuf_state->weight[pipe] == 0) {
4189 new_dbuf_state->ddb[pipe].start = 0;
4190 new_dbuf_state->ddb[pipe].end = 0;
4194 dbuf_slice_mask = new_dbuf_state->slices[pipe];
4196 skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4197 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask);
4198 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4200 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4201 &weight_start, &weight_end, &weight_total);
4203 start = ddb_range_size * weight_start / weight_total;
4204 end = ddb_range_size * weight_end / weight_total;
4206 new_dbuf_state->ddb[pipe].start = ddb_slices.start - mbus_offset + start;
4207 new_dbuf_state->ddb[pipe].end = ddb_slices.start - mbus_offset + end;
4209 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
4210 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4211 &new_dbuf_state->ddb[pipe]))
4214 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4218 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4219 if (IS_ERR(crtc_state))
4220 return PTR_ERR(crtc_state);
4223 * Used for checking overlaps, so we need absolute
4224 * offsets instead of MBUS relative offsets.
4226 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
4227 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
4229 drm_dbg_kms(&dev_priv->drm,
4230 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4231 crtc->base.base.id, crtc->base.name,
4232 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4233 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4234 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4235 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4240 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4241 int width, const struct drm_format_info *format,
4242 u64 modifier, unsigned int rotation,
4243 u32 plane_pixel_rate, struct skl_wm_params *wp,
4245 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4247 unsigned int latency,
4248 const struct skl_wm_params *wp,
4249 const struct skl_wm_level *result_prev,
4250 struct skl_wm_level *result /* out */);
4253 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4256 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4257 int level, max_level = ilk_wm_max_level(dev_priv);
4258 struct skl_wm_level wm = {};
4259 int ret, min_ddb_alloc = 0;
4260 struct skl_wm_params wp;
4262 ret = skl_compute_wm_params(crtc_state, 256,
4263 drm_format_info(DRM_FORMAT_ARGB8888),
4264 DRM_FORMAT_MOD_LINEAR,
4266 crtc_state->pixel_rate, &wp, 0);
4267 drm_WARN_ON(&dev_priv->drm, ret);
4269 for (level = 0; level <= max_level; level++) {
4270 unsigned int latency = dev_priv->wm.skl_latency[level];
4272 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4273 if (wm.min_ddb_alloc == U16_MAX)
4276 min_ddb_alloc = wm.min_ddb_alloc;
4279 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4282 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4283 struct skl_ddb_entry *entry, u32 reg)
4285 entry->start = reg & DDB_ENTRY_MASK;
4286 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4293 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4294 const enum pipe pipe,
4295 const enum plane_id plane_id,
4296 struct skl_ddb_entry *ddb_y,
4297 struct skl_ddb_entry *ddb_uv)
4302 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4303 if (plane_id == PLANE_CURSOR) {
4304 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4305 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4309 val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id));
4311 /* No DDB allocated for disabled planes */
4312 if (val & PLANE_CTL_ENABLE)
4313 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4314 val & PLANE_CTL_ORDER_RGBX,
4315 val & PLANE_CTL_ALPHA_MASK);
4317 if (DISPLAY_VER(dev_priv) >= 11) {
4318 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4319 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4321 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4322 val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4325 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4328 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4329 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4333 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4334 struct skl_ddb_entry *ddb_y,
4335 struct skl_ddb_entry *ddb_uv)
4337 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4338 enum intel_display_power_domain power_domain;
4339 enum pipe pipe = crtc->pipe;
4340 intel_wakeref_t wakeref;
4341 enum plane_id plane_id;
4343 power_domain = POWER_DOMAIN_PIPE(pipe);
4344 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4348 for_each_plane_id_on_crtc(crtc, plane_id)
4349 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4354 intel_display_power_put(dev_priv, power_domain, wakeref);
4358 * Determines the downscale amount of a plane for the purposes of watermark calculations.
4359 * The bspec defines downscale amount as:
4362 * Horizontal down scale amount = maximum[1, Horizontal source size /
4363 * Horizontal destination size]
4364 * Vertical down scale amount = maximum[1, Vertical source size /
4365 * Vertical destination size]
4366 * Total down scale amount = Horizontal down scale amount *
4367 * Vertical down scale amount
4370 * Return value is provided in 16.16 fixed point form to retain fractional part.
4371 * Caller should take care of dividing & rounding off the value.
4373 static uint_fixed_16_16_t
4374 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4375 const struct intel_plane_state *plane_state)
4377 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4378 u32 src_w, src_h, dst_w, dst_h;
4379 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4380 uint_fixed_16_16_t downscale_h, downscale_w;
4382 if (drm_WARN_ON(&dev_priv->drm,
4383 !intel_wm_plane_visible(crtc_state, plane_state)))
4384 return u32_to_fixed16(0);
4387 * Src coordinates are already rotated by 270 degrees for
4388 * the 90/270 degree plane rotation cases (to match the
4389 * GTT mapping), hence no need to account for rotation here.
4391 * n.b., src is 16.16 fixed point, dst is whole integer.
4393 src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4394 src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4395 dst_w = drm_rect_width(&plane_state->uapi.dst);
4396 dst_h = drm_rect_height(&plane_state->uapi.dst);
4398 fp_w_ratio = div_fixed16(src_w, dst_w);
4399 fp_h_ratio = div_fixed16(src_h, dst_h);
4400 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4401 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4403 return mul_fixed16(downscale_w, downscale_h);
4406 struct dbuf_slice_conf_entry {
4408 u8 dbuf_mask[I915_MAX_PIPES];
4413 * Table taken from Bspec 12716
4414 * Pipes do have some preferred DBuf slice affinity,
4415 * plus there are some hardcoded requirements on how
4416 * those should be distributed for multipipe scenarios.
4417 * For more DBuf slices algorithm can get even more messy
4418 * and less readable, so decided to use a table almost
4419 * as is from BSpec itself - that way it is at least easier
4420 * to compare, change and check.
4422 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4423 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4426 .active_pipes = BIT(PIPE_A),
4428 [PIPE_A] = BIT(DBUF_S1),
4432 .active_pipes = BIT(PIPE_B),
4434 [PIPE_B] = BIT(DBUF_S1),
4438 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4440 [PIPE_A] = BIT(DBUF_S1),
4441 [PIPE_B] = BIT(DBUF_S2),
4445 .active_pipes = BIT(PIPE_C),
4447 [PIPE_C] = BIT(DBUF_S2),
4451 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4453 [PIPE_A] = BIT(DBUF_S1),
4454 [PIPE_C] = BIT(DBUF_S2),
4458 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4460 [PIPE_B] = BIT(DBUF_S1),
4461 [PIPE_C] = BIT(DBUF_S2),
4465 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4467 [PIPE_A] = BIT(DBUF_S1),
4468 [PIPE_B] = BIT(DBUF_S1),
4469 [PIPE_C] = BIT(DBUF_S2),
4476 * Table taken from Bspec 49255
4477 * Pipes do have some preferred DBuf slice affinity,
4478 * plus there are some hardcoded requirements on how
4479 * those should be distributed for multipipe scenarios.
4480 * For more DBuf slices algorithm can get even more messy
4481 * and less readable, so decided to use a table almost
4482 * as is from BSpec itself - that way it is at least easier
4483 * to compare, change and check.
4485 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4486 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4489 .active_pipes = BIT(PIPE_A),
4491 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4495 .active_pipes = BIT(PIPE_B),
4497 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4501 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4503 [PIPE_A] = BIT(DBUF_S2),
4504 [PIPE_B] = BIT(DBUF_S1),
4508 .active_pipes = BIT(PIPE_C),
4510 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4514 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4516 [PIPE_A] = BIT(DBUF_S1),
4517 [PIPE_C] = BIT(DBUF_S2),
4521 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4523 [PIPE_B] = BIT(DBUF_S1),
4524 [PIPE_C] = BIT(DBUF_S2),
4528 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4530 [PIPE_A] = BIT(DBUF_S1),
4531 [PIPE_B] = BIT(DBUF_S1),
4532 [PIPE_C] = BIT(DBUF_S2),
4536 .active_pipes = BIT(PIPE_D),
4538 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4542 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4544 [PIPE_A] = BIT(DBUF_S1),
4545 [PIPE_D] = BIT(DBUF_S2),
4549 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4551 [PIPE_B] = BIT(DBUF_S1),
4552 [PIPE_D] = BIT(DBUF_S2),
4556 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4558 [PIPE_A] = BIT(DBUF_S1),
4559 [PIPE_B] = BIT(DBUF_S1),
4560 [PIPE_D] = BIT(DBUF_S2),
4564 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4566 [PIPE_C] = BIT(DBUF_S1),
4567 [PIPE_D] = BIT(DBUF_S2),
4571 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4573 [PIPE_A] = BIT(DBUF_S1),
4574 [PIPE_C] = BIT(DBUF_S2),
4575 [PIPE_D] = BIT(DBUF_S2),
4579 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4581 [PIPE_B] = BIT(DBUF_S1),
4582 [PIPE_C] = BIT(DBUF_S2),
4583 [PIPE_D] = BIT(DBUF_S2),
4587 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4589 [PIPE_A] = BIT(DBUF_S1),
4590 [PIPE_B] = BIT(DBUF_S1),
4591 [PIPE_C] = BIT(DBUF_S2),
4592 [PIPE_D] = BIT(DBUF_S2),
4598 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
4600 .active_pipes = BIT(PIPE_A),
4602 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4606 .active_pipes = BIT(PIPE_B),
4608 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4612 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4614 [PIPE_A] = BIT(DBUF_S1),
4615 [PIPE_B] = BIT(DBUF_S2),
4619 .active_pipes = BIT(PIPE_C),
4621 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4625 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4627 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4628 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4632 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4634 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4635 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4639 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4641 [PIPE_A] = BIT(DBUF_S1),
4642 [PIPE_B] = BIT(DBUF_S2),
4643 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4647 .active_pipes = BIT(PIPE_D),
4649 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4653 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4655 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4656 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4660 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4662 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4663 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4667 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4669 [PIPE_A] = BIT(DBUF_S1),
4670 [PIPE_B] = BIT(DBUF_S2),
4671 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4675 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4677 [PIPE_C] = BIT(DBUF_S3),
4678 [PIPE_D] = BIT(DBUF_S4),
4682 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4684 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4685 [PIPE_C] = BIT(DBUF_S3),
4686 [PIPE_D] = BIT(DBUF_S4),
4690 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4692 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4693 [PIPE_C] = BIT(DBUF_S3),
4694 [PIPE_D] = BIT(DBUF_S4),
4698 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4700 [PIPE_A] = BIT(DBUF_S1),
4701 [PIPE_B] = BIT(DBUF_S2),
4702 [PIPE_C] = BIT(DBUF_S3),
4703 [PIPE_D] = BIT(DBUF_S4),
4709 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
4711 .active_pipes = BIT(PIPE_A),
4713 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4718 .active_pipes = BIT(PIPE_B),
4720 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4725 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4727 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4728 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4732 .active_pipes = BIT(PIPE_C),
4734 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4738 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4740 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4741 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4745 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4747 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4748 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4752 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4754 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4755 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4756 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4760 .active_pipes = BIT(PIPE_D),
4762 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4766 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4768 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4769 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4773 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4775 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4776 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4780 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4782 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4783 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4784 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4788 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4790 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4791 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4795 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4797 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4798 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4799 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4803 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4805 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4806 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4807 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4811 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4813 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4814 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4815 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4816 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4823 static bool check_mbus_joined(u8 active_pipes,
4824 const struct dbuf_slice_conf_entry *dbuf_slices)
4828 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4829 if (dbuf_slices[i].active_pipes == active_pipes)
4830 return dbuf_slices[i].join_mbus;
4835 static bool adlp_check_mbus_joined(u8 active_pipes)
4837 return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
4840 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4841 const struct dbuf_slice_conf_entry *dbuf_slices)
4845 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4846 if (dbuf_slices[i].active_pipes == active_pipes)
4847 return dbuf_slices[i].dbuf_mask[pipe];
4853 * This function finds an entry with same enabled pipe configuration and
4854 * returns correspondent DBuf slice mask as stated in BSpec for particular
4857 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4860 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4861 * required calculating "pipe ratio" in order to determine
4862 * if one or two slices can be used for single pipe configurations
4863 * as additional constraint to the existing table.
4864 * However based on recent info, it should be not "pipe ratio"
4865 * but rather ratio between pixel_rate and cdclk with additional
4866 * constants, so for now we are using only table until this is
4867 * clarified. Also this is the reason why crtc_state param is
4868 * still here - we will need it once those additional constraints
4871 return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4874 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4876 return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4879 static u32 adlp_compute_dbuf_slices(enum pipe pipe, u32 active_pipes)
4881 return compute_dbuf_slices(pipe, active_pipes, adlp_allowed_dbufs);
4884 static u32 dg2_compute_dbuf_slices(enum pipe pipe, u32 active_pipes)
4886 return compute_dbuf_slices(pipe, active_pipes, dg2_allowed_dbufs);
4889 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes)
4891 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4892 enum pipe pipe = crtc->pipe;
4894 if (IS_DG2(dev_priv))
4895 return dg2_compute_dbuf_slices(pipe, active_pipes);
4896 else if (IS_ALDERLAKE_P(dev_priv))
4897 return adlp_compute_dbuf_slices(pipe, active_pipes);
4898 else if (DISPLAY_VER(dev_priv) == 12)
4899 return tgl_compute_dbuf_slices(pipe, active_pipes);
4900 else if (DISPLAY_VER(dev_priv) == 11)
4901 return icl_compute_dbuf_slices(pipe, active_pipes);
4903 * For anything else just return one slice yet.
4904 * Should be extended for other platforms.
4906 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4910 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4911 const struct intel_plane_state *plane_state,
4914 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4915 const struct drm_framebuffer *fb = plane_state->hw.fb;
4917 u32 width = 0, height = 0;
4918 uint_fixed_16_16_t down_scale_amount;
4921 if (!plane_state->uapi.visible)
4924 if (plane->id == PLANE_CURSOR)
4927 if (color_plane == 1 &&
4928 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4932 * Src coordinates are already rotated by 270 degrees for
4933 * the 90/270 degree plane rotation cases (to match the
4934 * GTT mapping), hence no need to account for rotation here.
4936 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4937 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4939 /* UV plane does 1/2 pixel sub-sampling */
4940 if (color_plane == 1) {
4945 data_rate = width * height;
4947 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4949 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4951 rate *= fb->format->cpp[color_plane];
4956 skl_get_total_relative_data_rate(struct intel_atomic_state *state,
4957 struct intel_crtc *crtc)
4959 struct intel_crtc_state *crtc_state =
4960 intel_atomic_get_new_crtc_state(state, crtc);
4961 const struct intel_plane_state *plane_state;
4962 struct intel_plane *plane;
4963 u64 total_data_rate = 0;
4964 enum plane_id plane_id;
4967 /* Calculate and cache data rate for each plane */
4968 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4969 if (plane->pipe != crtc->pipe)
4972 plane_id = plane->id;
4975 crtc_state->plane_data_rate[plane_id] =
4976 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4979 crtc_state->uv_plane_data_rate[plane_id] =
4980 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4983 for_each_plane_id_on_crtc(crtc, plane_id) {
4984 total_data_rate += crtc_state->plane_data_rate[plane_id];
4985 total_data_rate += crtc_state->uv_plane_data_rate[plane_id];
4988 return total_data_rate;
4992 icl_get_total_relative_data_rate(struct intel_atomic_state *state,
4993 struct intel_crtc *crtc)
4995 struct intel_crtc_state *crtc_state =
4996 intel_atomic_get_new_crtc_state(state, crtc);
4997 const struct intel_plane_state *plane_state;
4998 struct intel_plane *plane;
4999 u64 total_data_rate = 0;
5000 enum plane_id plane_id;
5003 /* Calculate and cache data rate for each plane */
5004 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5005 if (plane->pipe != crtc->pipe)
5008 plane_id = plane->id;
5010 if (!plane_state->planar_linked_plane) {
5011 crtc_state->plane_data_rate[plane_id] =
5012 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5014 enum plane_id y_plane_id;
5017 * The slave plane might not iterate in
5018 * intel_atomic_crtc_state_for_each_plane_state(),
5019 * and needs the master plane state which may be
5020 * NULL if we try get_new_plane_state(), so we
5021 * always calculate from the master.
5023 if (plane_state->planar_slave)
5026 /* Y plane rate is calculated on the slave */
5027 y_plane_id = plane_state->planar_linked_plane->id;
5028 crtc_state->plane_data_rate[y_plane_id] =
5029 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5031 crtc_state->plane_data_rate[plane_id] =
5032 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
5036 for_each_plane_id_on_crtc(crtc, plane_id)
5037 total_data_rate += crtc_state->plane_data_rate[plane_id];
5039 return total_data_rate;
5042 const struct skl_wm_level *
5043 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
5044 enum plane_id plane_id,
5047 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5049 if (level == 0 && pipe_wm->use_sagv_wm)
5050 return &wm->sagv.wm0;
5052 return &wm->wm[level];
5055 const struct skl_wm_level *
5056 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
5057 enum plane_id plane_id)
5059 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5061 if (pipe_wm->use_sagv_wm)
5062 return &wm->sagv.trans_wm;
5064 return &wm->trans_wm;
5068 * We only disable the watermarks for each plane if
5069 * they exceed the ddb allocation of said plane. This
5070 * is done so that we don't end up touching cursor
5071 * watermarks needlessly when some other plane reduces
5072 * our max possible watermark level.
5074 * Bspec has this to say about the PLANE_WM enable bit:
5075 * "All the watermarks at this level for all enabled
5076 * planes must be enabled before the level will be used."
5077 * So this is actually safe to do.
5080 skl_check_wm_level(struct skl_wm_level *wm, u64 total)
5082 if (wm->min_ddb_alloc > total)
5083 memset(wm, 0, sizeof(*wm));
5087 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
5088 u64 total, u64 uv_total)
5090 if (wm->min_ddb_alloc > total ||
5091 uv_wm->min_ddb_alloc > uv_total) {
5092 memset(wm, 0, sizeof(*wm));
5093 memset(uv_wm, 0, sizeof(*uv_wm));
5098 skl_allocate_plane_ddb(struct intel_atomic_state *state,
5099 struct intel_crtc *crtc)
5101 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5102 struct intel_crtc_state *crtc_state =
5103 intel_atomic_get_new_crtc_state(state, crtc);
5104 const struct intel_dbuf_state *dbuf_state =
5105 intel_atomic_get_new_dbuf_state(state);
5106 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
5107 int num_active = hweight8(dbuf_state->active_pipes);
5108 u16 alloc_size, start = 0;
5109 u16 total[I915_MAX_PLANES] = {};
5110 u16 uv_total[I915_MAX_PLANES] = {};
5111 u64 total_data_rate;
5112 enum plane_id plane_id;
5116 /* Clear the partitioning for disabled planes. */
5117 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
5118 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
5120 if (!crtc_state->hw.active)
5123 if (DISPLAY_VER(dev_priv) >= 11)
5125 icl_get_total_relative_data_rate(state, crtc);
5128 skl_get_total_relative_data_rate(state, crtc);
5130 alloc_size = skl_ddb_entry_size(alloc);
5131 if (alloc_size == 0)
5134 /* Allocate fixed number of blocks for cursor. */
5135 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
5136 alloc_size -= total[PLANE_CURSOR];
5137 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
5138 alloc->end - total[PLANE_CURSOR];
5139 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
5141 if (total_data_rate == 0)
5145 * Find the highest watermark level for which we can satisfy the block
5146 * requirement of active planes.
5148 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
5150 for_each_plane_id_on_crtc(crtc, plane_id) {
5151 const struct skl_plane_wm *wm =
5152 &crtc_state->wm.skl.optimal.planes[plane_id];
5154 if (plane_id == PLANE_CURSOR) {
5155 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
5156 drm_WARN_ON(&dev_priv->drm,
5157 wm->wm[level].min_ddb_alloc != U16_MAX);
5164 blocks += wm->wm[level].min_ddb_alloc;
5165 blocks += wm->uv_wm[level].min_ddb_alloc;
5168 if (blocks <= alloc_size) {
5169 alloc_size -= blocks;
5175 drm_dbg_kms(&dev_priv->drm,
5176 "Requested display configuration exceeds system DDB limitations");
5177 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
5178 blocks, alloc_size);
5183 * Grant each plane the blocks it requires at the highest achievable
5184 * watermark level, plus an extra share of the leftover blocks
5185 * proportional to its relative data rate.
5187 for_each_plane_id_on_crtc(crtc, plane_id) {
5188 const struct skl_plane_wm *wm =
5189 &crtc_state->wm.skl.optimal.planes[plane_id];
5193 if (plane_id == PLANE_CURSOR)
5197 * We've accounted for all active planes; remaining planes are
5200 if (total_data_rate == 0)
5203 rate = crtc_state->plane_data_rate[plane_id];
5204 extra = min_t(u16, alloc_size,
5205 DIV64_U64_ROUND_UP(alloc_size * rate,
5207 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
5208 alloc_size -= extra;
5209 total_data_rate -= rate;
5211 if (total_data_rate == 0)
5214 rate = crtc_state->uv_plane_data_rate[plane_id];
5215 extra = min_t(u16, alloc_size,
5216 DIV64_U64_ROUND_UP(alloc_size * rate,
5218 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
5219 alloc_size -= extra;
5220 total_data_rate -= rate;
5222 drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
5224 /* Set the actual DDB start/end points for each plane */
5225 start = alloc->start;
5226 for_each_plane_id_on_crtc(crtc, plane_id) {
5227 struct skl_ddb_entry *plane_alloc =
5228 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5229 struct skl_ddb_entry *uv_plane_alloc =
5230 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5232 if (plane_id == PLANE_CURSOR)
5235 /* Gen11+ uses a separate plane for UV watermarks */
5236 drm_WARN_ON(&dev_priv->drm,
5237 DISPLAY_VER(dev_priv) >= 11 && uv_total[plane_id]);
5239 /* Leave disabled planes at (0,0) */
5240 if (total[plane_id]) {
5241 plane_alloc->start = start;
5242 start += total[plane_id];
5243 plane_alloc->end = start;
5246 if (uv_total[plane_id]) {
5247 uv_plane_alloc->start = start;
5248 start += uv_total[plane_id];
5249 uv_plane_alloc->end = start;
5254 * When we calculated watermark values we didn't know how high
5255 * of a level we'd actually be able to hit, so we just marked
5256 * all levels as "enabled." Go back now and disable the ones
5257 * that aren't actually possible.
5259 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
5260 for_each_plane_id_on_crtc(crtc, plane_id) {
5261 struct skl_plane_wm *wm =
5262 &crtc_state->wm.skl.optimal.planes[plane_id];
5264 skl_check_nv12_wm_level(&wm->wm[level], &wm->uv_wm[level],
5265 total[plane_id], uv_total[plane_id]);
5268 * Wa_1408961008:icl, ehl
5269 * Underruns with WM1+ disabled
5271 if (DISPLAY_VER(dev_priv) == 11 &&
5272 level == 1 && wm->wm[0].enable) {
5273 wm->wm[level].blocks = wm->wm[0].blocks;
5274 wm->wm[level].lines = wm->wm[0].lines;
5275 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5281 * Go back and disable the transition and SAGV watermarks
5282 * if it turns out we don't have enough DDB blocks for them.
5284 for_each_plane_id_on_crtc(crtc, plane_id) {
5285 struct skl_plane_wm *wm =
5286 &crtc_state->wm.skl.optimal.planes[plane_id];
5288 skl_check_wm_level(&wm->trans_wm, total[plane_id]);
5289 skl_check_wm_level(&wm->sagv.wm0, total[plane_id]);
5290 skl_check_wm_level(&wm->sagv.trans_wm, total[plane_id]);
5297 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5298 * for the read latency) and cpp should always be <= 8, so that
5299 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5300 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5302 static uint_fixed_16_16_t
5303 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5304 u8 cpp, u32 latency, u32 dbuf_block_size)
5306 u32 wm_intermediate_val;
5307 uint_fixed_16_16_t ret;
5310 return FP_16_16_MAX;
5312 wm_intermediate_val = latency * pixel_rate * cpp;
5313 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5315 if (DISPLAY_VER(dev_priv) >= 10)
5316 ret = add_fixed16_u32(ret, 1);
5321 static uint_fixed_16_16_t
5322 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5323 uint_fixed_16_16_t plane_blocks_per_line)
5325 u32 wm_intermediate_val;
5326 uint_fixed_16_16_t ret;
5329 return FP_16_16_MAX;
5331 wm_intermediate_val = latency * pixel_rate;
5332 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5333 pipe_htotal * 1000);
5334 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5338 static uint_fixed_16_16_t
5339 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5341 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5344 uint_fixed_16_16_t linetime_us;
5346 if (!crtc_state->hw.active)
5347 return u32_to_fixed16(0);
5349 pixel_rate = crtc_state->pixel_rate;
5351 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5352 return u32_to_fixed16(0);
5354 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5355 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5361 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5362 int width, const struct drm_format_info *format,
5363 u64 modifier, unsigned int rotation,
5364 u32 plane_pixel_rate, struct skl_wm_params *wp,
5367 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5368 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5371 /* only planar format has two planes */
5372 if (color_plane == 1 &&
5373 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5374 drm_dbg_kms(&dev_priv->drm,
5375 "Non planar format have single plane\n");
5379 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5380 modifier == I915_FORMAT_MOD_Yf_TILED ||
5381 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5382 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5383 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5384 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5385 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5386 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5389 if (color_plane == 1 && wp->is_planar)
5392 wp->cpp = format->cpp[color_plane];
5393 wp->plane_pixel_rate = plane_pixel_rate;
5395 if (DISPLAY_VER(dev_priv) >= 11 &&
5396 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5397 wp->dbuf_block_size = 256;
5399 wp->dbuf_block_size = 512;
5401 if (drm_rotation_90_or_270(rotation)) {
5404 wp->y_min_scanlines = 16;
5407 wp->y_min_scanlines = 8;
5410 wp->y_min_scanlines = 4;
5413 MISSING_CASE(wp->cpp);
5417 wp->y_min_scanlines = 4;
5420 if (skl_needs_memory_bw_wa(dev_priv))
5421 wp->y_min_scanlines *= 2;
5423 wp->plane_bytes_per_line = wp->width * wp->cpp;
5425 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5426 wp->y_min_scanlines,
5427 wp->dbuf_block_size);
5429 if (DISPLAY_VER(dev_priv) >= 10)
5432 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5433 wp->y_min_scanlines);
5435 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5436 wp->dbuf_block_size);
5438 if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5441 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5444 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5445 wp->plane_blocks_per_line);
5447 wp->linetime_us = fixed16_to_u32_round_up(
5448 intel_get_linetime_us(crtc_state));
5454 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5455 const struct intel_plane_state *plane_state,
5456 struct skl_wm_params *wp, int color_plane)
5458 const struct drm_framebuffer *fb = plane_state->hw.fb;
5462 * Src coordinates are already rotated by 270 degrees for
5463 * the 90/270 degree plane rotation cases (to match the
5464 * GTT mapping), hence no need to account for rotation here.
5466 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5468 return skl_compute_wm_params(crtc_state, width,
5469 fb->format, fb->modifier,
5470 plane_state->hw.rotation,
5471 intel_plane_pixel_rate(crtc_state, plane_state),
5475 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5477 if (DISPLAY_VER(dev_priv) >= 10)
5480 /* The number of lines are ignored for the level 0 watermark. */
5484 static int skl_wm_max_lines(struct drm_i915_private *dev_priv)
5486 if (DISPLAY_VER(dev_priv) >= 13)
5492 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5494 unsigned int latency,
5495 const struct skl_wm_params *wp,
5496 const struct skl_wm_level *result_prev,
5497 struct skl_wm_level *result /* out */)
5499 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5500 uint_fixed_16_16_t method1, method2;
5501 uint_fixed_16_16_t selected_result;
5502 u32 blocks, lines, min_ddb_alloc = 0;
5506 result->min_ddb_alloc = U16_MAX;
5511 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5512 * Display WA #1141: kbl,cfl
5514 if ((IS_KABYLAKE(dev_priv) ||
5515 IS_COFFEELAKE(dev_priv) ||
5516 IS_COMETLAKE(dev_priv)) &&
5517 dev_priv->ipc_enabled)
5520 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5523 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5524 wp->cpp, latency, wp->dbuf_block_size);
5525 method2 = skl_wm_method2(wp->plane_pixel_rate,
5526 crtc_state->hw.pipe_mode.crtc_htotal,
5528 wp->plane_blocks_per_line);
5531 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5533 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5534 wp->dbuf_block_size < 1) &&
5535 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5536 selected_result = method2;
5537 } else if (latency >= wp->linetime_us) {
5538 if (DISPLAY_VER(dev_priv) == 9)
5539 selected_result = min_fixed16(method1, method2);
5541 selected_result = method2;
5543 selected_result = method1;
5547 blocks = fixed16_to_u32_round_up(selected_result) + 1;
5548 lines = div_round_up_fixed16(selected_result,
5549 wp->plane_blocks_per_line);
5551 if (DISPLAY_VER(dev_priv) == 9) {
5552 /* Display WA #1125: skl,bxt,kbl */
5553 if (level == 0 && wp->rc_surface)
5554 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5556 /* Display WA #1126: skl,bxt,kbl */
5557 if (level >= 1 && level <= 7) {
5559 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5560 lines += wp->y_min_scanlines;
5566 * Make sure result blocks for higher latency levels are
5567 * atleast as high as level below the current level.
5568 * Assumption in DDB algorithm optimization for special
5569 * cases. Also covers Display WA #1125 for RC.
5571 if (result_prev->blocks > blocks)
5572 blocks = result_prev->blocks;
5576 if (DISPLAY_VER(dev_priv) >= 11) {
5580 if (lines % wp->y_min_scanlines == 0)
5581 extra_lines = wp->y_min_scanlines;
5583 extra_lines = wp->y_min_scanlines * 2 -
5584 lines % wp->y_min_scanlines;
5586 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5587 wp->plane_blocks_per_line);
5589 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5593 if (!skl_wm_has_lines(dev_priv, level))
5596 if (lines > skl_wm_max_lines(dev_priv)) {
5598 result->min_ddb_alloc = U16_MAX;
5603 * If lines is valid, assume we can use this watermark level
5604 * for now. We'll come back and disable it after we calculate the
5605 * DDB allocation if it turns out we don't actually have enough
5606 * blocks to satisfy it.
5608 result->blocks = blocks;
5609 result->lines = lines;
5610 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5611 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
5612 result->enable = true;
5614 if (DISPLAY_VER(dev_priv) < 12)
5615 result->can_sagv = latency >= dev_priv->sagv_block_time_us;
5619 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5620 const struct skl_wm_params *wm_params,
5621 struct skl_wm_level *levels)
5623 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5624 int level, max_level = ilk_wm_max_level(dev_priv);
5625 struct skl_wm_level *result_prev = &levels[0];
5627 for (level = 0; level <= max_level; level++) {
5628 struct skl_wm_level *result = &levels[level];
5629 unsigned int latency = dev_priv->wm.skl_latency[level];
5631 skl_compute_plane_wm(crtc_state, level, latency,
5632 wm_params, result_prev, result);
5634 result_prev = result;
5638 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5639 const struct skl_wm_params *wm_params,
5640 struct skl_plane_wm *plane_wm)
5642 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5643 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5644 struct skl_wm_level *levels = plane_wm->wm;
5645 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5647 skl_compute_plane_wm(crtc_state, 0, latency,
5648 wm_params, &levels[0],
5652 static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5653 struct skl_wm_level *trans_wm,
5654 const struct skl_wm_level *wm0,
5655 const struct skl_wm_params *wp)
5657 u16 trans_min, trans_amount, trans_y_tile_min;
5658 u16 wm0_blocks, trans_offset, blocks;
5660 /* Transition WM don't make any sense if ipc is disabled */
5661 if (!dev_priv->ipc_enabled)
5665 * WaDisableTWM:skl,kbl,cfl,bxt
5666 * Transition WM are not recommended by HW team for GEN9
5668 if (DISPLAY_VER(dev_priv) == 9)
5671 if (DISPLAY_VER(dev_priv) >= 11)
5676 /* Display WA #1140: glk,cnl */
5677 if (DISPLAY_VER(dev_priv) == 10)
5680 trans_amount = 10; /* This is configurable amount */
5682 trans_offset = trans_min + trans_amount;
5685 * The spec asks for Selected Result Blocks for wm0 (the real value),
5686 * not Result Blocks (the integer value). Pay attention to the capital
5687 * letters. The value wm_l0->blocks is actually Result Blocks, but
5688 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5689 * and since we later will have to get the ceiling of the sum in the
5690 * transition watermarks calculation, we can just pretend Selected
5691 * Result Blocks is Result Blocks minus 1 and it should work for the
5692 * current platforms.
5694 wm0_blocks = wm0->blocks - 1;
5698 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5699 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5701 blocks = wm0_blocks + trans_offset;
5706 * Just assume we can enable the transition watermark. After
5707 * computing the DDB we'll come back and disable it if that
5708 * assumption turns out to be false.
5710 trans_wm->blocks = blocks;
5711 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
5712 trans_wm->enable = true;
5715 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5716 const struct intel_plane_state *plane_state,
5717 enum plane_id plane_id, int color_plane)
5719 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5720 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5721 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5722 struct skl_wm_params wm_params;
5725 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5726 &wm_params, color_plane);
5730 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5732 skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5733 &wm->wm[0], &wm_params);
5735 if (DISPLAY_VER(dev_priv) >= 12) {
5736 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5738 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5739 &wm->sagv.wm0, &wm_params);
5745 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5746 const struct intel_plane_state *plane_state,
5747 enum plane_id plane_id)
5749 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5750 struct skl_wm_params wm_params;
5753 wm->is_planar = true;
5755 /* uv plane watermarks must also be validated for NV12/Planar */
5756 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5761 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5766 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5767 const struct intel_plane_state *plane_state)
5769 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5770 enum plane_id plane_id = plane->id;
5771 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5772 const struct drm_framebuffer *fb = plane_state->hw.fb;
5775 memset(wm, 0, sizeof(*wm));
5777 if (!intel_wm_plane_visible(crtc_state, plane_state))
5780 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5785 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5786 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5795 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5796 const struct intel_plane_state *plane_state)
5798 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5799 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5800 enum plane_id plane_id = plane->id;
5801 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5804 /* Watermarks calculated in master */
5805 if (plane_state->planar_slave)
5808 memset(wm, 0, sizeof(*wm));
5810 if (plane_state->planar_linked_plane) {
5811 const struct drm_framebuffer *fb = plane_state->hw.fb;
5812 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5814 drm_WARN_ON(&dev_priv->drm,
5815 !intel_wm_plane_visible(crtc_state, plane_state));
5816 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5817 fb->format->num_planes == 1);
5819 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5824 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5828 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5829 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5838 static int skl_build_pipe_wm(struct intel_atomic_state *state,
5839 struct intel_crtc *crtc)
5841 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5842 struct intel_crtc_state *crtc_state =
5843 intel_atomic_get_new_crtc_state(state, crtc);
5844 const struct intel_plane_state *plane_state;
5845 struct intel_plane *plane;
5848 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5850 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
5851 * instead but we don't populate that correctly for NV12 Y
5852 * planes so for now hack this.
5854 if (plane->pipe != crtc->pipe)
5857 if (DISPLAY_VER(dev_priv) >= 11)
5858 ret = icl_build_plane_wm(crtc_state, plane_state);
5860 ret = skl_build_plane_wm(crtc_state, plane_state);
5865 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5870 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5872 const struct skl_ddb_entry *entry)
5875 intel_de_write_fw(dev_priv, reg,
5876 (entry->end - 1) << 16 | entry->start);
5878 intel_de_write_fw(dev_priv, reg, 0);
5881 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5883 const struct skl_wm_level *level)
5889 if (level->ignore_lines)
5890 val |= PLANE_WM_IGNORE_LINES;
5891 val |= level->blocks;
5892 val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
5894 intel_de_write_fw(dev_priv, reg, val);
5897 void skl_write_plane_wm(struct intel_plane *plane,
5898 const struct intel_crtc_state *crtc_state)
5900 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5901 int level, max_level = ilk_wm_max_level(dev_priv);
5902 enum plane_id plane_id = plane->id;
5903 enum pipe pipe = plane->pipe;
5904 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5905 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5906 const struct skl_ddb_entry *ddb_y =
5907 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5908 const struct skl_ddb_entry *ddb_uv =
5909 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5911 for (level = 0; level <= max_level; level++)
5912 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5913 skl_plane_wm_level(pipe_wm, plane_id, level));
5915 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5916 skl_plane_trans_wm(pipe_wm, plane_id));
5918 if (HAS_HW_SAGV_WM(dev_priv)) {
5919 skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id),
5921 skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id),
5922 &wm->sagv.trans_wm);
5925 if (DISPLAY_VER(dev_priv) >= 11) {
5926 skl_ddb_entry_write(dev_priv,
5927 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5932 swap(ddb_y, ddb_uv);
5934 skl_ddb_entry_write(dev_priv,
5935 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5936 skl_ddb_entry_write(dev_priv,
5937 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5940 void skl_write_cursor_wm(struct intel_plane *plane,
5941 const struct intel_crtc_state *crtc_state)
5943 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5944 int level, max_level = ilk_wm_max_level(dev_priv);
5945 enum plane_id plane_id = plane->id;
5946 enum pipe pipe = plane->pipe;
5947 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5948 const struct skl_ddb_entry *ddb =
5949 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5951 for (level = 0; level <= max_level; level++)
5952 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5953 skl_plane_wm_level(pipe_wm, plane_id, level));
5955 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5956 skl_plane_trans_wm(pipe_wm, plane_id));
5958 if (HAS_HW_SAGV_WM(dev_priv)) {
5959 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5961 skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe),
5963 skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe),
5964 &wm->sagv.trans_wm);
5967 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5970 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5971 const struct skl_wm_level *l2)
5973 return l1->enable == l2->enable &&
5974 l1->ignore_lines == l2->ignore_lines &&
5975 l1->lines == l2->lines &&
5976 l1->blocks == l2->blocks;
5979 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5980 const struct skl_plane_wm *wm1,
5981 const struct skl_plane_wm *wm2)
5983 int level, max_level = ilk_wm_max_level(dev_priv);
5985 for (level = 0; level <= max_level; level++) {
5987 * We don't check uv_wm as the hardware doesn't actually
5988 * use it. It only gets used for calculating the required
5991 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5995 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
5996 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
5997 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
6000 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
6001 const struct skl_ddb_entry *b)
6003 return a->start < b->end && b->start < a->end;
6006 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
6007 const struct skl_ddb_entry *b)
6009 if (a->end && b->end) {
6010 a->start = min(a->start, b->start);
6011 a->end = max(a->end, b->end);
6012 } else if (b->end) {
6013 a->start = b->start;
6018 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
6019 const struct skl_ddb_entry *entries,
6020 int num_entries, int ignore_idx)
6024 for (i = 0; i < num_entries; i++) {
6025 if (i != ignore_idx &&
6026 skl_ddb_entries_overlap(ddb, &entries[i]))
6034 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
6035 struct intel_crtc_state *new_crtc_state)
6037 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
6038 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6039 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6040 struct intel_plane *plane;
6042 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6043 struct intel_plane_state *plane_state;
6044 enum plane_id plane_id = plane->id;
6046 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
6047 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
6048 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
6049 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
6052 plane_state = intel_atomic_get_plane_state(state, plane);
6053 if (IS_ERR(plane_state))
6054 return PTR_ERR(plane_state);
6056 new_crtc_state->update_planes |= BIT(plane_id);
6062 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
6064 struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
6069 * FIXME: For now we always enable slice S1 as per
6070 * the Bspec display initialization sequence.
6072 enabled_slices = BIT(DBUF_S1);
6074 for_each_pipe(dev_priv, pipe)
6075 enabled_slices |= dbuf_state->slices[pipe];
6077 return enabled_slices;
6081 skl_compute_ddb(struct intel_atomic_state *state)
6083 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6084 const struct intel_dbuf_state *old_dbuf_state;
6085 struct intel_dbuf_state *new_dbuf_state = NULL;
6086 const struct intel_crtc_state *old_crtc_state;
6087 struct intel_crtc_state *new_crtc_state;
6088 struct intel_crtc *crtc;
6091 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6092 new_dbuf_state = intel_atomic_get_dbuf_state(state);
6093 if (IS_ERR(new_dbuf_state))
6094 return PTR_ERR(new_dbuf_state);
6096 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6100 if (!new_dbuf_state)
6103 new_dbuf_state->active_pipes =
6104 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6106 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
6107 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6112 for_each_intel_crtc(&dev_priv->drm, crtc) {
6113 enum pipe pipe = crtc->pipe;
6115 new_dbuf_state->slices[pipe] =
6116 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes);
6118 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
6121 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6126 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
6128 if (IS_ALDERLAKE_P(dev_priv))
6129 new_dbuf_state->joined_mbus = adlp_check_mbus_joined(new_dbuf_state->active_pipes);
6131 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
6132 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6133 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
6137 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6138 /* TODO: Implement vblank synchronized MBUS joining changes */
6139 ret = intel_modeset_all_pipes(state);
6144 drm_dbg_kms(&dev_priv->drm,
6145 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
6146 old_dbuf_state->enabled_slices,
6147 new_dbuf_state->enabled_slices,
6148 INTEL_INFO(dev_priv)->dbuf.slice_mask,
6149 yesno(old_dbuf_state->joined_mbus),
6150 yesno(new_dbuf_state->joined_mbus));
6153 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6154 enum pipe pipe = crtc->pipe;
6156 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
6158 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
6161 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6166 for_each_intel_crtc(&dev_priv->drm, crtc) {
6167 ret = skl_crtc_allocate_ddb(state, crtc);
6172 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6173 new_crtc_state, i) {
6174 ret = skl_allocate_plane_ddb(state, crtc);
6178 ret = skl_ddb_add_affected_planes(old_crtc_state,
6187 static char enast(bool enable)
6189 return enable ? '*' : ' ';
6193 skl_print_wm_changes(struct intel_atomic_state *state)
6195 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6196 const struct intel_crtc_state *old_crtc_state;
6197 const struct intel_crtc_state *new_crtc_state;
6198 struct intel_plane *plane;
6199 struct intel_crtc *crtc;
6202 if (!drm_debug_enabled(DRM_UT_KMS))
6205 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6206 new_crtc_state, i) {
6207 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
6209 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
6210 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
6212 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6213 enum plane_id plane_id = plane->id;
6214 const struct skl_ddb_entry *old, *new;
6216 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
6217 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
6219 if (skl_ddb_entry_equal(old, new))
6222 drm_dbg_kms(&dev_priv->drm,
6223 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
6224 plane->base.base.id, plane->base.name,
6225 old->start, old->end, new->start, new->end,
6226 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
6229 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6230 enum plane_id plane_id = plane->id;
6231 const struct skl_plane_wm *old_wm, *new_wm;
6233 old_wm = &old_pipe_wm->planes[plane_id];
6234 new_wm = &new_pipe_wm->planes[plane_id];
6236 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
6239 drm_dbg_kms(&dev_priv->drm,
6240 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
6241 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
6242 plane->base.base.id, plane->base.name,
6243 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
6244 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
6245 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
6246 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
6247 enast(old_wm->trans_wm.enable),
6248 enast(old_wm->sagv.wm0.enable),
6249 enast(old_wm->sagv.trans_wm.enable),
6250 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
6251 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
6252 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
6253 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
6254 enast(new_wm->trans_wm.enable),
6255 enast(new_wm->sagv.wm0.enable),
6256 enast(new_wm->sagv.trans_wm.enable));
6258 drm_dbg_kms(&dev_priv->drm,
6259 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
6260 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
6261 plane->base.base.id, plane->base.name,
6262 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
6263 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
6264 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
6265 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
6266 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
6267 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
6268 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
6269 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
6270 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
6271 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
6272 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
6273 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
6274 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
6275 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
6276 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
6277 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
6278 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
6279 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
6280 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
6281 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
6282 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
6283 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
6285 drm_dbg_kms(&dev_priv->drm,
6286 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6287 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6288 plane->base.base.id, plane->base.name,
6289 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
6290 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
6291 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
6292 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
6293 old_wm->trans_wm.blocks,
6294 old_wm->sagv.wm0.blocks,
6295 old_wm->sagv.trans_wm.blocks,
6296 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
6297 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
6298 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
6299 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
6300 new_wm->trans_wm.blocks,
6301 new_wm->sagv.wm0.blocks,
6302 new_wm->sagv.trans_wm.blocks);
6304 drm_dbg_kms(&dev_priv->drm,
6305 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6306 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6307 plane->base.base.id, plane->base.name,
6308 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
6309 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
6310 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
6311 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
6312 old_wm->trans_wm.min_ddb_alloc,
6313 old_wm->sagv.wm0.min_ddb_alloc,
6314 old_wm->sagv.trans_wm.min_ddb_alloc,
6315 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
6316 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
6317 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
6318 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6319 new_wm->trans_wm.min_ddb_alloc,
6320 new_wm->sagv.wm0.min_ddb_alloc,
6321 new_wm->sagv.trans_wm.min_ddb_alloc);
6326 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
6327 const struct skl_pipe_wm *old_pipe_wm,
6328 const struct skl_pipe_wm *new_pipe_wm)
6330 struct drm_i915_private *i915 = to_i915(plane->base.dev);
6331 int level, max_level = ilk_wm_max_level(i915);
6333 for (level = 0; level <= max_level; level++) {
6335 * We don't check uv_wm as the hardware doesn't actually
6336 * use it. It only gets used for calculating the required
6339 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
6340 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6344 if (HAS_HW_SAGV_WM(i915)) {
6345 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
6346 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
6348 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
6349 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
6353 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6354 skl_plane_trans_wm(new_pipe_wm, plane->id));
6358 * To make sure the cursor watermark registers are always consistent
6359 * with our computed state the following scenario needs special
6363 * 2. move cursor entirely offscreen
6366 * Step 2. does call .disable_plane() but does not zero the watermarks
6367 * (since we consider an offscreen cursor still active for the purposes
6368 * of watermarks). Step 3. would not normally call .disable_plane()
6369 * because the actual plane visibility isn't changing, and we don't
6370 * deallocate the cursor ddb until the pipe gets disabled. So we must
6371 * force step 3. to call .disable_plane() to update the watermark
6372 * registers properly.
6374 * Other planes do not suffer from this issues as their watermarks are
6375 * calculated based on the actual plane visibility. The only time this
6376 * can trigger for the other planes is during the initial readout as the
6377 * default value of the watermarks registers is not zero.
6379 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6380 struct intel_crtc *crtc)
6382 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6383 const struct intel_crtc_state *old_crtc_state =
6384 intel_atomic_get_old_crtc_state(state, crtc);
6385 struct intel_crtc_state *new_crtc_state =
6386 intel_atomic_get_new_crtc_state(state, crtc);
6387 struct intel_plane *plane;
6389 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6390 struct intel_plane_state *plane_state;
6391 enum plane_id plane_id = plane->id;
6394 * Force a full wm update for every plane on modeset.
6395 * Required because the reset value of the wm registers
6396 * is non-zero, whereas we want all disabled planes to
6397 * have zero watermarks. So if we turn off the relevant
6398 * power well the hardware state will go out of sync
6399 * with the software state.
6401 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6402 skl_plane_selected_wm_equals(plane,
6403 &old_crtc_state->wm.skl.optimal,
6404 &new_crtc_state->wm.skl.optimal))
6407 plane_state = intel_atomic_get_plane_state(state, plane);
6408 if (IS_ERR(plane_state))
6409 return PTR_ERR(plane_state);
6411 new_crtc_state->update_planes |= BIT(plane_id);
6418 skl_compute_wm(struct intel_atomic_state *state)
6420 struct intel_crtc *crtc;
6421 struct intel_crtc_state *new_crtc_state;
6424 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6425 ret = skl_build_pipe_wm(state, crtc);
6430 ret = skl_compute_ddb(state);
6434 ret = intel_compute_sagv_mask(state);
6439 * skl_compute_ddb() will have adjusted the final watermarks
6440 * based on how much ddb is available. Now we can actually
6441 * check if the final watermarks changed.
6443 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6444 ret = skl_wm_add_affected_planes(state, crtc);
6449 skl_print_wm_changes(state);
6454 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6455 struct intel_wm_config *config)
6457 struct intel_crtc *crtc;
6459 /* Compute the currently _active_ config */
6460 for_each_intel_crtc(&dev_priv->drm, crtc) {
6461 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6463 if (!wm->pipe_enabled)
6466 config->sprites_enabled |= wm->sprites_enabled;
6467 config->sprites_scaled |= wm->sprites_scaled;
6468 config->num_pipes_active++;
6472 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6474 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6475 struct ilk_wm_maximums max;
6476 struct intel_wm_config config = {};
6477 struct ilk_wm_values results = {};
6478 enum intel_ddb_partitioning partitioning;
6480 ilk_compute_wm_config(dev_priv, &config);
6482 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6483 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6485 /* 5/6 split only in single pipe config on IVB+ */
6486 if (DISPLAY_VER(dev_priv) >= 7 &&
6487 config.num_pipes_active == 1 && config.sprites_enabled) {
6488 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6489 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6491 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6493 best_lp_wm = &lp_wm_1_2;
6496 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6497 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6499 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6501 ilk_write_wm_values(dev_priv, &results);
6504 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6505 struct intel_crtc *crtc)
6507 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6508 const struct intel_crtc_state *crtc_state =
6509 intel_atomic_get_new_crtc_state(state, crtc);
6511 mutex_lock(&dev_priv->wm.wm_mutex);
6512 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6513 ilk_program_watermarks(dev_priv);
6514 mutex_unlock(&dev_priv->wm.wm_mutex);
6517 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6518 struct intel_crtc *crtc)
6520 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6521 const struct intel_crtc_state *crtc_state =
6522 intel_atomic_get_new_crtc_state(state, crtc);
6524 if (!crtc_state->wm.need_postvbl_update)
6527 mutex_lock(&dev_priv->wm.wm_mutex);
6528 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6529 ilk_program_watermarks(dev_priv);
6530 mutex_unlock(&dev_priv->wm.wm_mutex);
6533 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6535 level->enable = val & PLANE_WM_EN;
6536 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6537 level->blocks = val & PLANE_WM_BLOCKS_MASK;
6538 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
6541 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6542 struct skl_pipe_wm *out)
6544 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6545 enum pipe pipe = crtc->pipe;
6546 int level, max_level;
6547 enum plane_id plane_id;
6550 max_level = ilk_wm_max_level(dev_priv);
6552 for_each_plane_id_on_crtc(crtc, plane_id) {
6553 struct skl_plane_wm *wm = &out->planes[plane_id];
6555 for (level = 0; level <= max_level; level++) {
6556 if (plane_id != PLANE_CURSOR)
6557 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6559 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6561 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6564 if (plane_id != PLANE_CURSOR)
6565 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6567 val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6569 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6571 if (HAS_HW_SAGV_WM(dev_priv)) {
6572 if (plane_id != PLANE_CURSOR)
6573 val = intel_uncore_read(&dev_priv->uncore,
6574 PLANE_WM_SAGV(pipe, plane_id));
6576 val = intel_uncore_read(&dev_priv->uncore,
6579 skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
6581 if (plane_id != PLANE_CURSOR)
6582 val = intel_uncore_read(&dev_priv->uncore,
6583 PLANE_WM_SAGV_TRANS(pipe, plane_id));
6585 val = intel_uncore_read(&dev_priv->uncore,
6586 CUR_WM_SAGV_TRANS(pipe));
6588 skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
6589 } else if (DISPLAY_VER(dev_priv) >= 12) {
6590 wm->sagv.wm0 = wm->wm[0];
6591 wm->sagv.trans_wm = wm->trans_wm;
6596 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6598 struct intel_dbuf_state *dbuf_state =
6599 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6600 struct intel_crtc *crtc;
6602 if (IS_ALDERLAKE_P(dev_priv))
6603 dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
6605 for_each_intel_crtc(&dev_priv->drm, crtc) {
6606 struct intel_crtc_state *crtc_state =
6607 to_intel_crtc_state(crtc->base.state);
6608 enum pipe pipe = crtc->pipe;
6609 unsigned int mbus_offset;
6610 enum plane_id plane_id;
6612 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6613 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6615 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6617 for_each_plane_id_on_crtc(crtc, plane_id) {
6618 struct skl_ddb_entry *ddb_y =
6619 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6620 struct skl_ddb_entry *ddb_uv =
6621 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
6623 skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6624 plane_id, ddb_y, ddb_uv);
6626 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6627 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
6630 dbuf_state->slices[pipe] =
6631 skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes);
6633 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6636 * Used for checking overlaps, so we need absolute
6637 * offsets instead of MBUS relative offsets.
6639 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_state->slices[pipe]);
6640 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
6641 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
6643 drm_dbg_kms(&dev_priv->drm,
6644 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
6645 crtc->base.base.id, crtc->base.name,
6646 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6647 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
6648 yesno(dbuf_state->joined_mbus));
6651 dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6654 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6656 struct drm_device *dev = crtc->base.dev;
6657 struct drm_i915_private *dev_priv = to_i915(dev);
6658 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6659 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6660 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6661 enum pipe pipe = crtc->pipe;
6663 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6665 memset(active, 0, sizeof(*active));
6667 active->pipe_enabled = crtc->active;
6669 if (active->pipe_enabled) {
6670 u32 tmp = hw->wm_pipe[pipe];
6673 * For active pipes LP0 watermark is marked as
6674 * enabled, and LP1+ watermaks as disabled since
6675 * we can't really reverse compute them in case
6676 * multiple pipes are active.
6678 active->wm[0].enable = true;
6679 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6680 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6681 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6683 int level, max_level = ilk_wm_max_level(dev_priv);
6686 * For inactive pipes, all watermark levels
6687 * should be marked as enabled but zeroed,
6688 * which is what we'd compute them to.
6690 for (level = 0; level <= max_level; level++)
6691 active->wm[level].enable = true;
6694 crtc->wm.active.ilk = *active;
6697 #define _FW_WM(value, plane) \
6698 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6699 #define _FW_WM_VLV(value, plane) \
6700 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6702 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6703 struct g4x_wm_values *wm)
6707 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6708 wm->sr.plane = _FW_WM(tmp, SR);
6709 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6710 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6711 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6713 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6714 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6715 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6716 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6717 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6718 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6719 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6721 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6722 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6723 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6724 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6725 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6728 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6729 struct vlv_wm_values *wm)
6734 for_each_pipe(dev_priv, pipe) {
6735 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6737 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6738 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6739 wm->ddl[pipe].plane[PLANE_CURSOR] =
6740 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6741 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6742 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6743 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6744 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6747 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6748 wm->sr.plane = _FW_WM(tmp, SR);
6749 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6750 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6751 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6753 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6754 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6755 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6756 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6758 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6759 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6761 if (IS_CHERRYVIEW(dev_priv)) {
6762 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6763 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6764 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6766 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6767 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6768 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6770 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6771 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6772 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6774 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6775 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6776 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6777 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6778 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6779 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6780 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6781 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6782 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6783 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6784 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6786 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6787 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6788 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6790 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6791 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6792 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6793 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6794 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6795 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6796 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6797 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6804 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6806 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6807 struct intel_crtc *crtc;
6809 g4x_read_wm_values(dev_priv, wm);
6811 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6813 for_each_intel_crtc(&dev_priv->drm, crtc) {
6814 struct intel_crtc_state *crtc_state =
6815 to_intel_crtc_state(crtc->base.state);
6816 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6817 struct g4x_pipe_wm *raw;
6818 enum pipe pipe = crtc->pipe;
6819 enum plane_id plane_id;
6820 int level, max_level;
6822 active->cxsr = wm->cxsr;
6823 active->hpll_en = wm->hpll_en;
6824 active->fbc_en = wm->fbc_en;
6826 active->sr = wm->sr;
6827 active->hpll = wm->hpll;
6829 for_each_plane_id_on_crtc(crtc, plane_id) {
6830 active->wm.plane[plane_id] =
6831 wm->pipe[pipe].plane[plane_id];
6834 if (wm->cxsr && wm->hpll_en)
6835 max_level = G4X_WM_LEVEL_HPLL;
6837 max_level = G4X_WM_LEVEL_SR;
6839 max_level = G4X_WM_LEVEL_NORMAL;
6841 level = G4X_WM_LEVEL_NORMAL;
6842 raw = &crtc_state->wm.g4x.raw[level];
6843 for_each_plane_id_on_crtc(crtc, plane_id)
6844 raw->plane[plane_id] = active->wm.plane[plane_id];
6846 level = G4X_WM_LEVEL_SR;
6847 if (level > max_level)
6850 raw = &crtc_state->wm.g4x.raw[level];
6851 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6852 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6853 raw->plane[PLANE_SPRITE0] = 0;
6854 raw->fbc = active->sr.fbc;
6856 level = G4X_WM_LEVEL_HPLL;
6857 if (level > max_level)
6860 raw = &crtc_state->wm.g4x.raw[level];
6861 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6862 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6863 raw->plane[PLANE_SPRITE0] = 0;
6864 raw->fbc = active->hpll.fbc;
6868 for_each_plane_id_on_crtc(crtc, plane_id)
6869 g4x_raw_plane_wm_set(crtc_state, level,
6870 plane_id, USHRT_MAX);
6871 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6873 crtc_state->wm.g4x.optimal = *active;
6874 crtc_state->wm.g4x.intermediate = *active;
6876 drm_dbg_kms(&dev_priv->drm,
6877 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6879 wm->pipe[pipe].plane[PLANE_PRIMARY],
6880 wm->pipe[pipe].plane[PLANE_CURSOR],
6881 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6884 drm_dbg_kms(&dev_priv->drm,
6885 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6886 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6887 drm_dbg_kms(&dev_priv->drm,
6888 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6889 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6890 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6891 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6894 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6896 struct intel_plane *plane;
6897 struct intel_crtc *crtc;
6899 mutex_lock(&dev_priv->wm.wm_mutex);
6901 for_each_intel_plane(&dev_priv->drm, plane) {
6902 struct intel_crtc *crtc =
6903 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6904 struct intel_crtc_state *crtc_state =
6905 to_intel_crtc_state(crtc->base.state);
6906 struct intel_plane_state *plane_state =
6907 to_intel_plane_state(plane->base.state);
6908 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6909 enum plane_id plane_id = plane->id;
6912 if (plane_state->uapi.visible)
6915 for (level = 0; level < 3; level++) {
6916 struct g4x_pipe_wm *raw =
6917 &crtc_state->wm.g4x.raw[level];
6919 raw->plane[plane_id] = 0;
6920 wm_state->wm.plane[plane_id] = 0;
6923 if (plane_id == PLANE_PRIMARY) {
6924 for (level = 0; level < 3; level++) {
6925 struct g4x_pipe_wm *raw =
6926 &crtc_state->wm.g4x.raw[level];
6930 wm_state->sr.fbc = 0;
6931 wm_state->hpll.fbc = 0;
6932 wm_state->fbc_en = false;
6936 for_each_intel_crtc(&dev_priv->drm, crtc) {
6937 struct intel_crtc_state *crtc_state =
6938 to_intel_crtc_state(crtc->base.state);
6940 crtc_state->wm.g4x.intermediate =
6941 crtc_state->wm.g4x.optimal;
6942 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6945 g4x_program_watermarks(dev_priv);
6947 mutex_unlock(&dev_priv->wm.wm_mutex);
6950 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6952 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6953 struct intel_crtc *crtc;
6956 vlv_read_wm_values(dev_priv, wm);
6958 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6959 wm->level = VLV_WM_LEVEL_PM2;
6961 if (IS_CHERRYVIEW(dev_priv)) {
6962 vlv_punit_get(dev_priv);
6964 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6965 if (val & DSP_MAXFIFO_PM5_ENABLE)
6966 wm->level = VLV_WM_LEVEL_PM5;
6969 * If DDR DVFS is disabled in the BIOS, Punit
6970 * will never ack the request. So if that happens
6971 * assume we don't have to enable/disable DDR DVFS
6972 * dynamically. To test that just set the REQ_ACK
6973 * bit to poke the Punit, but don't change the
6974 * HIGH/LOW bits so that we don't actually change
6975 * the current state.
6977 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6978 val |= FORCE_DDR_FREQ_REQ_ACK;
6979 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6981 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6982 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6983 drm_dbg_kms(&dev_priv->drm,
6984 "Punit not acking DDR DVFS request, "
6985 "assuming DDR DVFS is disabled\n");
6986 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6988 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6989 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6990 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6993 vlv_punit_put(dev_priv);
6996 for_each_intel_crtc(&dev_priv->drm, crtc) {
6997 struct intel_crtc_state *crtc_state =
6998 to_intel_crtc_state(crtc->base.state);
6999 struct vlv_wm_state *active = &crtc->wm.active.vlv;
7000 const struct vlv_fifo_state *fifo_state =
7001 &crtc_state->wm.vlv.fifo_state;
7002 enum pipe pipe = crtc->pipe;
7003 enum plane_id plane_id;
7006 vlv_get_fifo_size(crtc_state);
7008 active->num_levels = wm->level + 1;
7009 active->cxsr = wm->cxsr;
7011 for (level = 0; level < active->num_levels; level++) {
7012 struct g4x_pipe_wm *raw =
7013 &crtc_state->wm.vlv.raw[level];
7015 active->sr[level].plane = wm->sr.plane;
7016 active->sr[level].cursor = wm->sr.cursor;
7018 for_each_plane_id_on_crtc(crtc, plane_id) {
7019 active->wm[level].plane[plane_id] =
7020 wm->pipe[pipe].plane[plane_id];
7022 raw->plane[plane_id] =
7023 vlv_invert_wm_value(active->wm[level].plane[plane_id],
7024 fifo_state->plane[plane_id]);
7028 for_each_plane_id_on_crtc(crtc, plane_id)
7029 vlv_raw_plane_wm_set(crtc_state, level,
7030 plane_id, USHRT_MAX);
7031 vlv_invalidate_wms(crtc, active, level);
7033 crtc_state->wm.vlv.optimal = *active;
7034 crtc_state->wm.vlv.intermediate = *active;
7036 drm_dbg_kms(&dev_priv->drm,
7037 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
7039 wm->pipe[pipe].plane[PLANE_PRIMARY],
7040 wm->pipe[pipe].plane[PLANE_CURSOR],
7041 wm->pipe[pipe].plane[PLANE_SPRITE0],
7042 wm->pipe[pipe].plane[PLANE_SPRITE1]);
7045 drm_dbg_kms(&dev_priv->drm,
7046 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
7047 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
7050 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
7052 struct intel_plane *plane;
7053 struct intel_crtc *crtc;
7055 mutex_lock(&dev_priv->wm.wm_mutex);
7057 for_each_intel_plane(&dev_priv->drm, plane) {
7058 struct intel_crtc *crtc =
7059 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
7060 struct intel_crtc_state *crtc_state =
7061 to_intel_crtc_state(crtc->base.state);
7062 struct intel_plane_state *plane_state =
7063 to_intel_plane_state(plane->base.state);
7064 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
7065 const struct vlv_fifo_state *fifo_state =
7066 &crtc_state->wm.vlv.fifo_state;
7067 enum plane_id plane_id = plane->id;
7070 if (plane_state->uapi.visible)
7073 for (level = 0; level < wm_state->num_levels; level++) {
7074 struct g4x_pipe_wm *raw =
7075 &crtc_state->wm.vlv.raw[level];
7077 raw->plane[plane_id] = 0;
7079 wm_state->wm[level].plane[plane_id] =
7080 vlv_invert_wm_value(raw->plane[plane_id],
7081 fifo_state->plane[plane_id]);
7085 for_each_intel_crtc(&dev_priv->drm, crtc) {
7086 struct intel_crtc_state *crtc_state =
7087 to_intel_crtc_state(crtc->base.state);
7089 crtc_state->wm.vlv.intermediate =
7090 crtc_state->wm.vlv.optimal;
7091 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
7094 vlv_program_watermarks(dev_priv);
7096 mutex_unlock(&dev_priv->wm.wm_mutex);
7100 * FIXME should probably kill this and improve
7101 * the real watermark readout/sanitation instead
7103 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
7105 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM1_LP_SR_EN);
7106 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM1_LP_SR_EN);
7107 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM1_LP_SR_EN);
7110 * Don't touch WM1S_LP_EN here.
7111 * Doing so could cause underruns.
7115 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
7117 struct ilk_wm_values *hw = &dev_priv->wm.hw;
7118 struct intel_crtc *crtc;
7120 ilk_init_lp_watermarks(dev_priv);
7122 for_each_intel_crtc(&dev_priv->drm, crtc)
7123 ilk_pipe_wm_get_hw_state(crtc);
7125 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
7126 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
7127 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
7129 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
7130 if (DISPLAY_VER(dev_priv) >= 7) {
7131 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
7132 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
7135 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
7136 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
7137 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7138 else if (IS_IVYBRIDGE(dev_priv))
7139 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
7140 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7143 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
7146 void intel_enable_ipc(struct drm_i915_private *dev_priv)
7150 if (!HAS_IPC(dev_priv))
7153 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
7155 if (dev_priv->ipc_enabled)
7156 val |= DISP_IPC_ENABLE;
7158 val &= ~DISP_IPC_ENABLE;
7160 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
7163 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
7165 /* Display WA #0477 WaDisableIPC: skl */
7166 if (IS_SKYLAKE(dev_priv))
7169 /* Display WA #1141: SKL:all KBL:all CFL */
7170 if (IS_KABYLAKE(dev_priv) ||
7171 IS_COFFEELAKE(dev_priv) ||
7172 IS_COMETLAKE(dev_priv))
7173 return dev_priv->dram_info.symmetric_memory;
7178 void intel_init_ipc(struct drm_i915_private *dev_priv)
7180 if (!HAS_IPC(dev_priv))
7183 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
7185 intel_enable_ipc(dev_priv);
7188 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
7191 * On Ibex Peak and Cougar Point, we need to disable clock
7192 * gating for the panel power sequencer or it will fail to
7193 * start up when no ports are active.
7195 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
7198 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
7202 for_each_pipe(dev_priv, pipe) {
7203 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
7204 intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
7205 DISPPLANE_TRICKLE_FEED_DISABLE);
7207 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
7208 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
7212 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
7214 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7218 * WaFbcDisableDpfcClockGating:ilk
7220 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
7221 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
7222 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
7224 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
7225 MARIUNIT_CLOCK_GATE_DISABLE |
7226 SVSMUNIT_CLOCK_GATE_DISABLE);
7227 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
7228 VFMUNIT_CLOCK_GATE_DISABLE);
7231 * According to the spec the following bits should be set in
7232 * order to enable memory self-refresh
7233 * The bit 22/21 of 0x42004
7234 * The bit 5 of 0x42020
7235 * The bit 15 of 0x45000
7237 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7238 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7239 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
7240 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
7241 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
7242 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7246 * Based on the document from hardware guys the following bits
7247 * should be set unconditionally in order to enable FBC.
7248 * The bit 22 of 0x42000
7249 * The bit 22 of 0x42004
7250 * The bit 7,8,9 of 0x42020.
7252 if (IS_IRONLAKE_M(dev_priv)) {
7253 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
7254 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7255 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7257 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7258 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7262 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7264 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7265 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7266 ILK_ELPIN_409_SELECT);
7268 g4x_disable_trickle_feed(dev_priv);
7270 ibx_init_clock_gating(dev_priv);
7273 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
7279 * On Ibex Peak and Cougar Point, we need to disable clock
7280 * gating for the panel power sequencer or it will fail to
7281 * start up when no ports are active.
7283 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
7284 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
7285 PCH_CPUNIT_CLOCK_GATE_DISABLE);
7286 intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
7287 DPLS_EDP_PPS_FIX_DIS);
7288 /* The below fixes the weird display corruption, a few pixels shifted
7289 * downward, on (only) LVDS of some HP laptops with IVY.
7291 for_each_pipe(dev_priv, pipe) {
7292 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
7293 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
7294 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7295 if (dev_priv->vbt.fdi_rx_polarity_inverted)
7296 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7297 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
7298 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
7299 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
7301 /* WADP0ClockGatingDisable */
7302 for_each_pipe(dev_priv, pipe) {
7303 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
7304 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7308 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
7312 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
7313 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
7314 drm_dbg_kms(&dev_priv->drm,
7315 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
7319 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7321 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7323 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7325 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7326 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7327 ILK_ELPIN_409_SELECT);
7329 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7330 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7331 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7332 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7334 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7335 * gating disable must be set. Failure to set it results in
7336 * flickering pixels due to Z write ordering failures after
7337 * some amount of runtime in the Mesa "fire" demo, and Unigine
7338 * Sanctuary and Tropics, and apparently anything else with
7339 * alpha test or pixel discard.
7341 * According to the spec, bit 11 (RCCUNIT) must also be set,
7342 * but we didn't debug actual testcases to find it out.
7344 * WaDisableRCCUnitClockGating:snb
7345 * WaDisableRCPBUnitClockGating:snb
7347 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7348 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7349 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7352 * According to the spec the following bits should be
7353 * set in order to enable memory self-refresh and fbc:
7354 * The bit21 and bit22 of 0x42000
7355 * The bit21 and bit22 of 0x42004
7356 * The bit5 and bit7 of 0x42020
7357 * The bit14 of 0x70180
7358 * The bit14 of 0x71180
7360 * WaFbcAsynchFlipDisableFbcQueue:snb
7362 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7363 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7364 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7365 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7366 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7367 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7368 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7369 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7370 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7371 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7373 g4x_disable_trickle_feed(dev_priv);
7375 cpt_init_clock_gating(dev_priv);
7377 gen6_check_mch_setup(dev_priv);
7380 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7383 * TODO: this bit should only be enabled when really needed, then
7384 * disabled when not needed anymore in order to save power.
7386 if (HAS_PCH_LPT_LP(dev_priv))
7387 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7388 intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7389 PCH_LP_PARTITION_LEVEL_DISABLE);
7391 /* WADPOClockGatingDisable:hsw */
7392 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7393 intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7394 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7397 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7399 if (HAS_PCH_LPT_LP(dev_priv)) {
7400 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7402 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7403 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7407 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7408 int general_prio_credits,
7409 int high_prio_credits)
7414 /* WaTempDisableDOPClkGating:bdw */
7415 misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7416 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7418 val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7419 val &= ~L3_PRIO_CREDITS_MASK;
7420 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7421 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7422 intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7425 * Wait at least 100 clocks before re-enabling clock gating.
7426 * See the definition of L3SQCREG1 in BSpec.
7428 intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7430 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7433 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7435 /* Wa_1409120013:icl,ehl */
7436 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7437 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7439 /*Wa_14010594013:icl, ehl */
7440 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7441 0, ICL_DELAY_PMRSP);
7444 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7446 /* Wa_1409120013:tgl,rkl,adl-s,dg1 */
7447 if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv) ||
7448 IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv))
7449 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7450 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7452 /* Wa_1409825376:tgl (pre-prod)*/
7453 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
7454 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7455 TGL_VRH_GATING_DIS);
7457 /* Wa_14013723622:tgl,rkl,dg1,adl-s */
7458 if (DISPLAY_VER(dev_priv) == 12)
7459 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
7460 CLKREQ_POLICY_MEM_UP_OVRD, 0);
7463 static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
7465 gen12lp_init_clock_gating(dev_priv);
7467 /* Wa_22011091694:adlp */
7468 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
7471 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7473 gen12lp_init_clock_gating(dev_priv);
7475 /* Wa_1409836686:dg1[a0] */
7476 if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
7477 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7481 static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
7483 /* Wa_22010146351:xehpsdv */
7484 if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
7485 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
7488 static void dg2_init_clock_gating(struct drm_i915_private *i915)
7490 /* Wa_22010954014:dg2_g10 */
7491 if (IS_DG2_G10(i915))
7492 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
7496 * Wa_14010733611:dg2_g10
7497 * Wa_22010146351:dg2_g10
7499 if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
7500 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
7501 SGR_DIS | SGGI_DIS);
7504 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7506 if (!HAS_PCH_CNP(dev_priv))
7509 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7510 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7511 CNP_PWM_CGE_GATING_DISABLE);
7514 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7516 cnp_init_clock_gating(dev_priv);
7517 gen9_init_clock_gating(dev_priv);
7519 /* WAC6entrylatency:cfl */
7520 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7521 FBC_LLC_FULLY_OPEN);
7524 * WaFbcTurnOffFbcWatermark:cfl
7525 * Display WA #0562: cfl
7527 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7531 * WaFbcNukeOnHostModify:cfl
7532 * Display WA #0873: cfl
7534 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7535 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7538 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7540 gen9_init_clock_gating(dev_priv);
7542 /* WAC6entrylatency:kbl */
7543 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7544 FBC_LLC_FULLY_OPEN);
7546 /* WaDisableSDEUnitClockGating:kbl */
7547 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
7548 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7549 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7551 /* WaDisableGamClockGating:kbl */
7552 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
7553 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7554 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7557 * WaFbcTurnOffFbcWatermark:kbl
7558 * Display WA #0562: kbl
7560 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7564 * WaFbcNukeOnHostModify:kbl
7565 * Display WA #0873: kbl
7567 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7568 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7571 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7573 gen9_init_clock_gating(dev_priv);
7575 /* WaDisableDopClockGating:skl */
7576 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7577 ~GEN7_DOP_CLOCK_GATE_ENABLE);
7579 /* WAC6entrylatency:skl */
7580 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7581 FBC_LLC_FULLY_OPEN);
7584 * WaFbcTurnOffFbcWatermark:skl
7585 * Display WA #0562: skl
7587 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7591 * WaFbcNukeOnHostModify:skl
7592 * Display WA #0873: skl
7594 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7595 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7598 * WaFbcHighMemBwCorruptionAvoidance:skl
7599 * Display WA #0883: skl
7601 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7602 ILK_DPFC_DISABLE_DUMMY0);
7605 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7609 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7610 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7611 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7614 /* WaSwitchSolVfFArbitrationPriority:bdw */
7615 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7617 /* WaPsrDPAMaskVBlankInSRD:bdw */
7618 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7619 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7621 for_each_pipe(dev_priv, pipe) {
7622 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7623 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7624 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7625 BDW_DPRS_MASK_VBLANK_SRD);
7628 /* WaVSRefCountFullforceMissDisable:bdw */
7629 /* WaDSRefCountFullforceMissDisable:bdw */
7630 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7631 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7632 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7634 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7635 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7637 /* WaDisableSDEUnitClockGating:bdw */
7638 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7639 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7641 /* WaProgramL3SqcReg1Default:bdw */
7642 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7644 /* WaKVMNotificationOnConfigChange:bdw */
7645 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7646 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7648 lpt_init_clock_gating(dev_priv);
7650 /* WaDisableDopClockGating:bdw
7652 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7655 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7656 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7659 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7661 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7662 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7663 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7666 /* This is required by WaCatErrorRejectionIssue:hsw */
7667 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7668 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7669 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7671 /* WaSwitchSolVfFArbitrationPriority:hsw */
7672 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7674 lpt_init_clock_gating(dev_priv);
7677 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7681 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7683 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7684 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7685 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7688 /* WaDisableBackToBackFlipFix:ivb */
7689 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7690 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7691 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7693 if (IS_IVB_GT1(dev_priv))
7694 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7695 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7697 /* must write both registers */
7698 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7699 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7700 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7701 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7705 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7706 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7708 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7709 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7711 /* This is required by WaCatErrorRejectionIssue:ivb */
7712 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7713 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7714 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7716 g4x_disable_trickle_feed(dev_priv);
7718 snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7719 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7720 snpcr |= GEN6_MBC_SNPCR_MED;
7721 intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7723 if (!HAS_PCH_NOP(dev_priv))
7724 cpt_init_clock_gating(dev_priv);
7726 gen6_check_mch_setup(dev_priv);
7729 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7731 /* WaDisableBackToBackFlipFix:vlv */
7732 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7733 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7734 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7736 /* WaDisableDopClockGating:vlv */
7737 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7738 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7740 /* This is required by WaCatErrorRejectionIssue:vlv */
7741 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7742 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7743 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7746 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7747 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7749 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7750 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7752 /* WaDisableL3Bank2xClockGate:vlv
7753 * Disabling L3 clock gating- MMIO 940c[25] = 1
7754 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7755 intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7756 intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7759 * WaDisableVLVClockGating_VBIIssue:vlv
7760 * Disable clock gating on th GCFG unit to prevent a delay
7761 * in the reporting of vblank events.
7763 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7766 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7768 /* WaVSRefCountFullforceMissDisable:chv */
7769 /* WaDSRefCountFullforceMissDisable:chv */
7770 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7771 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7772 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7774 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7775 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7776 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7778 /* WaDisableCSUnitClockGating:chv */
7779 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7780 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7782 /* WaDisableSDEUnitClockGating:chv */
7783 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7784 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7787 * WaProgramL3SqcReg1Default:chv
7788 * See gfxspecs/Related Documents/Performance Guide/
7789 * LSQC Setting Recommendations.
7791 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7794 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7798 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7799 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7800 GS_UNIT_CLOCK_GATE_DISABLE |
7801 CL_UNIT_CLOCK_GATE_DISABLE);
7802 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7803 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7804 OVRUNIT_CLOCK_GATE_DISABLE |
7805 OVCUNIT_CLOCK_GATE_DISABLE;
7806 if (IS_GM45(dev_priv))
7807 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7808 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7810 g4x_disable_trickle_feed(dev_priv);
7813 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7815 struct intel_uncore *uncore = &dev_priv->uncore;
7817 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7818 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7819 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7820 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7821 intel_uncore_write16(uncore, DEUC, 0);
7822 intel_uncore_write(uncore,
7824 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7827 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7829 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7830 I965_RCC_CLOCK_GATE_DISABLE |
7831 I965_RCPB_CLOCK_GATE_DISABLE |
7832 I965_ISC_CLOCK_GATE_DISABLE |
7833 I965_FBC_CLOCK_GATE_DISABLE);
7834 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
7835 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7836 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7839 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7841 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
7843 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7844 DSTATE_DOT_CLOCK_GATING;
7845 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
7847 if (IS_PINEVIEW(dev_priv))
7848 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7850 /* IIR "flip pending" means done if this bit is set */
7851 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7853 /* interrupts should cause a wake up from C3 */
7854 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7856 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7857 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7859 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7860 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7863 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7865 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7867 /* interrupts should cause a wake up from C3 */
7868 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7869 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7871 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7872 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7875 * Have FBC ignore 3D activity since we use software
7876 * render tracking, and otherwise a pure 3D workload
7877 * (even if it just renders a single frame and then does
7878 * abosultely nothing) would not allow FBC to recompress
7879 * until a 2D blit occurs.
7881 intel_uncore_write(&dev_priv->uncore, SCPD0,
7882 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7885 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7887 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7888 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7889 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7892 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7894 dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
7897 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7899 if (HAS_PCH_LPT(dev_priv))
7900 lpt_suspend_hw(dev_priv);
7903 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7905 drm_dbg_kms(&dev_priv->drm,
7906 "No clock gating settings or workarounds applied.\n");
7909 #define CG_FUNCS(platform) \
7910 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
7911 .init_clock_gating = platform##_init_clock_gating, \
7942 * intel_init_clock_gating_hooks - setup the clock gating hooks
7943 * @dev_priv: device private
7945 * Setup the hooks that configure which clocks of a given platform can be
7946 * gated and also apply various GT and display specific workarounds for these
7947 * platforms. Note that some GT specific workarounds are applied separately
7948 * when GPU contexts or batchbuffers start their execution.
7950 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7952 if (IS_DG2(dev_priv))
7953 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
7954 else if (IS_XEHPSDV(dev_priv))
7955 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
7956 else if (IS_ALDERLAKE_P(dev_priv))
7957 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
7958 else if (IS_DG1(dev_priv))
7959 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
7960 else if (GRAPHICS_VER(dev_priv) == 12)
7961 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
7962 else if (GRAPHICS_VER(dev_priv) == 11)
7963 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
7964 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7965 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
7966 else if (IS_SKYLAKE(dev_priv))
7967 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
7968 else if (IS_KABYLAKE(dev_priv))
7969 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
7970 else if (IS_BROXTON(dev_priv))
7971 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
7972 else if (IS_GEMINILAKE(dev_priv))
7973 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
7974 else if (IS_BROADWELL(dev_priv))
7975 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
7976 else if (IS_CHERRYVIEW(dev_priv))
7977 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
7978 else if (IS_HASWELL(dev_priv))
7979 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
7980 else if (IS_IVYBRIDGE(dev_priv))
7981 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
7982 else if (IS_VALLEYVIEW(dev_priv))
7983 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
7984 else if (GRAPHICS_VER(dev_priv) == 6)
7985 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
7986 else if (GRAPHICS_VER(dev_priv) == 5)
7987 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
7988 else if (IS_G4X(dev_priv))
7989 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
7990 else if (IS_I965GM(dev_priv))
7991 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
7992 else if (IS_I965G(dev_priv))
7993 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
7994 else if (GRAPHICS_VER(dev_priv) == 3)
7995 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
7996 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7997 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
7998 else if (GRAPHICS_VER(dev_priv) == 2)
7999 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
8001 MISSING_CASE(INTEL_DEVID(dev_priv));
8002 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
8006 static const struct drm_i915_wm_disp_funcs skl_wm_funcs = {
8007 .compute_global_watermarks = skl_compute_wm,
8010 static const struct drm_i915_wm_disp_funcs ilk_wm_funcs = {
8011 .compute_pipe_wm = ilk_compute_pipe_wm,
8012 .compute_intermediate_wm = ilk_compute_intermediate_wm,
8013 .initial_watermarks = ilk_initial_watermarks,
8014 .optimize_watermarks = ilk_optimize_watermarks,
8017 static const struct drm_i915_wm_disp_funcs vlv_wm_funcs = {
8018 .compute_pipe_wm = vlv_compute_pipe_wm,
8019 .compute_intermediate_wm = vlv_compute_intermediate_wm,
8020 .initial_watermarks = vlv_initial_watermarks,
8021 .optimize_watermarks = vlv_optimize_watermarks,
8022 .atomic_update_watermarks = vlv_atomic_update_fifo,
8025 static const struct drm_i915_wm_disp_funcs g4x_wm_funcs = {
8026 .compute_pipe_wm = g4x_compute_pipe_wm,
8027 .compute_intermediate_wm = g4x_compute_intermediate_wm,
8028 .initial_watermarks = g4x_initial_watermarks,
8029 .optimize_watermarks = g4x_optimize_watermarks,
8032 static const struct drm_i915_wm_disp_funcs pnv_wm_funcs = {
8033 .update_wm = pnv_update_wm,
8036 static const struct drm_i915_wm_disp_funcs i965_wm_funcs = {
8037 .update_wm = i965_update_wm,
8040 static const struct drm_i915_wm_disp_funcs i9xx_wm_funcs = {
8041 .update_wm = i9xx_update_wm,
8044 static const struct drm_i915_wm_disp_funcs i845_wm_funcs = {
8045 .update_wm = i845_update_wm,
8048 static const struct drm_i915_wm_disp_funcs nop_funcs = {
8051 /* Set up chip specific power management-related functions */
8052 void intel_init_pm(struct drm_i915_private *dev_priv)
8055 if (IS_PINEVIEW(dev_priv))
8056 pnv_get_mem_freq(dev_priv);
8057 else if (GRAPHICS_VER(dev_priv) == 5)
8058 ilk_get_mem_freq(dev_priv);
8060 if (intel_has_sagv(dev_priv))
8061 skl_setup_sagv_block_time(dev_priv);
8063 /* For FIFO watermark updates */
8064 if (DISPLAY_VER(dev_priv) >= 9) {
8065 skl_setup_wm_latency(dev_priv);
8066 dev_priv->wm_disp = &skl_wm_funcs;
8067 } else if (HAS_PCH_SPLIT(dev_priv)) {
8068 ilk_setup_wm_latency(dev_priv);
8070 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->wm.pri_latency[1] &&
8071 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
8072 (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] &&
8073 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
8074 dev_priv->wm_disp = &ilk_wm_funcs;
8076 drm_dbg_kms(&dev_priv->drm,
8077 "Failed to read display plane latency. "
8079 dev_priv->wm_disp = &nop_funcs;
8081 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
8082 vlv_setup_wm_latency(dev_priv);
8083 dev_priv->wm_disp = &vlv_wm_funcs;
8084 } else if (IS_G4X(dev_priv)) {
8085 g4x_setup_wm_latency(dev_priv);
8086 dev_priv->wm_disp = &g4x_wm_funcs;
8087 } else if (IS_PINEVIEW(dev_priv)) {
8088 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
8091 dev_priv->mem_freq)) {
8092 drm_info(&dev_priv->drm,
8093 "failed to find known CxSR latency "
8094 "(found ddr%s fsb freq %d, mem freq %d), "
8096 (dev_priv->is_ddr3 == 1) ? "3" : "2",
8097 dev_priv->fsb_freq, dev_priv->mem_freq);
8098 /* Disable CxSR and never update its watermark again */
8099 intel_set_memory_cxsr(dev_priv, false);
8100 dev_priv->wm_disp = &nop_funcs;
8102 dev_priv->wm_disp = &pnv_wm_funcs;
8103 } else if (DISPLAY_VER(dev_priv) == 4) {
8104 dev_priv->wm_disp = &i965_wm_funcs;
8105 } else if (DISPLAY_VER(dev_priv) == 3) {
8106 dev_priv->wm_disp = &i9xx_wm_funcs;
8107 } else if (DISPLAY_VER(dev_priv) == 2) {
8108 if (INTEL_NUM_PIPES(dev_priv) == 1)
8109 dev_priv->wm_disp = &i845_wm_funcs;
8111 dev_priv->wm_disp = &i9xx_wm_funcs;
8113 drm_err(&dev_priv->drm,
8114 "unexpected fall-through in %s\n", __func__);
8115 dev_priv->wm_disp = &nop_funcs;
8119 void intel_pm_setup(struct drm_i915_private *dev_priv)
8121 dev_priv->runtime_pm.suspended = false;
8122 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
8125 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
8127 struct intel_dbuf_state *dbuf_state;
8129 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
8133 return &dbuf_state->base;
8136 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
8137 struct intel_global_state *state)
8142 static const struct intel_global_state_funcs intel_dbuf_funcs = {
8143 .atomic_duplicate_state = intel_dbuf_duplicate_state,
8144 .atomic_destroy_state = intel_dbuf_destroy_state,
8147 struct intel_dbuf_state *
8148 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
8150 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8151 struct intel_global_state *dbuf_state;
8153 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
8154 if (IS_ERR(dbuf_state))
8155 return ERR_CAST(dbuf_state);
8157 return to_intel_dbuf_state(dbuf_state);
8160 int intel_dbuf_init(struct drm_i915_private *dev_priv)
8162 struct intel_dbuf_state *dbuf_state;
8164 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
8168 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
8169 &dbuf_state->base, &intel_dbuf_funcs);
8175 * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
8176 * update the request state of all DBUS slices.
8178 static void update_mbus_pre_enable(struct intel_atomic_state *state)
8180 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8181 u32 mbus_ctl, dbuf_min_tracker_val;
8182 enum dbuf_slice slice;
8183 const struct intel_dbuf_state *dbuf_state =
8184 intel_atomic_get_new_dbuf_state(state);
8186 if (!IS_ALDERLAKE_P(dev_priv))
8190 * TODO: Implement vblank synchronized MBUS joining changes.
8191 * Must be properly coordinated with dbuf reprogramming.
8193 if (dbuf_state->joined_mbus) {
8194 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
8195 MBUS_JOIN_PIPE_SELECT_NONE;
8196 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
8198 mbus_ctl = MBUS_HASHING_MODE_2x2 |
8199 MBUS_JOIN_PIPE_SELECT_NONE;
8200 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
8203 intel_de_rmw(dev_priv, MBUS_CTL,
8204 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
8205 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
8207 for_each_dbuf_slice(dev_priv, slice)
8208 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
8209 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
8210 dbuf_min_tracker_val);
8213 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
8215 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8216 const struct intel_dbuf_state *new_dbuf_state =
8217 intel_atomic_get_new_dbuf_state(state);
8218 const struct intel_dbuf_state *old_dbuf_state =
8219 intel_atomic_get_old_dbuf_state(state);
8221 if (!new_dbuf_state ||
8222 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8223 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8226 WARN_ON(!new_dbuf_state->base.changed);
8228 update_mbus_pre_enable(state);
8229 gen9_dbuf_slices_update(dev_priv,
8230 old_dbuf_state->enabled_slices |
8231 new_dbuf_state->enabled_slices);
8234 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
8236 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8237 const struct intel_dbuf_state *new_dbuf_state =
8238 intel_atomic_get_new_dbuf_state(state);
8239 const struct intel_dbuf_state *old_dbuf_state =
8240 intel_atomic_get_old_dbuf_state(state);
8242 if (!new_dbuf_state ||
8243 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8244 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8247 WARN_ON(!new_dbuf_state->base.changed);
8249 gen9_dbuf_slices_update(dev_priv,
8250 new_dbuf_state->enabled_slices);