drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include <drm/drm_atomic_helper.h>
  32 #include <drm/drm_fourcc.h>
  33 #include <drm/drm_plane_helper.h>
  34
  35 #include "display/intel_atomic.h"
  36 #include "display/intel_atomic_plane.h"
  37 #include "display/intel_bw.h"
  38 #include "display/intel_display_types.h"
  39 #include "display/intel_fbc.h"
  40 #include "display/intel_sprite.h"
  41 #include "display/skl_universal_plane.h"
  42
  43 #include "gt/intel_llc.h"
  44
  45 #include "i915_drv.h"
  46 #include "i915_fixed.h"
  47 #include "i915_irq.h"
  48 #include "i915_trace.h"
  49 #include "intel_pm.h"
  50 #include "intel_sideband.h"
  51 #include "../../../platform/x86/intel_ips.h"
  52
  53 /* Stores plane specific WM parameters */
  54 struct skl_wm_params {
  55         bool x_tiled, y_tiled;
  56         bool rc_surface;
  57         bool is_planar;
  58         u32 width;
  59         u8 cpp;
  60         u32 plane_pixel_rate;
  61         u32 y_min_scanlines;
  62         u32 plane_bytes_per_line;
  63         uint_fixed_16_16_t plane_blocks_per_line;
  64         uint_fixed_16_16_t y_tile_minimum;
  65         u32 linetime_us;
  66         u32 dbuf_block_size;
  67 };
  68
  69 /* used in computing the new watermarks state */
  70 struct intel_wm_config {
  71         unsigned int num_pipes_active;
  72         bool sprites_enabled;
  73         bool sprites_scaled;
  74 };
  75
  76 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  77 {
  78         if (HAS_LLC(dev_priv)) {
  79                 /*
  80                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  81                  * Display WA #0390: skl,kbl
  82                  *
  83                  * Must match Sampler, Pixel Back End, and Media. See
  84                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  85                  */
  86                 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
  87                            intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
  88                            SKL_DE_COMPRESSED_HASH_MODE);
  89         }
  90
  91         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  92         intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
  93                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  94
  95         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  96         intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
  97                    intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  98
  99         /*
 100          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
 101          * Display WA #0859: skl,bxt,kbl,glk,cfl
 102          */
 103         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
 104                    DISP_FBC_MEMORY_WAKE);
 105 }
 106
 107 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
 108 {
 109         gen9_init_clock_gating(dev_priv);
 110
 111         /* WaDisableSDEUnitClockGating:bxt */
 112         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
 113                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 114
 115         /*
 116          * FIXME:
 117          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 118          */
 119         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
 120                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 121
 122         /*
 123          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 124          * to stay fully on.
 125          */
 126         intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
 127                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 128
 129         /*
 130          * Lower the display internal timeout.
 131          * This is needed to avoid any hard hangs when DSI port PLL
 132          * is off and a MMIO access is attempted by any privilege
 133          * application, using batch buffers or any other means.
 134          */
 135         intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 136
 137         /*
 138          * WaFbcTurnOffFbcWatermark:bxt
 139          * Display WA #0562: bxt
 140          */
 141         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
 142                    DISP_FBC_WM_DIS);
 143
 144         /*
 145          * WaFbcHighMemBwCorruptionAvoidance:bxt
 146          * Display WA #0883: bxt
 147          */
 148         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
 149                    ILK_DPFC_DISABLE_DUMMY0);
 150 }
 151
 152 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 153 {
 154         gen9_init_clock_gating(dev_priv);
 155
 156         /*
 157          * WaDisablePWMClockGating:glk
 158          * Backlight PWM may stop in the asserted state, causing backlight
 159          * to stay fully on.
 160          */
 161         intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
 162                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 163 }
 164
 165 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
 166 {
 167         u32 tmp;
 168
 169         tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
 170
 171         switch (tmp & CLKCFG_FSB_MASK) {
 172         case CLKCFG_FSB_533:
 173                 dev_priv->fsb_freq = 533; /* 133*4 */
 174                 break;
 175         case CLKCFG_FSB_800:
 176                 dev_priv->fsb_freq = 800; /* 200*4 */
 177                 break;
 178         case CLKCFG_FSB_667:
 179                 dev_priv->fsb_freq =  667; /* 167*4 */
 180                 break;
 181         case CLKCFG_FSB_400:
 182                 dev_priv->fsb_freq = 400; /* 100*4 */
 183                 break;
 184         }
 185
 186         switch (tmp & CLKCFG_MEM_MASK) {
 187         case CLKCFG_MEM_533:
 188                 dev_priv->mem_freq = 533;
 189                 break;
 190         case CLKCFG_MEM_667:
 191                 dev_priv->mem_freq = 667;
 192                 break;
 193         case CLKCFG_MEM_800:
 194                 dev_priv->mem_freq = 800;
 195                 break;
 196         }
 197
 198         /* detect pineview DDR3 setting */
 199         tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
 200         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 201 }
 202
 203 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
 204 {
 205         u16 ddrpll, csipll;
 206
 207         ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
 208         csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
 209
 210         switch (ddrpll & 0xff) {
 211         case 0xc:
 212                 dev_priv->mem_freq = 800;
 213                 break;
 214         case 0x10:
 215                 dev_priv->mem_freq = 1066;
 216                 break;
 217         case 0x14:
 218                 dev_priv->mem_freq = 1333;
 219                 break;
 220         case 0x18:
 221                 dev_priv->mem_freq = 1600;
 222                 break;
 223         default:
 224                 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
 225                         ddrpll & 0xff);
 226                 dev_priv->mem_freq = 0;
 227                 break;
 228         }
 229
 230         switch (csipll & 0x3ff) {
 231         case 0x00c:
 232                 dev_priv->fsb_freq = 3200;
 233                 break;
 234         case 0x00e:
 235                 dev_priv->fsb_freq = 3733;
 236                 break;
 237         case 0x010:
 238                 dev_priv->fsb_freq = 4266;
 239                 break;
 240         case 0x012:
 241                 dev_priv->fsb_freq = 4800;
 242                 break;
 243         case 0x014:
 244                 dev_priv->fsb_freq = 5333;
 245                 break;
 246         case 0x016:
 247                 dev_priv->fsb_freq = 5866;
 248                 break;
 249         case 0x018:
 250                 dev_priv->fsb_freq = 6400;
 251                 break;
 252         default:
 253                 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
 254                         csipll & 0x3ff);
 255                 dev_priv->fsb_freq = 0;
 256                 break;
 257         }
 258 }
 259
 260 static const struct cxsr_latency cxsr_latency_table[] = {
 261         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 262         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 263         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 264         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 265         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 266
 267         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 268         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 269         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 270         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 271         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 272
 273         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 274         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 275         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 276         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 277         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 278
 279         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 280         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 281         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 282         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 283         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 284
 285         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 286         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 287         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 288         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 289         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 290
 291         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 292         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 293         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 294         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 295         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 296 };
 297
 298 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 299                                                          bool is_ddr3,
 300                                                          int fsb,
 301                                                          int mem)
 302 {
 303         const struct cxsr_latency *latency;
 304         int i;
 305
 306         if (fsb == 0 || mem == 0)
 307                 return NULL;
 308
 309         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 310                 latency = &cxsr_latency_table[i];
 311                 if (is_desktop == latency->is_desktop &&
 312                     is_ddr3 == latency->is_ddr3 &&
 313                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 314                         return latency;
 315         }
 316
 317         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 318
 319         return NULL;
 320 }
 321
 322 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 323 {
 324         u32 val;
 325
 326         vlv_punit_get(dev_priv);
 327
 328         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 329         if (enable)
 330                 val &= ~FORCE_DDR_HIGH_FREQ;
 331         else
 332                 val |= FORCE_DDR_HIGH_FREQ;
 333         val &= ~FORCE_DDR_LOW_FREQ;
 334         val |= FORCE_DDR_FREQ_REQ_ACK;
 335         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 336
 337         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 338                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 339                 drm_err(&dev_priv->drm,
 340                         "timed out waiting for Punit DDR DVFS request\n");
 341
 342         vlv_punit_put(dev_priv);
 343 }
 344
 345 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 346 {
 347         u32 val;
 348
 349         vlv_punit_get(dev_priv);
 350
 351         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 352         if (enable)
 353                 val |= DSP_MAXFIFO_PM5_ENABLE;
 354         else
 355                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 356         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 357
 358         vlv_punit_put(dev_priv);
 359 }
 360
 361 #define FW_WM(value, plane) \
 362         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 363
 364 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 365 {
 366         bool was_enabled;
 367         u32 val;
 368
 369         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 370                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 371                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 372                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
 373         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 374                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 375                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 376                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 377         } else if (IS_PINEVIEW(dev_priv)) {
 378                 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 379                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 380                 if (enable)
 381                         val |= PINEVIEW_SELF_REFRESH_EN;
 382                 else
 383                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 384                 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
 385                 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
 386         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 387                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 388                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 389                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 390                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
 391                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 392         } else if (IS_I915GM(dev_priv)) {
 393                 /*
 394                  * FIXME can't find a bit like this for 915G, and
 395                  * and yet it does have the related watermark in
 396                  * FW_BLC_SELF. What's going on?
 397                  */
 398                 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
 399                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 400                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 401                 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
 402                 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
 403         } else {
 404                 return false;
 405         }
 406
 407         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 408
 409         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 410                     enableddisabled(enable),
 411                     enableddisabled(was_enabled));
 412
 413         return was_enabled;
 414 }
 415
 416 /**
 417  * intel_set_memory_cxsr - Configure CxSR state
 418  * @dev_priv: i915 device
 419  * @enable: Allow vs. disallow CxSR
 420  *
 421  * Allow or disallow the system to enter a special CxSR
 422  * (C-state self refresh) state. What typically happens in CxSR mode
 423  * is that several display FIFOs may get combined into a single larger
 424  * FIFO for a particular plane (so called max FIFO mode) to allow the
 425  * system to defer memory fetches longer, and the memory will enter
 426  * self refresh.
 427  *
 428  * Note that enabling CxSR does not guarantee that the system enter
 429  * this special mode, nor does it guarantee that the system stays
 430  * in that mode once entered. So this just allows/disallows the system
 431  * to autonomously utilize the CxSR mode. Other factors such as core
 432  * C-states will affect when/if the system actually enters/exits the
 433  * CxSR mode.
 434  *
 435  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 436  * and the system is free to enter/exit memory self refresh at any time
 437  * even when the use of CxSR has been disallowed.
 438  *
 439  * While the system is actually in the CxSR/max FIFO mode, some plane
 440  * control registers will not get latched on vblank. Thus in order to
 441  * guarantee the system will respond to changes in the plane registers
 442  * we must always disallow CxSR prior to making changes to those registers.
 443  * Unfortunately the system will re-evaluate the CxSR conditions at
 444  * frame start which happens after vblank start (which is when the plane
 445  * registers would get latched), so we can't proceed with the plane update
 446  * during the same frame where we disallowed CxSR.
 447  *
 448  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 449  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 450  * the hardware w.r.t. HPLL SR when writing to plane registers.
 451  * Disallowing just CxSR is sufficient.
 452  */
 453 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 454 {
 455         bool ret;
 456
 457         mutex_lock(&dev_priv->wm.wm_mutex);
 458         ret = _intel_set_memory_cxsr(dev_priv, enable);
 459         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 460                 dev_priv->wm.vlv.cxsr = enable;
 461         else if (IS_G4X(dev_priv))
 462                 dev_priv->wm.g4x.cxsr = enable;
 463         mutex_unlock(&dev_priv->wm.wm_mutex);
 464
 465         return ret;
 466 }
 467
 468 /*
 469  * Latency for FIFO fetches is dependent on several factors:
 470  *   - memory configuration (speed, channels)
 471  *   - chipset
 472  *   - current MCH state
 473  * It can be fairly high in some situations, so here we assume a fairly
 474  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 475  * set this value too high, the FIFO will fetch frequently to stay full)
 476  * and power consumption (set it too low to save power and we might see
 477  * FIFO underruns and display "flicker").
 478  *
 479  * A value of 5us seems to be a good balance; safe for very low end
 480  * platforms but not overly aggressive on lower latency configs.
 481  */
 482 static const int pessimal_latency_ns = 5000;
 483
 484 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 485         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 486
 487 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 488 {
 489         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 490         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 491         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 492         enum pipe pipe = crtc->pipe;
 493         int sprite0_start, sprite1_start;
 494         u32 dsparb, dsparb2, dsparb3;
 495
 496         switch (pipe) {
 497         case PIPE_A:
 498                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 499                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 500                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 501                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 502                 break;
 503         case PIPE_B:
 504                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 505                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 506                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 507                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 508                 break;
 509         case PIPE_C:
 510                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 511                 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
 512                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 513                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 514                 break;
 515         default:
 516                 MISSING_CASE(pipe);
 517                 return;
 518         }
 519
 520         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 521         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 522         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 523         fifo_state->plane[PLANE_CURSOR] = 63;
 524 }
 525
 526 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 527                               enum i9xx_plane_id i9xx_plane)
 528 {
 529         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 530         int size;
 531
 532         size = dsparb & 0x7f;
 533         if (i9xx_plane == PLANE_B)
 534                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 535
 536         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 537                     dsparb, plane_name(i9xx_plane), size);
 538
 539         return size;
 540 }
 541
 542 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 543                               enum i9xx_plane_id i9xx_plane)
 544 {
 545         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 546         int size;
 547
 548         size = dsparb & 0x1ff;
 549         if (i9xx_plane == PLANE_B)
 550                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 551         size >>= 1; /* Convert to cachelines */
 552
 553         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 554                     dsparb, plane_name(i9xx_plane), size);
 555
 556         return size;
 557 }
 558
 559 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 560                               enum i9xx_plane_id i9xx_plane)
 561 {
 562         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 563         int size;
 564
 565         size = dsparb & 0x7f;
 566         size >>= 2; /* Convert to cachelines */
 567
 568         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 569                     dsparb, plane_name(i9xx_plane), size);
 570
 571         return size;
 572 }
 573
 574 /* Pineview has different values for various configs */
 575 static const struct intel_watermark_params pnv_display_wm = {
 576         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 577         .max_wm = PINEVIEW_MAX_WM,
 578         .default_wm = PINEVIEW_DFT_WM,
 579         .guard_size = PINEVIEW_GUARD_WM,
 580         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 581 };
 582
 583 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 584         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 585         .max_wm = PINEVIEW_MAX_WM,
 586         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 587         .guard_size = PINEVIEW_GUARD_WM,
 588         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 589 };
 590
 591 static const struct intel_watermark_params pnv_cursor_wm = {
 592         .fifo_size = PINEVIEW_CURSOR_FIFO,
 593         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 594         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 595         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 596         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 597 };
 598
 599 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 600         .fifo_size = PINEVIEW_CURSOR_FIFO,
 601         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 602         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 603         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 604         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 605 };
 606
 607 static const struct intel_watermark_params i965_cursor_wm_info = {
 608         .fifo_size = I965_CURSOR_FIFO,
 609         .max_wm = I965_CURSOR_MAX_WM,
 610         .default_wm = I965_CURSOR_DFT_WM,
 611         .guard_size = 2,
 612         .cacheline_size = I915_FIFO_LINE_SIZE,
 613 };
 614
 615 static const struct intel_watermark_params i945_wm_info = {
 616         .fifo_size = I945_FIFO_SIZE,
 617         .max_wm = I915_MAX_WM,
 618         .default_wm = 1,
 619         .guard_size = 2,
 620         .cacheline_size = I915_FIFO_LINE_SIZE,
 621 };
 622
 623 static const struct intel_watermark_params i915_wm_info = {
 624         .fifo_size = I915_FIFO_SIZE,
 625         .max_wm = I915_MAX_WM,
 626         .default_wm = 1,
 627         .guard_size = 2,
 628         .cacheline_size = I915_FIFO_LINE_SIZE,
 629 };
 630
 631 static const struct intel_watermark_params i830_a_wm_info = {
 632         .fifo_size = I855GM_FIFO_SIZE,
 633         .max_wm = I915_MAX_WM,
 634         .default_wm = 1,
 635         .guard_size = 2,
 636         .cacheline_size = I830_FIFO_LINE_SIZE,
 637 };
 638
 639 static const struct intel_watermark_params i830_bc_wm_info = {
 640         .fifo_size = I855GM_FIFO_SIZE,
 641         .max_wm = I915_MAX_WM/2,
 642         .default_wm = 1,
 643         .guard_size = 2,
 644         .cacheline_size = I830_FIFO_LINE_SIZE,
 645 };
 646
 647 static const struct intel_watermark_params i845_wm_info = {
 648         .fifo_size = I830_FIFO_SIZE,
 649         .max_wm = I915_MAX_WM,
 650         .default_wm = 1,
 651         .guard_size = 2,
 652         .cacheline_size = I830_FIFO_LINE_SIZE,
 653 };
 654
 655 /**
 656  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 657  * @pixel_rate: Pipe pixel rate in kHz
 658  * @cpp: Plane bytes per pixel
 659  * @latency: Memory wakeup latency in 0.1us units
 660  *
 661  * Compute the watermark using the method 1 or "small buffer"
 662  * formula. The caller may additonally add extra cachelines
 663  * to account for TLB misses and clock crossings.
 664  *
 665  * This method is concerned with the short term drain rate
 666  * of the FIFO, ie. it does not account for blanking periods
 667  * which would effectively reduce the average drain rate across
 668  * a longer period. The name "small" refers to the fact the
 669  * FIFO is relatively small compared to the amount of data
 670  * fetched.
 671  *
 672  * The FIFO level vs. time graph might look something like:
 673  *
 674  *   |\   |\
 675  *   | \  | \
 676  * __---__---__ (- plane active, _ blanking)
 677  * -> time
 678  *
 679  * or perhaps like this:
 680  *
 681  *   |\|\  |\|\
 682  * __----__----__ (- plane active, _ blanking)
 683  * -> time
 684  *
 685  * Returns:
 686  * The watermark in bytes
 687  */
 688 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 689                                      unsigned int cpp,
 690                                      unsigned int latency)
 691 {
 692         u64 ret;
 693
 694         ret = mul_u32_u32(pixel_rate, cpp * latency);
 695         ret = DIV_ROUND_UP_ULL(ret, 10000);
 696
 697         return ret;
 698 }
 699
 700 /**
 701  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 702  * @pixel_rate: Pipe pixel rate in kHz
 703  * @htotal: Pipe horizontal total
 704  * @width: Plane width in pixels
 705  * @cpp: Plane bytes per pixel
 706  * @latency: Memory wakeup latency in 0.1us units
 707  *
 708  * Compute the watermark using the method 2 or "large buffer"
 709  * formula. The caller may additonally add extra cachelines
 710  * to account for TLB misses and clock crossings.
 711  *
 712  * This method is concerned with the long term drain rate
 713  * of the FIFO, ie. it does account for blanking periods
 714  * which effectively reduce the average drain rate across
 715  * a longer period. The name "large" refers to the fact the
 716  * FIFO is relatively large compared to the amount of data
 717  * fetched.
 718  *
 719  * The FIFO level vs. time graph might look something like:
 720  *
 721  *    |\___       |\___
 722  *    |    \___   |    \___
 723  *    |        \  |        \
 724  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 725  * -> time
 726  *
 727  * Returns:
 728  * The watermark in bytes
 729  */
 730 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 731                                      unsigned int htotal,
 732                                      unsigned int width,
 733                                      unsigned int cpp,
 734                                      unsigned int latency)
 735 {
 736         unsigned int ret;
 737
 738         /*
 739          * FIXME remove once all users are computing
 740          * watermarks in the correct place.
 741          */
 742         if (WARN_ON_ONCE(htotal == 0))
 743                 htotal = 1;
 744
 745         ret = (latency * pixel_rate) / (htotal * 10000);
 746         ret = (ret + 1) * width * cpp;
 747
 748         return ret;
 749 }
 750
 751 /**
 752  * intel_calculate_wm - calculate watermark level
 753  * @pixel_rate: pixel clock
 754  * @wm: chip FIFO params
 755  * @fifo_size: size of the FIFO buffer
 756  * @cpp: bytes per pixel
 757  * @latency_ns: memory latency for the platform
 758  *
 759  * Calculate the watermark level (the level at which the display plane will
 760  * start fetching from memory again).  Each chip has a different display
 761  * FIFO size and allocation, so the caller needs to figure that out and pass
 762  * in the correct intel_watermark_params structure.
 763  *
 764  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 765  * on the pixel size.  When it reaches the watermark level, it'll start
 766  * fetching FIFO line sized based chunks from memory until the FIFO fills
 767  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 768  * will occur, and a display engine hang could result.
 769  */
 770 static unsigned int intel_calculate_wm(int pixel_rate,
 771                                        const struct intel_watermark_params *wm,
 772                                        int fifo_size, int cpp,
 773                                        unsigned int latency_ns)
 774 {
 775         int entries, wm_size;
 776
 777         /*
 778          * Note: we need to make sure we don't overflow for various clock &
 779          * latency values.
 780          * clocks go from a few thousand to several hundred thousand.
 781          * latency is usually a few thousand
 782          */
 783         entries = intel_wm_method1(pixel_rate, cpp,
 784                                    latency_ns / 100);
 785         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 786                 wm->guard_size;
 787         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 788
 789         wm_size = fifo_size - entries;
 790         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 791
 792         /* Don't promote wm_size to unsigned... */
 793         if (wm_size > wm->max_wm)
 794                 wm_size = wm->max_wm;
 795         if (wm_size <= 0)
 796                 wm_size = wm->default_wm;
 797
 798         /*
 799          * Bspec seems to indicate that the value shouldn't be lower than
 800          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 801          * Lets go for 8 which is the burst size since certain platforms
 802          * already use a hardcoded 8 (which is what the spec says should be
 803          * done).
 804          */
 805         if (wm_size <= 8)
 806                 wm_size = 8;
 807
 808         return wm_size;
 809 }
 810
 811 static bool is_disabling(int old, int new, int threshold)
 812 {
 813         return old >= threshold && new < threshold;
 814 }
 815
 816 static bool is_enabling(int old, int new, int threshold)
 817 {
 818         return old < threshold && new >= threshold;
 819 }
 820
 821 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 822 {
 823         return dev_priv->wm.max_level + 1;
 824 }
 825
 826 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 827                                    const struct intel_plane_state *plane_state)
 828 {
 829         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 830
 831         /* FIXME check the 'enable' instead */
 832         if (!crtc_state->hw.active)
 833                 return false;
 834
 835         /*
 836          * Treat cursor with fb as always visible since cursor updates
 837          * can happen faster than the vrefresh rate, and the current
 838          * watermark code doesn't handle that correctly. Cursor updates
 839          * which set/clear the fb or change the cursor size are going
 840          * to get throttled by intel_legacy_cursor_update() to work
 841          * around this problem with the watermark code.
 842          */
 843         if (plane->id == PLANE_CURSOR)
 844                 return plane_state->hw.fb != NULL;
 845         else
 846                 return plane_state->uapi.visible;
 847 }
 848
 849 static bool intel_crtc_active(struct intel_crtc *crtc)
 850 {
 851         /* Be paranoid as we can arrive here with only partial
 852          * state retrieved from the hardware during setup.
 853          *
 854          * We can ditch the adjusted_mode.crtc_clock check as soon
 855          * as Haswell has gained clock readout/fastboot support.
 856          *
 857          * We can ditch the crtc->primary->state->fb check as soon as we can
 858          * properly reconstruct framebuffers.
 859          *
 860          * FIXME: The intel_crtc->active here should be switched to
 861          * crtc->state->active once we have proper CRTC states wired up
 862          * for atomic.
 863          */
 864         return crtc->active && crtc->base.primary->state->fb &&
 865                 crtc->config->hw.adjusted_mode.crtc_clock;
 866 }
 867
 868 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 869 {
 870         struct intel_crtc *crtc, *enabled = NULL;
 871
 872         for_each_intel_crtc(&dev_priv->drm, crtc) {
 873                 if (intel_crtc_active(crtc)) {
 874                         if (enabled)
 875                                 return NULL;
 876                         enabled = crtc;
 877                 }
 878         }
 879
 880         return enabled;
 881 }
 882
 883 static void pnv_update_wm(struct intel_crtc *unused_crtc)
 884 {
 885         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 886         struct intel_crtc *crtc;
 887         const struct cxsr_latency *latency;
 888         u32 reg;
 889         unsigned int wm;
 890
 891         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 892                                          dev_priv->is_ddr3,
 893                                          dev_priv->fsb_freq,
 894                                          dev_priv->mem_freq);
 895         if (!latency) {
 896                 drm_dbg_kms(&dev_priv->drm,
 897                             "Unknown FSB/MEM found, disable CxSR\n");
 898                 intel_set_memory_cxsr(dev_priv, false);
 899                 return;
 900         }
 901
 902         crtc = single_enabled_crtc(dev_priv);
 903         if (crtc) {
 904                 const struct drm_display_mode *pipe_mode =
 905                         &crtc->config->hw.pipe_mode;
 906                 const struct drm_framebuffer *fb =
 907                         crtc->base.primary->state->fb;
 908                 int cpp = fb->format->cpp[0];
 909                 int clock = pipe_mode->crtc_clock;
 910
 911                 /* Display SR */
 912                 wm = intel_calculate_wm(clock, &pnv_display_wm,
 913                                         pnv_display_wm.fifo_size,
 914                                         cpp, latency->display_sr);
 915                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
 916                 reg &= ~DSPFW_SR_MASK;
 917                 reg |= FW_WM(wm, SR);
 918                 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
 919                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 920
 921                 /* cursor SR */
 922                 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
 923                                         pnv_display_wm.fifo_size,
 924                                         4, latency->cursor_sr);
 925                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 926                 reg &= ~DSPFW_CURSOR_SR_MASK;
 927                 reg |= FW_WM(wm, CURSOR_SR);
 928                 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
 929
 930                 /* Display HPLL off SR */
 931                 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
 932                                         pnv_display_hplloff_wm.fifo_size,
 933                                         cpp, latency->display_hpll_disable);
 934                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 935                 reg &= ~DSPFW_HPLL_SR_MASK;
 936                 reg |= FW_WM(wm, HPLL_SR);
 937                 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
 938
 939                 /* cursor HPLL off SR */
 940                 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
 941                                         pnv_display_hplloff_wm.fifo_size,
 942                                         4, latency->cursor_hpll_disable);
 943                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 944                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 945                 reg |= FW_WM(wm, HPLL_CURSOR);
 946                 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
 947                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 948
 949                 intel_set_memory_cxsr(dev_priv, true);
 950         } else {
 951                 intel_set_memory_cxsr(dev_priv, false);
 952         }
 953 }
 954
 955 /*
 956  * Documentation says:
 957  * "If the line size is small, the TLB fetches can get in the way of the
 958  *  data fetches, causing some lag in the pixel data return which is not
 959  *  accounted for in the above formulas. The following adjustment only
 960  *  needs to be applied if eight whole lines fit in the buffer at once.
 961  *  The WM is adjusted upwards by the difference between the FIFO size
 962  *  and the size of 8 whole lines. This adjustment is always performed
 963  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 964  */
 965 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 966 {
 967         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 968
 969         return max(0, tlb_miss);
 970 }
 971
 972 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 973                                 const struct g4x_wm_values *wm)
 974 {
 975         enum pipe pipe;
 976
 977         for_each_pipe(dev_priv, pipe)
 978                 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 979
 980         intel_uncore_write(&dev_priv->uncore, DSPFW1,
 981                    FW_WM(wm->sr.plane, SR) |
 982                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 983                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 984                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 985         intel_uncore_write(&dev_priv->uncore, DSPFW2,
 986                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 987                    FW_WM(wm->sr.fbc, FBC_SR) |
 988                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 989                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 990                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 991                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 992         intel_uncore_write(&dev_priv->uncore, DSPFW3,
 993                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 994                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 995                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 996                    FW_WM(wm->hpll.plane, HPLL_SR));
 997
 998         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
 999 }
1000
1001 #define FW_WM_VLV(value, plane) \
1002         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1003
1004 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1005                                 const struct vlv_wm_values *wm)
1006 {
1007         enum pipe pipe;
1008
1009         for_each_pipe(dev_priv, pipe) {
1010                 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1011
1012                 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1013                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1014                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1015                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1016                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1017         }
1018
1019         /*
1020          * Zero the (unused) WM1 watermarks, and also clear all the
1021          * high order bits so that there are no out of bounds values
1022          * present in the registers during the reprogramming.
1023          */
1024         intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1025         intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1026         intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1027         intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1028         intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1029
1030         intel_uncore_write(&dev_priv->uncore, DSPFW1,
1031                    FW_WM(wm->sr.plane, SR) |
1032                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1033                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1034                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1035         intel_uncore_write(&dev_priv->uncore, DSPFW2,
1036                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1037                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1038                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1039         intel_uncore_write(&dev_priv->uncore, DSPFW3,
1040                    FW_WM(wm->sr.cursor, CURSOR_SR));
1041
1042         if (IS_CHERRYVIEW(dev_priv)) {
1043                 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1044                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1045                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1046                 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1047                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1048                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1049                 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1050                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1051                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1052                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1053                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1054                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1055                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1056                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1057                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1058                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1059                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1060                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1061                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1062                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1063         } else {
1064                 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1065                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1066                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1067                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1068                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1069                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1070                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1071                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1072                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1073                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1074                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1075         }
1076
1077         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1078 }
1079
1080 #undef FW_WM_VLV
1081
1082 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1083 {
1084         /* all latencies in usec */
1085         dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1086         dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1087         dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1088
1089         dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1090 }
1091
1092 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1093 {
1094         /*
1095          * DSPCNTR[13] supposedly controls whether the
1096          * primary plane can use the FIFO space otherwise
1097          * reserved for the sprite plane. It's not 100% clear
1098          * what the actual FIFO size is, but it looks like we
1099          * can happily set both primary and sprite watermarks
1100          * up to 127 cachelines. So that would seem to mean
1101          * that either DSPCNTR[13] doesn't do anything, or that
1102          * the total FIFO is >= 256 cachelines in size. Either
1103          * way, we don't seem to have to worry about this
1104          * repartitioning as the maximum watermark value the
1105          * register can hold for each plane is lower than the
1106          * minimum FIFO size.
1107          */
1108         switch (plane_id) {
1109         case PLANE_CURSOR:
1110                 return 63;
1111         case PLANE_PRIMARY:
1112                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1113         case PLANE_SPRITE0:
1114                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1115         default:
1116                 MISSING_CASE(plane_id);
1117                 return 0;
1118         }
1119 }
1120
1121 static int g4x_fbc_fifo_size(int level)
1122 {
1123         switch (level) {
1124         case G4X_WM_LEVEL_SR:
1125                 return 7;
1126         case G4X_WM_LEVEL_HPLL:
1127                 return 15;
1128         default:
1129                 MISSING_CASE(level);
1130                 return 0;
1131         }
1132 }
1133
1134 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1135                           const struct intel_plane_state *plane_state,
1136                           int level)
1137 {
1138         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1139         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1140         const struct drm_display_mode *pipe_mode =
1141                 &crtc_state->hw.pipe_mode;
1142         unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1143         unsigned int clock, htotal, cpp, width, wm;
1144
1145         if (latency == 0)
1146                 return USHRT_MAX;
1147
1148         if (!intel_wm_plane_visible(crtc_state, plane_state))
1149                 return 0;
1150
1151         cpp = plane_state->hw.fb->format->cpp[0];
1152
1153         /*
1154          * Not 100% sure which way ELK should go here as the
1155          * spec only says CL/CTG should assume 32bpp and BW
1156          * doesn't need to. But as these things followed the
1157          * mobile vs. desktop lines on gen3 as well, let's
1158          * assume ELK doesn't need this.
1159          *
1160          * The spec also fails to list such a restriction for
1161          * the HPLL watermark, which seems a little strange.
1162          * Let's use 32bpp for the HPLL watermark as well.
1163          */
1164         if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1165             level != G4X_WM_LEVEL_NORMAL)
1166                 cpp = max(cpp, 4u);
1167
1168         clock = pipe_mode->crtc_clock;
1169         htotal = pipe_mode->crtc_htotal;
1170
1171         width = drm_rect_width(&plane_state->uapi.dst);
1172
1173         if (plane->id == PLANE_CURSOR) {
1174                 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1175         } else if (plane->id == PLANE_PRIMARY &&
1176                    level == G4X_WM_LEVEL_NORMAL) {
1177                 wm = intel_wm_method1(clock, cpp, latency);
1178         } else {
1179                 unsigned int small, large;
1180
1181                 small = intel_wm_method1(clock, cpp, latency);
1182                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1183
1184                 wm = min(small, large);
1185         }
1186
1187         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1188                               width, cpp);
1189
1190         wm = DIV_ROUND_UP(wm, 64) + 2;
1191
1192         return min_t(unsigned int, wm, USHRT_MAX);
1193 }
1194
1195 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1196                                  int level, enum plane_id plane_id, u16 value)
1197 {
1198         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1199         bool dirty = false;
1200
1201         for (; level < intel_wm_num_levels(dev_priv); level++) {
1202                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1203
1204                 dirty |= raw->plane[plane_id] != value;
1205                 raw->plane[plane_id] = value;
1206         }
1207
1208         return dirty;
1209 }
1210
1211 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1212                                int level, u16 value)
1213 {
1214         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1215         bool dirty = false;
1216
1217         /* NORMAL level doesn't have an FBC watermark */
1218         level = max(level, G4X_WM_LEVEL_SR);
1219
1220         for (; level < intel_wm_num_levels(dev_priv); level++) {
1221                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1222
1223                 dirty |= raw->fbc != value;
1224                 raw->fbc = value;
1225         }
1226
1227         return dirty;
1228 }
1229
1230 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1231                               const struct intel_plane_state *plane_state,
1232                               u32 pri_val);
1233
1234 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1235                                      const struct intel_plane_state *plane_state)
1236 {
1237         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1238         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1239         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1240         enum plane_id plane_id = plane->id;
1241         bool dirty = false;
1242         int level;
1243
1244         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1245                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1246                 if (plane_id == PLANE_PRIMARY)
1247                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1248                 goto out;
1249         }
1250
1251         for (level = 0; level < num_levels; level++) {
1252                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1253                 int wm, max_wm;
1254
1255                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1256                 max_wm = g4x_plane_fifo_size(plane_id, level);
1257
1258                 if (wm > max_wm)
1259                         break;
1260
1261                 dirty |= raw->plane[plane_id] != wm;
1262                 raw->plane[plane_id] = wm;
1263
1264                 if (plane_id != PLANE_PRIMARY ||
1265                     level == G4X_WM_LEVEL_NORMAL)
1266                         continue;
1267
1268                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1269                                         raw->plane[plane_id]);
1270                 max_wm = g4x_fbc_fifo_size(level);
1271
1272                 /*
1273                  * FBC wm is not mandatory as we
1274                  * can always just disable its use.
1275                  */
1276                 if (wm > max_wm)
1277                         wm = USHRT_MAX;
1278
1279                 dirty |= raw->fbc != wm;
1280                 raw->fbc = wm;
1281         }
1282
1283         /* mark watermarks as invalid */
1284         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1285
1286         if (plane_id == PLANE_PRIMARY)
1287                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1288
1289  out:
1290         if (dirty) {
1291                 drm_dbg_kms(&dev_priv->drm,
1292                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1293                             plane->base.name,
1294                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1295                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1296                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1297
1298                 if (plane_id == PLANE_PRIMARY)
1299                         drm_dbg_kms(&dev_priv->drm,
1300                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1301                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1302                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1303         }
1304
1305         return dirty;
1306 }
1307
1308 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1309                                       enum plane_id plane_id, int level)
1310 {
1311         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1312
1313         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1314 }
1315
1316 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1317                                      int level)
1318 {
1319         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1320
1321         if (level > dev_priv->wm.max_level)
1322                 return false;
1323
1324         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1325                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1326                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1327 }
1328
1329 /* mark all levels starting from 'level' as invalid */
1330 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1331                                struct g4x_wm_state *wm_state, int level)
1332 {
1333         if (level <= G4X_WM_LEVEL_NORMAL) {
1334                 enum plane_id plane_id;
1335
1336                 for_each_plane_id_on_crtc(crtc, plane_id)
1337                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1338         }
1339
1340         if (level <= G4X_WM_LEVEL_SR) {
1341                 wm_state->cxsr = false;
1342                 wm_state->sr.cursor = USHRT_MAX;
1343                 wm_state->sr.plane = USHRT_MAX;
1344                 wm_state->sr.fbc = USHRT_MAX;
1345         }
1346
1347         if (level <= G4X_WM_LEVEL_HPLL) {
1348                 wm_state->hpll_en = false;
1349                 wm_state->hpll.cursor = USHRT_MAX;
1350                 wm_state->hpll.plane = USHRT_MAX;
1351                 wm_state->hpll.fbc = USHRT_MAX;
1352         }
1353 }
1354
1355 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1356                                int level)
1357 {
1358         if (level < G4X_WM_LEVEL_SR)
1359                 return false;
1360
1361         if (level >= G4X_WM_LEVEL_SR &&
1362             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1363                 return false;
1364
1365         if (level >= G4X_WM_LEVEL_HPLL &&
1366             wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1367                 return false;
1368
1369         return true;
1370 }
1371
1372 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1373 {
1374         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1375         struct intel_atomic_state *state =
1376                 to_intel_atomic_state(crtc_state->uapi.state);
1377         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1378         int num_active_planes = hweight8(crtc_state->active_planes &
1379                                          ~BIT(PLANE_CURSOR));
1380         const struct g4x_pipe_wm *raw;
1381         const struct intel_plane_state *old_plane_state;
1382         const struct intel_plane_state *new_plane_state;
1383         struct intel_plane *plane;
1384         enum plane_id plane_id;
1385         int i, level;
1386         unsigned int dirty = 0;
1387
1388         for_each_oldnew_intel_plane_in_state(state, plane,
1389                                              old_plane_state,
1390                                              new_plane_state, i) {
1391                 if (new_plane_state->hw.crtc != &crtc->base &&
1392                     old_plane_state->hw.crtc != &crtc->base)
1393                         continue;
1394
1395                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1396                         dirty |= BIT(plane->id);
1397         }
1398
1399         if (!dirty)
1400                 return 0;
1401
1402         level = G4X_WM_LEVEL_NORMAL;
1403         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1404                 goto out;
1405
1406         raw = &crtc_state->wm.g4x.raw[level];
1407         for_each_plane_id_on_crtc(crtc, plane_id)
1408                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1409
1410         level = G4X_WM_LEVEL_SR;
1411         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1412                 goto out;
1413
1414         raw = &crtc_state->wm.g4x.raw[level];
1415         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1416         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1417         wm_state->sr.fbc = raw->fbc;
1418
1419         wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1420
1421         level = G4X_WM_LEVEL_HPLL;
1422         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1423                 goto out;
1424
1425         raw = &crtc_state->wm.g4x.raw[level];
1426         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1427         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1428         wm_state->hpll.fbc = raw->fbc;
1429
1430         wm_state->hpll_en = wm_state->cxsr;
1431
1432         level++;
1433
1434  out:
1435         if (level == G4X_WM_LEVEL_NORMAL)
1436                 return -EINVAL;
1437
1438         /* invalidate the higher levels */
1439         g4x_invalidate_wms(crtc, wm_state, level);
1440
1441         /*
1442          * Determine if the FBC watermark(s) can be used. IF
1443          * this isn't the case we prefer to disable the FBC
1444          * watermark(s) rather than disable the SR/HPLL
1445          * level(s) entirely. 'level-1' is the highest valid
1446          * level here.
1447          */
1448         wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1449
1450         return 0;
1451 }
1452
1453 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1454 {
1455         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1456         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1457         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1458         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1459         struct intel_atomic_state *intel_state =
1460                 to_intel_atomic_state(new_crtc_state->uapi.state);
1461         const struct intel_crtc_state *old_crtc_state =
1462                 intel_atomic_get_old_crtc_state(intel_state, crtc);
1463         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1464         enum plane_id plane_id;
1465
1466         if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1467                 *intermediate = *optimal;
1468
1469                 intermediate->cxsr = false;
1470                 intermediate->hpll_en = false;
1471                 goto out;
1472         }
1473
1474         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1475                 !new_crtc_state->disable_cxsr;
1476         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1477                 !new_crtc_state->disable_cxsr;
1478         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1479
1480         for_each_plane_id_on_crtc(crtc, plane_id) {
1481                 intermediate->wm.plane[plane_id] =
1482                         max(optimal->wm.plane[plane_id],
1483                             active->wm.plane[plane_id]);
1484
1485                 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1486                             g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1487         }
1488
1489         intermediate->sr.plane = max(optimal->sr.plane,
1490                                      active->sr.plane);
1491         intermediate->sr.cursor = max(optimal->sr.cursor,
1492                                       active->sr.cursor);
1493         intermediate->sr.fbc = max(optimal->sr.fbc,
1494                                    active->sr.fbc);
1495
1496         intermediate->hpll.plane = max(optimal->hpll.plane,
1497                                        active->hpll.plane);
1498         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1499                                         active->hpll.cursor);
1500         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1501                                      active->hpll.fbc);
1502
1503         drm_WARN_ON(&dev_priv->drm,
1504                     (intermediate->sr.plane >
1505                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1506                      intermediate->sr.cursor >
1507                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1508                     intermediate->cxsr);
1509         drm_WARN_ON(&dev_priv->drm,
1510                     (intermediate->sr.plane >
1511                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1512                      intermediate->sr.cursor >
1513                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1514                     intermediate->hpll_en);
1515
1516         drm_WARN_ON(&dev_priv->drm,
1517                     intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1518                     intermediate->fbc_en && intermediate->cxsr);
1519         drm_WARN_ON(&dev_priv->drm,
1520                     intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1521                     intermediate->fbc_en && intermediate->hpll_en);
1522
1523 out:
1524         /*
1525          * If our intermediate WM are identical to the final WM, then we can
1526          * omit the post-vblank programming; only update if it's different.
1527          */
1528         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1529                 new_crtc_state->wm.need_postvbl_update = true;
1530
1531         return 0;
1532 }
1533
1534 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1535                          struct g4x_wm_values *wm)
1536 {
1537         struct intel_crtc *crtc;
1538         int num_active_pipes = 0;
1539
1540         wm->cxsr = true;
1541         wm->hpll_en = true;
1542         wm->fbc_en = true;
1543
1544         for_each_intel_crtc(&dev_priv->drm, crtc) {
1545                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1546
1547                 if (!crtc->active)
1548                         continue;
1549
1550                 if (!wm_state->cxsr)
1551                         wm->cxsr = false;
1552                 if (!wm_state->hpll_en)
1553                         wm->hpll_en = false;
1554                 if (!wm_state->fbc_en)
1555                         wm->fbc_en = false;
1556
1557                 num_active_pipes++;
1558         }
1559
1560         if (num_active_pipes != 1) {
1561                 wm->cxsr = false;
1562                 wm->hpll_en = false;
1563                 wm->fbc_en = false;
1564         }
1565
1566         for_each_intel_crtc(&dev_priv->drm, crtc) {
1567                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1568                 enum pipe pipe = crtc->pipe;
1569
1570                 wm->pipe[pipe] = wm_state->wm;
1571                 if (crtc->active && wm->cxsr)
1572                         wm->sr = wm_state->sr;
1573                 if (crtc->active && wm->hpll_en)
1574                         wm->hpll = wm_state->hpll;
1575         }
1576 }
1577
1578 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1579 {
1580         struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1581         struct g4x_wm_values new_wm = {};
1582
1583         g4x_merge_wm(dev_priv, &new_wm);
1584
1585         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1586                 return;
1587
1588         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1589                 _intel_set_memory_cxsr(dev_priv, false);
1590
1591         g4x_write_wm_values(dev_priv, &new_wm);
1592
1593         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1594                 _intel_set_memory_cxsr(dev_priv, true);
1595
1596         *old_wm = new_wm;
1597 }
1598
1599 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1600                                    struct intel_crtc *crtc)
1601 {
1602         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1603         const struct intel_crtc_state *crtc_state =
1604                 intel_atomic_get_new_crtc_state(state, crtc);
1605
1606         mutex_lock(&dev_priv->wm.wm_mutex);
1607         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1608         g4x_program_watermarks(dev_priv);
1609         mutex_unlock(&dev_priv->wm.wm_mutex);
1610 }
1611
1612 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1613                                     struct intel_crtc *crtc)
1614 {
1615         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1616         const struct intel_crtc_state *crtc_state =
1617                 intel_atomic_get_new_crtc_state(state, crtc);
1618
1619         if (!crtc_state->wm.need_postvbl_update)
1620                 return;
1621
1622         mutex_lock(&dev_priv->wm.wm_mutex);
1623         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1624         g4x_program_watermarks(dev_priv);
1625         mutex_unlock(&dev_priv->wm.wm_mutex);
1626 }
1627
1628 /* latency must be in 0.1us units. */
1629 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1630                                    unsigned int htotal,
1631                                    unsigned int width,
1632                                    unsigned int cpp,
1633                                    unsigned int latency)
1634 {
1635         unsigned int ret;
1636
1637         ret = intel_wm_method2(pixel_rate, htotal,
1638                                width, cpp, latency);
1639         ret = DIV_ROUND_UP(ret, 64);
1640
1641         return ret;
1642 }
1643
1644 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1645 {
1646         /* all latencies in usec */
1647         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1648
1649         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1650
1651         if (IS_CHERRYVIEW(dev_priv)) {
1652                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1653                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1654
1655                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1656         }
1657 }
1658
1659 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1660                                 const struct intel_plane_state *plane_state,
1661                                 int level)
1662 {
1663         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1664         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1665         const struct drm_display_mode *pipe_mode =
1666                 &crtc_state->hw.pipe_mode;
1667         unsigned int clock, htotal, cpp, width, wm;
1668
1669         if (dev_priv->wm.pri_latency[level] == 0)
1670                 return USHRT_MAX;
1671
1672         if (!intel_wm_plane_visible(crtc_state, plane_state))
1673                 return 0;
1674
1675         cpp = plane_state->hw.fb->format->cpp[0];
1676         clock = pipe_mode->crtc_clock;
1677         htotal = pipe_mode->crtc_htotal;
1678         width = crtc_state->pipe_src_w;
1679
1680         if (plane->id == PLANE_CURSOR) {
1681                 /*
1682                  * FIXME the formula gives values that are
1683                  * too big for the cursor FIFO, and hence we
1684                  * would never be able to use cursors. For
1685                  * now just hardcode the watermark.
1686                  */
1687                 wm = 63;
1688         } else {
1689                 wm = vlv_wm_method2(clock, htotal, width, cpp,
1690                                     dev_priv->wm.pri_latency[level] * 10);
1691         }
1692
1693         return min_t(unsigned int, wm, USHRT_MAX);
1694 }
1695
1696 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1697 {
1698         return (active_planes & (BIT(PLANE_SPRITE0) |
1699                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1700 }
1701
1702 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1703 {
1704         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1705         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1706         const struct g4x_pipe_wm *raw =
1707                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1708         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1709         unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1710         int num_active_planes = hweight8(active_planes);
1711         const int fifo_size = 511;
1712         int fifo_extra, fifo_left = fifo_size;
1713         int sprite0_fifo_extra = 0;
1714         unsigned int total_rate;
1715         enum plane_id plane_id;
1716
1717         /*
1718          * When enabling sprite0 after sprite1 has already been enabled
1719          * we tend to get an underrun unless sprite0 already has some
1720          * FIFO space allcoated. Hence we always allocate at least one
1721          * cacheline for sprite0 whenever sprite1 is enabled.
1722          *
1723          * All other plane enable sequences appear immune to this problem.
1724          */
1725         if (vlv_need_sprite0_fifo_workaround(active_planes))
1726                 sprite0_fifo_extra = 1;
1727
1728         total_rate = raw->plane[PLANE_PRIMARY] +
1729                 raw->plane[PLANE_SPRITE0] +
1730                 raw->plane[PLANE_SPRITE1] +
1731                 sprite0_fifo_extra;
1732
1733         if (total_rate > fifo_size)
1734                 return -EINVAL;
1735
1736         if (total_rate == 0)
1737                 total_rate = 1;
1738
1739         for_each_plane_id_on_crtc(crtc, plane_id) {
1740                 unsigned int rate;
1741
1742                 if ((active_planes & BIT(plane_id)) == 0) {
1743                         fifo_state->plane[plane_id] = 0;
1744                         continue;
1745                 }
1746
1747                 rate = raw->plane[plane_id];
1748                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1749                 fifo_left -= fifo_state->plane[plane_id];
1750         }
1751
1752         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1753         fifo_left -= sprite0_fifo_extra;
1754
1755         fifo_state->plane[PLANE_CURSOR] = 63;
1756
1757         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1758
1759         /* spread the remainder evenly */
1760         for_each_plane_id_on_crtc(crtc, plane_id) {
1761                 int plane_extra;
1762
1763                 if (fifo_left == 0)
1764                         break;
1765
1766                 if ((active_planes & BIT(plane_id)) == 0)
1767                         continue;
1768
1769                 plane_extra = min(fifo_extra, fifo_left);
1770                 fifo_state->plane[plane_id] += plane_extra;
1771                 fifo_left -= plane_extra;
1772         }
1773
1774         drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1775
1776         /* give it all to the first plane if none are active */
1777         if (active_planes == 0) {
1778                 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1779                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1780         }
1781
1782         return 0;
1783 }
1784
1785 /* mark all levels starting from 'level' as invalid */
1786 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1787                                struct vlv_wm_state *wm_state, int level)
1788 {
1789         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1790
1791         for (; level < intel_wm_num_levels(dev_priv); level++) {
1792                 enum plane_id plane_id;
1793
1794                 for_each_plane_id_on_crtc(crtc, plane_id)
1795                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1796
1797                 wm_state->sr[level].cursor = USHRT_MAX;
1798                 wm_state->sr[level].plane = USHRT_MAX;
1799         }
1800 }
1801
1802 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1803 {
1804         if (wm > fifo_size)
1805                 return USHRT_MAX;
1806         else
1807                 return fifo_size - wm;
1808 }
1809
1810 /*
1811  * Starting from 'level' set all higher
1812  * levels to 'value' in the "raw" watermarks.
1813  */
1814 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1815                                  int level, enum plane_id plane_id, u16 value)
1816 {
1817         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1818         int num_levels = intel_wm_num_levels(dev_priv);
1819         bool dirty = false;
1820
1821         for (; level < num_levels; level++) {
1822                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1823
1824                 dirty |= raw->plane[plane_id] != value;
1825                 raw->plane[plane_id] = value;
1826         }
1827
1828         return dirty;
1829 }
1830
1831 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1832                                      const struct intel_plane_state *plane_state)
1833 {
1834         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1835         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1836         enum plane_id plane_id = plane->id;
1837         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1838         int level;
1839         bool dirty = false;
1840
1841         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1842                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1843                 goto out;
1844         }
1845
1846         for (level = 0; level < num_levels; level++) {
1847                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1848                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1849                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1850
1851                 if (wm > max_wm)
1852                         break;
1853
1854                 dirty |= raw->plane[plane_id] != wm;
1855                 raw->plane[plane_id] = wm;
1856         }
1857
1858         /* mark all higher levels as invalid */
1859         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1860
1861 out:
1862         if (dirty)
1863                 drm_dbg_kms(&dev_priv->drm,
1864                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1865                             plane->base.name,
1866                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1867                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1868                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1869
1870         return dirty;
1871 }
1872
1873 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1874                                       enum plane_id plane_id, int level)
1875 {
1876         const struct g4x_pipe_wm *raw =
1877                 &crtc_state->wm.vlv.raw[level];
1878         const struct vlv_fifo_state *fifo_state =
1879                 &crtc_state->wm.vlv.fifo_state;
1880
1881         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1882 }
1883
1884 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1885 {
1886         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1887                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1888                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1889                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1890 }
1891
1892 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1893 {
1894         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1895         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1896         struct intel_atomic_state *state =
1897                 to_intel_atomic_state(crtc_state->uapi.state);
1898         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1899         const struct vlv_fifo_state *fifo_state =
1900                 &crtc_state->wm.vlv.fifo_state;
1901         int num_active_planes = hweight8(crtc_state->active_planes &
1902                                          ~BIT(PLANE_CURSOR));
1903         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1904         const struct intel_plane_state *old_plane_state;
1905         const struct intel_plane_state *new_plane_state;
1906         struct intel_plane *plane;
1907         enum plane_id plane_id;
1908         int level, ret, i;
1909         unsigned int dirty = 0;
1910
1911         for_each_oldnew_intel_plane_in_state(state, plane,
1912                                              old_plane_state,
1913                                              new_plane_state, i) {
1914                 if (new_plane_state->hw.crtc != &crtc->base &&
1915                     old_plane_state->hw.crtc != &crtc->base)
1916                         continue;
1917
1918                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1919                         dirty |= BIT(plane->id);
1920         }
1921
1922         /*
1923          * DSPARB registers may have been reset due to the
1924          * power well being turned off. Make sure we restore
1925          * them to a consistent state even if no primary/sprite
1926          * planes are initially active.
1927          */
1928         if (needs_modeset)
1929                 crtc_state->fifo_changed = true;
1930
1931         if (!dirty)
1932                 return 0;
1933
1934         /* cursor changes don't warrant a FIFO recompute */
1935         if (dirty & ~BIT(PLANE_CURSOR)) {
1936                 const struct intel_crtc_state *old_crtc_state =
1937                         intel_atomic_get_old_crtc_state(state, crtc);
1938                 const struct vlv_fifo_state *old_fifo_state =
1939                         &old_crtc_state->wm.vlv.fifo_state;
1940
1941                 ret = vlv_compute_fifo(crtc_state);
1942                 if (ret)
1943                         return ret;
1944
1945                 if (needs_modeset ||
1946                     memcmp(old_fifo_state, fifo_state,
1947                            sizeof(*fifo_state)) != 0)
1948                         crtc_state->fifo_changed = true;
1949         }
1950
1951         /* initially allow all levels */
1952         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1953         /*
1954          * Note that enabling cxsr with no primary/sprite planes
1955          * enabled can wedge the pipe. Hence we only allow cxsr
1956          * with exactly one enabled primary/sprite plane.
1957          */
1958         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1959
1960         for (level = 0; level < wm_state->num_levels; level++) {
1961                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1962                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1963
1964                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1965                         break;
1966
1967                 for_each_plane_id_on_crtc(crtc, plane_id) {
1968                         wm_state->wm[level].plane[plane_id] =
1969                                 vlv_invert_wm_value(raw->plane[plane_id],
1970                                                     fifo_state->plane[plane_id]);
1971                 }
1972
1973                 wm_state->sr[level].plane =
1974                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1975                                                  raw->plane[PLANE_SPRITE0],
1976                                                  raw->plane[PLANE_SPRITE1]),
1977                                             sr_fifo_size);
1978
1979                 wm_state->sr[level].cursor =
1980                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1981                                             63);
1982         }
1983
1984         if (level == 0)
1985                 return -EINVAL;
1986
1987         /* limit to only levels we can actually handle */
1988         wm_state->num_levels = level;
1989
1990         /* invalidate the higher levels */
1991         vlv_invalidate_wms(crtc, wm_state, level);
1992
1993         return 0;
1994 }
1995
1996 #define VLV_FIFO(plane, value) \
1997         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1998
1999 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2000                                    struct intel_crtc *crtc)
2001 {
2002         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2003         struct intel_uncore *uncore = &dev_priv->uncore;
2004         const struct intel_crtc_state *crtc_state =
2005                 intel_atomic_get_new_crtc_state(state, crtc);
2006         const struct vlv_fifo_state *fifo_state =
2007                 &crtc_state->wm.vlv.fifo_state;
2008         int sprite0_start, sprite1_start, fifo_size;
2009         u32 dsparb, dsparb2, dsparb3;
2010
2011         if (!crtc_state->fifo_changed)
2012                 return;
2013
2014         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2015         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2016         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2017
2018         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2019         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2020
2021         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2022
2023         /*
2024          * uncore.lock serves a double purpose here. It allows us to
2025          * use the less expensive I915_{READ,WRITE}_FW() functions, and
2026          * it protects the DSPARB registers from getting clobbered by
2027          * parallel updates from multiple pipes.
2028          *
2029          * intel_pipe_update_start() has already disabled interrupts
2030          * for us, so a plain spin_lock() is sufficient here.
2031          */
2032         spin_lock(&uncore->lock);
2033
2034         switch (crtc->pipe) {
2035         case PIPE_A:
2036                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2037                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2038
2039                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2040                             VLV_FIFO(SPRITEB, 0xff));
2041                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2042                            VLV_FIFO(SPRITEB, sprite1_start));
2043
2044                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2045                              VLV_FIFO(SPRITEB_HI, 0x1));
2046                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2047                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2048
2049                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2050                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2051                 break;
2052         case PIPE_B:
2053                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2054                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2055
2056                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2057                             VLV_FIFO(SPRITED, 0xff));
2058                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2059                            VLV_FIFO(SPRITED, sprite1_start));
2060
2061                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2062                              VLV_FIFO(SPRITED_HI, 0xff));
2063                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2064                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2065
2066                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2067                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2068                 break;
2069         case PIPE_C:
2070                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2071                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2072
2073                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2074                              VLV_FIFO(SPRITEF, 0xff));
2075                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2076                             VLV_FIFO(SPRITEF, sprite1_start));
2077
2078                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2079                              VLV_FIFO(SPRITEF_HI, 0xff));
2080                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2081                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2082
2083                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2084                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2085                 break;
2086         default:
2087                 break;
2088         }
2089
2090         intel_uncore_posting_read_fw(uncore, DSPARB);
2091
2092         spin_unlock(&uncore->lock);
2093 }
2094
2095 #undef VLV_FIFO
2096
2097 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2098 {
2099         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2100         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2101         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2102         struct intel_atomic_state *intel_state =
2103                 to_intel_atomic_state(new_crtc_state->uapi.state);
2104         const struct intel_crtc_state *old_crtc_state =
2105                 intel_atomic_get_old_crtc_state(intel_state, crtc);
2106         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2107         int level;
2108
2109         if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2110                 *intermediate = *optimal;
2111
2112                 intermediate->cxsr = false;
2113                 goto out;
2114         }
2115
2116         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2117         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2118                 !new_crtc_state->disable_cxsr;
2119
2120         for (level = 0; level < intermediate->num_levels; level++) {
2121                 enum plane_id plane_id;
2122
2123                 for_each_plane_id_on_crtc(crtc, plane_id) {
2124                         intermediate->wm[level].plane[plane_id] =
2125                                 min(optimal->wm[level].plane[plane_id],
2126                                     active->wm[level].plane[plane_id]);
2127                 }
2128
2129                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2130                                                     active->sr[level].plane);
2131                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2132                                                      active->sr[level].cursor);
2133         }
2134
2135         vlv_invalidate_wms(crtc, intermediate, level);
2136
2137 out:
2138         /*
2139          * If our intermediate WM are identical to the final WM, then we can
2140          * omit the post-vblank programming; only update if it's different.
2141          */
2142         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2143                 new_crtc_state->wm.need_postvbl_update = true;
2144
2145         return 0;
2146 }
2147
2148 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2149                          struct vlv_wm_values *wm)
2150 {
2151         struct intel_crtc *crtc;
2152         int num_active_pipes = 0;
2153
2154         wm->level = dev_priv->wm.max_level;
2155         wm->cxsr = true;
2156
2157         for_each_intel_crtc(&dev_priv->drm, crtc) {
2158                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2159
2160                 if (!crtc->active)
2161                         continue;
2162
2163                 if (!wm_state->cxsr)
2164                         wm->cxsr = false;
2165
2166                 num_active_pipes++;
2167                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2168         }
2169
2170         if (num_active_pipes != 1)
2171                 wm->cxsr = false;
2172
2173         if (num_active_pipes > 1)
2174                 wm->level = VLV_WM_LEVEL_PM2;
2175
2176         for_each_intel_crtc(&dev_priv->drm, crtc) {
2177                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2178                 enum pipe pipe = crtc->pipe;
2179
2180                 wm->pipe[pipe] = wm_state->wm[wm->level];
2181                 if (crtc->active && wm->cxsr)
2182                         wm->sr = wm_state->sr[wm->level];
2183
2184                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2185                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2186                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2187                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2188         }
2189 }
2190
2191 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2192 {
2193         struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2194         struct vlv_wm_values new_wm = {};
2195
2196         vlv_merge_wm(dev_priv, &new_wm);
2197
2198         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2199                 return;
2200
2201         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2202                 chv_set_memory_dvfs(dev_priv, false);
2203
2204         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2205                 chv_set_memory_pm5(dev_priv, false);
2206
2207         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2208                 _intel_set_memory_cxsr(dev_priv, false);
2209
2210         vlv_write_wm_values(dev_priv, &new_wm);
2211
2212         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2213                 _intel_set_memory_cxsr(dev_priv, true);
2214
2215         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2216                 chv_set_memory_pm5(dev_priv, true);
2217
2218         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2219                 chv_set_memory_dvfs(dev_priv, true);
2220
2221         *old_wm = new_wm;
2222 }
2223
2224 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2225                                    struct intel_crtc *crtc)
2226 {
2227         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2228         const struct intel_crtc_state *crtc_state =
2229                 intel_atomic_get_new_crtc_state(state, crtc);
2230
2231         mutex_lock(&dev_priv->wm.wm_mutex);
2232         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2233         vlv_program_watermarks(dev_priv);
2234         mutex_unlock(&dev_priv->wm.wm_mutex);
2235 }
2236
2237 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2238                                     struct intel_crtc *crtc)
2239 {
2240         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2241         const struct intel_crtc_state *crtc_state =
2242                 intel_atomic_get_new_crtc_state(state, crtc);
2243
2244         if (!crtc_state->wm.need_postvbl_update)
2245                 return;
2246
2247         mutex_lock(&dev_priv->wm.wm_mutex);
2248         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2249         vlv_program_watermarks(dev_priv);
2250         mutex_unlock(&dev_priv->wm.wm_mutex);
2251 }
2252
2253 static void i965_update_wm(struct intel_crtc *unused_crtc)
2254 {
2255         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2256         struct intel_crtc *crtc;
2257         int srwm = 1;
2258         int cursor_sr = 16;
2259         bool cxsr_enabled;
2260
2261         /* Calc sr entries for one plane configs */
2262         crtc = single_enabled_crtc(dev_priv);
2263         if (crtc) {
2264                 /* self-refresh has much higher latency */
2265                 static const int sr_latency_ns = 12000;
2266                 const struct drm_display_mode *pipe_mode =
2267                         &crtc->config->hw.pipe_mode;
2268                 const struct drm_framebuffer *fb =
2269                         crtc->base.primary->state->fb;
2270                 int clock = pipe_mode->crtc_clock;
2271                 int htotal = pipe_mode->crtc_htotal;
2272                 int hdisplay = crtc->config->pipe_src_w;
2273                 int cpp = fb->format->cpp[0];
2274                 int entries;
2275
2276                 entries = intel_wm_method2(clock, htotal,
2277                                            hdisplay, cpp, sr_latency_ns / 100);
2278                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2279                 srwm = I965_FIFO_SIZE - entries;
2280                 if (srwm < 0)
2281                         srwm = 1;
2282                 srwm &= 0x1ff;
2283                 drm_dbg_kms(&dev_priv->drm,
2284                             "self-refresh entries: %d, wm: %d\n",
2285                             entries, srwm);
2286
2287                 entries = intel_wm_method2(clock, htotal,
2288                                            crtc->base.cursor->state->crtc_w, 4,
2289                                            sr_latency_ns / 100);
2290                 entries = DIV_ROUND_UP(entries,
2291                                        i965_cursor_wm_info.cacheline_size) +
2292                         i965_cursor_wm_info.guard_size;
2293
2294                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2295                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2296                         cursor_sr = i965_cursor_wm_info.max_wm;
2297
2298                 drm_dbg_kms(&dev_priv->drm,
2299                             "self-refresh watermark: display plane %d "
2300                             "cursor %d\n", srwm, cursor_sr);
2301
2302                 cxsr_enabled = true;
2303         } else {
2304                 cxsr_enabled = false;
2305                 /* Turn off self refresh if both pipes are enabled */
2306                 intel_set_memory_cxsr(dev_priv, false);
2307         }
2308
2309         drm_dbg_kms(&dev_priv->drm,
2310                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2311                     srwm);
2312
2313         /* 965 has limitations... */
2314         intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2315                    FW_WM(8, CURSORB) |
2316                    FW_WM(8, PLANEB) |
2317                    FW_WM(8, PLANEA));
2318         intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2319                    FW_WM(8, PLANEC_OLD));
2320         /* update cursor SR watermark */
2321         intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2322
2323         if (cxsr_enabled)
2324                 intel_set_memory_cxsr(dev_priv, true);
2325 }
2326
2327 #undef FW_WM
2328
2329 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2330 {
2331         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2332         const struct intel_watermark_params *wm_info;
2333         u32 fwater_lo;
2334         u32 fwater_hi;
2335         int cwm, srwm = 1;
2336         int fifo_size;
2337         int planea_wm, planeb_wm;
2338         struct intel_crtc *crtc, *enabled = NULL;
2339
2340         if (IS_I945GM(dev_priv))
2341                 wm_info = &i945_wm_info;
2342         else if (!IS_DISPLAY_VER(dev_priv, 2))
2343                 wm_info = &i915_wm_info;
2344         else
2345                 wm_info = &i830_a_wm_info;
2346
2347         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2348         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2349         if (intel_crtc_active(crtc)) {
2350                 const struct drm_display_mode *pipe_mode =
2351                         &crtc->config->hw.pipe_mode;
2352                 const struct drm_framebuffer *fb =
2353                         crtc->base.primary->state->fb;
2354                 int cpp;
2355
2356                 if (IS_DISPLAY_VER(dev_priv, 2))
2357                         cpp = 4;
2358                 else
2359                         cpp = fb->format->cpp[0];
2360
2361                 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2362                                                wm_info, fifo_size, cpp,
2363                                                pessimal_latency_ns);
2364                 enabled = crtc;
2365         } else {
2366                 planea_wm = fifo_size - wm_info->guard_size;
2367                 if (planea_wm > (long)wm_info->max_wm)
2368                         planea_wm = wm_info->max_wm;
2369         }
2370
2371         if (IS_DISPLAY_VER(dev_priv, 2))
2372                 wm_info = &i830_bc_wm_info;
2373
2374         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2375         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2376         if (intel_crtc_active(crtc)) {
2377                 const struct drm_display_mode *pipe_mode =
2378                         &crtc->config->hw.pipe_mode;
2379                 const struct drm_framebuffer *fb =
2380                         crtc->base.primary->state->fb;
2381                 int cpp;
2382
2383                 if (IS_DISPLAY_VER(dev_priv, 2))
2384                         cpp = 4;
2385                 else
2386                         cpp = fb->format->cpp[0];
2387
2388                 planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2389                                                wm_info, fifo_size, cpp,
2390                                                pessimal_latency_ns);
2391                 if (enabled == NULL)
2392                         enabled = crtc;
2393                 else
2394                         enabled = NULL;
2395         } else {
2396                 planeb_wm = fifo_size - wm_info->guard_size;
2397                 if (planeb_wm > (long)wm_info->max_wm)
2398                         planeb_wm = wm_info->max_wm;
2399         }
2400
2401         drm_dbg_kms(&dev_priv->drm,
2402                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2403
2404         if (IS_I915GM(dev_priv) && enabled) {
2405                 struct drm_i915_gem_object *obj;
2406
2407                 obj = intel_fb_obj(enabled->base.primary->state->fb);
2408
2409                 /* self-refresh seems busted with untiled */
2410                 if (!i915_gem_object_is_tiled(obj))
2411                         enabled = NULL;
2412         }
2413
2414         /*
2415          * Overlay gets an aggressive default since video jitter is bad.
2416          */
2417         cwm = 2;
2418
2419         /* Play safe and disable self-refresh before adjusting watermarks. */
2420         intel_set_memory_cxsr(dev_priv, false);
2421
2422         /* Calc sr entries for one plane configs */
2423         if (HAS_FW_BLC(dev_priv) && enabled) {
2424                 /* self-refresh has much higher latency */
2425                 static const int sr_latency_ns = 6000;
2426                 const struct drm_display_mode *pipe_mode =
2427                         &enabled->config->hw.pipe_mode;
2428                 const struct drm_framebuffer *fb =
2429                         enabled->base.primary->state->fb;
2430                 int clock = pipe_mode->crtc_clock;
2431                 int htotal = pipe_mode->crtc_htotal;
2432                 int hdisplay = enabled->config->pipe_src_w;
2433                 int cpp;
2434                 int entries;
2435
2436                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2437                         cpp = 4;
2438                 else
2439                         cpp = fb->format->cpp[0];
2440
2441                 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2442                                            sr_latency_ns / 100);
2443                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2444                 drm_dbg_kms(&dev_priv->drm,
2445                             "self-refresh entries: %d\n", entries);
2446                 srwm = wm_info->fifo_size - entries;
2447                 if (srwm < 0)
2448                         srwm = 1;
2449
2450                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2451                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2452                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2453                 else
2454                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2455         }
2456
2457         drm_dbg_kms(&dev_priv->drm,
2458                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2459                      planea_wm, planeb_wm, cwm, srwm);
2460
2461         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2462         fwater_hi = (cwm & 0x1f);
2463
2464         /* Set request length to 8 cachelines per fetch */
2465         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2466         fwater_hi = fwater_hi | (1 << 8);
2467
2468         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2469         intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2470
2471         if (enabled)
2472                 intel_set_memory_cxsr(dev_priv, true);
2473 }
2474
2475 static void i845_update_wm(struct intel_crtc *unused_crtc)
2476 {
2477         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2478         struct intel_crtc *crtc;
2479         const struct drm_display_mode *pipe_mode;
2480         u32 fwater_lo;
2481         int planea_wm;
2482
2483         crtc = single_enabled_crtc(dev_priv);
2484         if (crtc == NULL)
2485                 return;
2486
2487         pipe_mode = &crtc->config->hw.pipe_mode;
2488         planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2489                                        &i845_wm_info,
2490                                        dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2491                                        4, pessimal_latency_ns);
2492         fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2493         fwater_lo |= (3<<8) | planea_wm;
2494
2495         drm_dbg_kms(&dev_priv->drm,
2496                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2497
2498         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2499 }
2500
2501 /* latency must be in 0.1us units. */
2502 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2503                                    unsigned int cpp,
2504                                    unsigned int latency)
2505 {
2506         unsigned int ret;
2507
2508         ret = intel_wm_method1(pixel_rate, cpp, latency);
2509         ret = DIV_ROUND_UP(ret, 64) + 2;
2510
2511         return ret;
2512 }
2513
2514 /* latency must be in 0.1us units. */
2515 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2516                                    unsigned int htotal,
2517                                    unsigned int width,
2518                                    unsigned int cpp,
2519                                    unsigned int latency)
2520 {
2521         unsigned int ret;
2522
2523         ret = intel_wm_method2(pixel_rate, htotal,
2524                                width, cpp, latency);
2525         ret = DIV_ROUND_UP(ret, 64) + 2;
2526
2527         return ret;
2528 }
2529
2530 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2531 {
2532         /*
2533          * Neither of these should be possible since this function shouldn't be
2534          * called if the CRTC is off or the plane is invisible.  But let's be
2535          * extra paranoid to avoid a potential divide-by-zero if we screw up
2536          * elsewhere in the driver.
2537          */
2538         if (WARN_ON(!cpp))
2539                 return 0;
2540         if (WARN_ON(!horiz_pixels))
2541                 return 0;
2542
2543         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2544 }
2545
2546 struct ilk_wm_maximums {
2547         u16 pri;
2548         u16 spr;
2549         u16 cur;
2550         u16 fbc;
2551 };
2552
2553 /*
2554  * For both WM_PIPE and WM_LP.
2555  * mem_value must be in 0.1us units.
2556  */
2557 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2558                               const struct intel_plane_state *plane_state,
2559                               u32 mem_value, bool is_lp)
2560 {
2561         u32 method1, method2;
2562         int cpp;
2563
2564         if (mem_value == 0)
2565                 return U32_MAX;
2566
2567         if (!intel_wm_plane_visible(crtc_state, plane_state))
2568                 return 0;
2569
2570         cpp = plane_state->hw.fb->format->cpp[0];
2571
2572         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2573
2574         if (!is_lp)
2575                 return method1;
2576
2577         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2578                                  crtc_state->hw.pipe_mode.crtc_htotal,
2579                                  drm_rect_width(&plane_state->uapi.dst),
2580                                  cpp, mem_value);
2581
2582         return min(method1, method2);
2583 }
2584
2585 /*
2586  * For both WM_PIPE and WM_LP.
2587  * mem_value must be in 0.1us units.
2588  */
2589 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2590                               const struct intel_plane_state *plane_state,
2591                               u32 mem_value)
2592 {
2593         u32 method1, method2;
2594         int cpp;
2595
2596         if (mem_value == 0)
2597                 return U32_MAX;
2598
2599         if (!intel_wm_plane_visible(crtc_state, plane_state))
2600                 return 0;
2601
2602         cpp = plane_state->hw.fb->format->cpp[0];
2603
2604         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2605         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2606                                  crtc_state->hw.pipe_mode.crtc_htotal,
2607                                  drm_rect_width(&plane_state->uapi.dst),
2608                                  cpp, mem_value);
2609         return min(method1, method2);
2610 }
2611
2612 /*
2613  * For both WM_PIPE and WM_LP.
2614  * mem_value must be in 0.1us units.
2615  */
2616 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2617                               const struct intel_plane_state *plane_state,
2618                               u32 mem_value)
2619 {
2620         int cpp;
2621
2622         if (mem_value == 0)
2623                 return U32_MAX;
2624
2625         if (!intel_wm_plane_visible(crtc_state, plane_state))
2626                 return 0;
2627
2628         cpp = plane_state->hw.fb->format->cpp[0];
2629
2630         return ilk_wm_method2(crtc_state->pixel_rate,
2631                               crtc_state->hw.pipe_mode.crtc_htotal,
2632                               drm_rect_width(&plane_state->uapi.dst),
2633                               cpp, mem_value);
2634 }
2635
2636 /* Only for WM_LP. */
2637 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2638                               const struct intel_plane_state *plane_state,
2639                               u32 pri_val)
2640 {
2641         int cpp;
2642
2643         if (!intel_wm_plane_visible(crtc_state, plane_state))
2644                 return 0;
2645
2646         cpp = plane_state->hw.fb->format->cpp[0];
2647
2648         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2649                           cpp);
2650 }
2651
2652 static unsigned int
2653 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2654 {
2655         if (DISPLAY_VER(dev_priv) >= 8)
2656                 return 3072;
2657         else if (DISPLAY_VER(dev_priv) >= 7)
2658                 return 768;
2659         else
2660                 return 512;
2661 }
2662
2663 static unsigned int
2664 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2665                      int level, bool is_sprite)
2666 {
2667         if (DISPLAY_VER(dev_priv) >= 8)
2668                 /* BDW primary/sprite plane watermarks */
2669                 return level == 0 ? 255 : 2047;
2670         else if (DISPLAY_VER(dev_priv) >= 7)
2671                 /* IVB/HSW primary/sprite plane watermarks */
2672                 return level == 0 ? 127 : 1023;
2673         else if (!is_sprite)
2674                 /* ILK/SNB primary plane watermarks */
2675                 return level == 0 ? 127 : 511;
2676         else
2677                 /* ILK/SNB sprite plane watermarks */
2678                 return level == 0 ? 63 : 255;
2679 }
2680
2681 static unsigned int
2682 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2683 {
2684         if (DISPLAY_VER(dev_priv) >= 7)
2685                 return level == 0 ? 63 : 255;
2686         else
2687                 return level == 0 ? 31 : 63;
2688 }
2689
2690 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2691 {
2692         if (DISPLAY_VER(dev_priv) >= 8)
2693                 return 31;
2694         else
2695                 return 15;
2696 }
2697
2698 /* Calculate the maximum primary/sprite plane watermark */
2699 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2700                                      int level,
2701                                      const struct intel_wm_config *config,
2702                                      enum intel_ddb_partitioning ddb_partitioning,
2703                                      bool is_sprite)
2704 {
2705         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2706
2707         /* if sprites aren't enabled, sprites get nothing */
2708         if (is_sprite && !config->sprites_enabled)
2709                 return 0;
2710
2711         /* HSW allows LP1+ watermarks even with multiple pipes */
2712         if (level == 0 || config->num_pipes_active > 1) {
2713                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2714
2715                 /*
2716                  * For some reason the non self refresh
2717                  * FIFO size is only half of the self
2718                  * refresh FIFO size on ILK/SNB.
2719                  */
2720                 if (DISPLAY_VER(dev_priv) <= 6)
2721                         fifo_size /= 2;
2722         }
2723
2724         if (config->sprites_enabled) {
2725                 /* level 0 is always calculated with 1:1 split */
2726                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2727                         if (is_sprite)
2728                                 fifo_size *= 5;
2729                         fifo_size /= 6;
2730                 } else {
2731                         fifo_size /= 2;
2732                 }
2733         }
2734
2735         /* clamp to max that the registers can hold */
2736         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2737 }
2738
2739 /* Calculate the maximum cursor plane watermark */
2740 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2741                                       int level,
2742                                       const struct intel_wm_config *config)
2743 {
2744         /* HSW LP1+ watermarks w/ multiple pipes */
2745         if (level > 0 && config->num_pipes_active > 1)
2746                 return 64;
2747
2748         /* otherwise just report max that registers can hold */
2749         return ilk_cursor_wm_reg_max(dev_priv, level);
2750 }
2751
2752 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2753                                     int level,
2754                                     const struct intel_wm_config *config,
2755                                     enum intel_ddb_partitioning ddb_partitioning,
2756                                     struct ilk_wm_maximums *max)
2757 {
2758         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2759         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2760         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2761         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2762 }
2763
2764 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2765                                         int level,
2766                                         struct ilk_wm_maximums *max)
2767 {
2768         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2769         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2770         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2771         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2772 }
2773
2774 static bool ilk_validate_wm_level(int level,
2775                                   const struct ilk_wm_maximums *max,
2776                                   struct intel_wm_level *result)
2777 {
2778         bool ret;
2779
2780         /* already determined to be invalid? */
2781         if (!result->enable)
2782                 return false;
2783
2784         result->enable = result->pri_val <= max->pri &&
2785                          result->spr_val <= max->spr &&
2786                          result->cur_val <= max->cur;
2787
2788         ret = result->enable;
2789
2790         /*
2791          * HACK until we can pre-compute everything,
2792          * and thus fail gracefully if LP0 watermarks
2793          * are exceeded...
2794          */
2795         if (level == 0 && !result->enable) {
2796                 if (result->pri_val > max->pri)
2797                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2798                                       level, result->pri_val, max->pri);
2799                 if (result->spr_val > max->spr)
2800                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2801                                       level, result->spr_val, max->spr);
2802                 if (result->cur_val > max->cur)
2803                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2804                                       level, result->cur_val, max->cur);
2805
2806                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2807                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2808                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2809                 result->enable = true;
2810         }
2811
2812         return ret;
2813 }
2814
2815 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2816                                  const struct intel_crtc *crtc,
2817                                  int level,
2818                                  struct intel_crtc_state *crtc_state,
2819                                  const struct intel_plane_state *pristate,
2820                                  const struct intel_plane_state *sprstate,
2821                                  const struct intel_plane_state *curstate,
2822                                  struct intel_wm_level *result)
2823 {
2824         u16 pri_latency = dev_priv->wm.pri_latency[level];
2825         u16 spr_latency = dev_priv->wm.spr_latency[level];
2826         u16 cur_latency = dev_priv->wm.cur_latency[level];
2827
2828         /* WM1+ latency values stored in 0.5us units */
2829         if (level > 0) {
2830                 pri_latency *= 5;
2831                 spr_latency *= 5;
2832                 cur_latency *= 5;
2833         }
2834
2835         if (pristate) {
2836                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2837                                                      pri_latency, level);
2838                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2839         }
2840
2841         if (sprstate)
2842                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2843
2844         if (curstate)
2845                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2846
2847         result->enable = true;
2848 }
2849
2850 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2851                                   u16 wm[8])
2852 {
2853         struct intel_uncore *uncore = &dev_priv->uncore;
2854
2855         if (DISPLAY_VER(dev_priv) >= 9) {
2856                 u32 val;
2857                 int ret, i;
2858                 int level, max_level = ilk_wm_max_level(dev_priv);
2859
2860                 /* read the first set of memory latencies[0:3] */
2861                 val = 0; /* data0 to be programmed to 0 for first set */
2862                 ret = sandybridge_pcode_read(dev_priv,
2863                                              GEN9_PCODE_READ_MEM_LATENCY,
2864                                              &val, NULL);
2865
2866                 if (ret) {
2867                         drm_err(&dev_priv->drm,
2868                                 "SKL Mailbox read error = %d\n", ret);
2869                         return;
2870                 }
2871
2872                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2873                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2874                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2875                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2876                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2877                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2878                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2879
2880                 /* read the second set of memory latencies[4:7] */
2881                 val = 1; /* data0 to be programmed to 1 for second set */
2882                 ret = sandybridge_pcode_read(dev_priv,
2883                                              GEN9_PCODE_READ_MEM_LATENCY,
2884                                              &val, NULL);
2885                 if (ret) {
2886                         drm_err(&dev_priv->drm,
2887                                 "SKL Mailbox read error = %d\n", ret);
2888                         return;
2889                 }
2890
2891                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2892                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2893                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2894                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2895                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2896                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2897                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2898
2899                 /*
2900                  * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2901                  * need to be disabled. We make sure to sanitize the values out
2902                  * of the punit to satisfy this requirement.
2903                  */
2904                 for (level = 1; level <= max_level; level++) {
2905                         if (wm[level] == 0) {
2906                                 for (i = level + 1; i <= max_level; i++)
2907                                         wm[i] = 0;
2908                                 break;
2909                         }
2910                 }
2911
2912                 /*
2913                  * WaWmMemoryReadLatency:skl+,glk
2914                  *
2915                  * punit doesn't take into account the read latency so we need
2916                  * to add 2us to the various latency levels we retrieve from the
2917                  * punit when level 0 response data us 0us.
2918                  */
2919                 if (wm[0] == 0) {
2920                         wm[0] += 2;
2921                         for (level = 1; level <= max_level; level++) {
2922                                 if (wm[level] == 0)
2923                                         break;
2924                                 wm[level] += 2;
2925                         }
2926                 }
2927
2928                 /*
2929                  * WA Level-0 adjustment for 16GB DIMMs: SKL+
2930                  * If we could not get dimm info enable this WA to prevent from
2931                  * any underrun. If not able to get Dimm info assume 16GB dimm
2932                  * to avoid any underrun.
2933                  */
2934                 if (dev_priv->dram_info.wm_lv_0_adjust_needed)
2935                         wm[0] += 1;
2936
2937         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2938                 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2939
2940                 wm[0] = (sskpd >> 56) & 0xFF;
2941                 if (wm[0] == 0)
2942                         wm[0] = sskpd & 0xF;
2943                 wm[1] = (sskpd >> 4) & 0xFF;
2944                 wm[2] = (sskpd >> 12) & 0xFF;
2945                 wm[3] = (sskpd >> 20) & 0x1FF;
2946                 wm[4] = (sskpd >> 32) & 0x1FF;
2947         } else if (DISPLAY_VER(dev_priv) >= 6) {
2948                 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2949
2950                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2951                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2952                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2953                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2954         } else if (DISPLAY_VER(dev_priv) >= 5) {
2955                 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2956
2957                 /* ILK primary LP0 latency is 700 ns */
2958                 wm[0] = 7;
2959                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2960                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2961         } else {
2962                 MISSING_CASE(INTEL_DEVID(dev_priv));
2963         }
2964 }
2965
2966 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2967                                        u16 wm[5])
2968 {
2969         /* ILK sprite LP0 latency is 1300 ns */
2970         if (IS_DISPLAY_VER(dev_priv, 5))
2971                 wm[0] = 13;
2972 }
2973
2974 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2975                                        u16 wm[5])
2976 {
2977         /* ILK cursor LP0 latency is 1300 ns */
2978         if (IS_DISPLAY_VER(dev_priv, 5))
2979                 wm[0] = 13;
2980 }
2981
2982 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2983 {
2984         /* how many WM levels are we expecting */
2985         if (DISPLAY_VER(dev_priv) >= 9)
2986                 return 7;
2987         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2988                 return 4;
2989         else if (DISPLAY_VER(dev_priv) >= 6)
2990                 return 3;
2991         else
2992                 return 2;
2993 }
2994
2995 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2996                                    const char *name,
2997                                    const u16 wm[])
2998 {
2999         int level, max_level = ilk_wm_max_level(dev_priv);
3000
3001         for (level = 0; level <= max_level; level++) {
3002                 unsigned int latency = wm[level];
3003
3004                 if (latency == 0) {
3005                         drm_dbg_kms(&dev_priv->drm,
3006                                     "%s WM%d latency not provided\n",
3007                                     name, level);
3008                         continue;
3009                 }
3010
3011                 /*
3012                  * - latencies are in us on gen9.
3013                  * - before then, WM1+ latency values are in 0.5us units
3014                  */
3015                 if (DISPLAY_VER(dev_priv) >= 9)
3016                         latency *= 10;
3017                 else if (level > 0)
3018                         latency *= 5;
3019
3020                 drm_dbg_kms(&dev_priv->drm,
3021                             "%s WM%d latency %u (%u.%u usec)\n", name, level,
3022                             wm[level], latency / 10, latency % 10);
3023         }
3024 }
3025
3026 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3027                                     u16 wm[5], u16 min)
3028 {
3029         int level, max_level = ilk_wm_max_level(dev_priv);
3030
3031         if (wm[0] >= min)
3032                 return false;
3033
3034         wm[0] = max(wm[0], min);
3035         for (level = 1; level <= max_level; level++)
3036                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3037
3038         return true;
3039 }
3040
3041 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3042 {
3043         bool changed;
3044
3045         /*
3046          * The BIOS provided WM memory latency values are often
3047          * inadequate for high resolution displays. Adjust them.
3048          */
3049         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3050                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3051                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3052
3053         if (!changed)
3054                 return;
3055
3056         drm_dbg_kms(&dev_priv->drm,
3057                     "WM latency values increased to avoid potential underruns\n");
3058         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3059         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3060         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3061 }
3062
3063 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3064 {
3065         /*
3066          * On some SNB machines (Thinkpad X220 Tablet at least)
3067          * LP3 usage can cause vblank interrupts to be lost.
3068          * The DEIIR bit will go high but it looks like the CPU
3069          * never gets interrupted.
3070          *
3071          * It's not clear whether other interrupt source could
3072          * be affected or if this is somehow limited to vblank
3073          * interrupts only. To play it safe we disable LP3
3074          * watermarks entirely.
3075          */
3076         if (dev_priv->wm.pri_latency[3] == 0 &&
3077             dev_priv->wm.spr_latency[3] == 0 &&
3078             dev_priv->wm.cur_latency[3] == 0)
3079                 return;
3080
3081         dev_priv->wm.pri_latency[3] = 0;
3082         dev_priv->wm.spr_latency[3] = 0;
3083         dev_priv->wm.cur_latency[3] = 0;
3084
3085         drm_dbg_kms(&dev_priv->drm,
3086                     "LP3 watermarks disabled due to potential for lost interrupts\n");
3087         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3088         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3089         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3090 }
3091
3092 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3093 {
3094         intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3095
3096         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3097                sizeof(dev_priv->wm.pri_latency));
3098         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3099                sizeof(dev_priv->wm.pri_latency));
3100
3101         intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3102         intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3103
3104         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3105         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3106         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3107
3108         if (IS_DISPLAY_VER(dev_priv, 6)) {
3109                 snb_wm_latency_quirk(dev_priv);
3110                 snb_wm_lp3_irq_quirk(dev_priv);
3111         }
3112 }
3113
3114 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3115 {
3116         intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3117         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3118 }
3119
3120 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3121                                  struct intel_pipe_wm *pipe_wm)
3122 {
3123         /* LP0 watermark maximums depend on this pipe alone */
3124         const struct intel_wm_config config = {
3125                 .num_pipes_active = 1,
3126                 .sprites_enabled = pipe_wm->sprites_enabled,
3127                 .sprites_scaled = pipe_wm->sprites_scaled,
3128         };
3129         struct ilk_wm_maximums max;
3130
3131         /* LP0 watermarks always use 1/2 DDB partitioning */
3132         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3133
3134         /* At least LP0 must be valid */
3135         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3136                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3137                 return false;
3138         }
3139
3140         return true;
3141 }
3142
3143 /* Compute new watermarks for the pipe */
3144 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3145 {
3146         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3147         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3148         struct intel_pipe_wm *pipe_wm;
3149         struct intel_plane *plane;
3150         const struct intel_plane_state *plane_state;
3151         const struct intel_plane_state *pristate = NULL;
3152         const struct intel_plane_state *sprstate = NULL;
3153         const struct intel_plane_state *curstate = NULL;
3154         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3155         struct ilk_wm_maximums max;
3156
3157         pipe_wm = &crtc_state->wm.ilk.optimal;
3158
3159         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3160                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3161                         pristate = plane_state;
3162                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3163                         sprstate = plane_state;
3164                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3165                         curstate = plane_state;
3166         }
3167
3168         pipe_wm->pipe_enabled = crtc_state->hw.active;
3169         if (sprstate) {
3170                 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3171                 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3172                         (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3173                          drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3174         }
3175
3176         usable_level = max_level;
3177
3178         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3179         if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3180                 usable_level = 1;
3181
3182         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3183         if (pipe_wm->sprites_scaled)
3184                 usable_level = 0;
3185
3186         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3187         ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3188                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3189
3190         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3191                 return -EINVAL;
3192
3193         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3194
3195         for (level = 1; level <= usable_level; level++) {
3196                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3197
3198                 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3199                                      pristate, sprstate, curstate, wm);
3200
3201                 /*
3202                  * Disable any watermark level that exceeds the
3203                  * register maximums since such watermarks are
3204                  * always invalid.
3205                  */
3206                 if (!ilk_validate_wm_level(level, &max, wm)) {
3207                         memset(wm, 0, sizeof(*wm));
3208                         break;
3209                 }
3210         }
3211
3212         return 0;
3213 }
3214
3215 /*
3216  * Build a set of 'intermediate' watermark values that satisfy both the old
3217  * state and the new state.  These can be programmed to the hardware
3218  * immediately.
3219  */
3220 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3221 {
3222         struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3223         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3224         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3225         struct intel_atomic_state *intel_state =
3226                 to_intel_atomic_state(newstate->uapi.state);
3227         const struct intel_crtc_state *oldstate =
3228                 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3229         const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3230         int level, max_level = ilk_wm_max_level(dev_priv);
3231
3232         /*
3233          * Start with the final, target watermarks, then combine with the
3234          * currently active watermarks to get values that are safe both before
3235          * and after the vblank.
3236          */
3237         *a = newstate->wm.ilk.optimal;
3238         if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3239             intel_state->skip_intermediate_wm)
3240                 return 0;
3241
3242         a->pipe_enabled |= b->pipe_enabled;
3243         a->sprites_enabled |= b->sprites_enabled;
3244         a->sprites_scaled |= b->sprites_scaled;
3245
3246         for (level = 0; level <= max_level; level++) {
3247                 struct intel_wm_level *a_wm = &a->wm[level];
3248                 const struct intel_wm_level *b_wm = &b->wm[level];
3249
3250                 a_wm->enable &= b_wm->enable;
3251                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3252                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3253                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3254                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3255         }
3256
3257         /*
3258          * We need to make sure that these merged watermark values are
3259          * actually a valid configuration themselves.  If they're not,
3260          * there's no safe way to transition from the old state to
3261          * the new state, so we need to fail the atomic transaction.
3262          */
3263         if (!ilk_validate_pipe_wm(dev_priv, a))
3264                 return -EINVAL;
3265
3266         /*
3267          * If our intermediate WM are identical to the final WM, then we can
3268          * omit the post-vblank programming; only update if it's different.
3269          */
3270         if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3271                 newstate->wm.need_postvbl_update = true;
3272
3273         return 0;
3274 }
3275
3276 /*
3277  * Merge the watermarks from all active pipes for a specific level.
3278  */
3279 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3280                                int level,
3281                                struct intel_wm_level *ret_wm)
3282 {
3283         const struct intel_crtc *intel_crtc;
3284
3285         ret_wm->enable = true;
3286
3287         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3288                 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3289                 const struct intel_wm_level *wm = &active->wm[level];
3290
3291                 if (!active->pipe_enabled)
3292                         continue;
3293
3294                 /*
3295                  * The watermark values may have been used in the past,
3296                  * so we must maintain them in the registers for some
3297                  * time even if the level is now disabled.
3298                  */
3299                 if (!wm->enable)
3300                         ret_wm->enable = false;
3301
3302                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3303                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3304                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3305                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3306         }
3307 }
3308
3309 /*
3310  * Merge all low power watermarks for all active pipes.
3311  */
3312 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3313                          const struct intel_wm_config *config,
3314                          const struct ilk_wm_maximums *max,
3315                          struct intel_pipe_wm *merged)
3316 {
3317         int level, max_level = ilk_wm_max_level(dev_priv);
3318         int last_enabled_level = max_level;
3319
3320         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3321         if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3322             config->num_pipes_active > 1)
3323                 last_enabled_level = 0;
3324
3325         /* ILK: FBC WM must be disabled always */
3326         merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3327
3328         /* merge each WM1+ level */
3329         for (level = 1; level <= max_level; level++) {
3330                 struct intel_wm_level *wm = &merged->wm[level];
3331
3332                 ilk_merge_wm_level(dev_priv, level, wm);
3333
3334                 if (level > last_enabled_level)
3335                         wm->enable = false;
3336                 else if (!ilk_validate_wm_level(level, max, wm))
3337                         /* make sure all following levels get disabled */
3338                         last_enabled_level = level - 1;
3339
3340                 /*
3341                  * The spec says it is preferred to disable
3342                  * FBC WMs instead of disabling a WM level.
3343                  */
3344                 if (wm->fbc_val > max->fbc) {
3345                         if (wm->enable)
3346                                 merged->fbc_wm_enabled = false;
3347                         wm->fbc_val = 0;
3348                 }
3349         }
3350
3351         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3352         /*
3353          * FIXME this is racy. FBC might get enabled later.
3354          * What we should check here is whether FBC can be
3355          * enabled sometime later.
3356          */
3357         if (IS_DISPLAY_VER(dev_priv, 5) && !merged->fbc_wm_enabled &&
3358             intel_fbc_is_active(dev_priv)) {
3359                 for (level = 2; level <= max_level; level++) {
3360                         struct intel_wm_level *wm = &merged->wm[level];
3361
3362                         wm->enable = false;
3363                 }
3364         }
3365 }
3366
3367 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3368 {
3369         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3370         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3371 }
3372
3373 /* The value we need to program into the WM_LPx latency field */
3374 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3375                                       int level)
3376 {
3377         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3378                 return 2 * level;
3379         else
3380                 return dev_priv->wm.pri_latency[level];
3381 }
3382
3383 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3384                                    const struct intel_pipe_wm *merged,
3385                                    enum intel_ddb_partitioning partitioning,
3386                                    struct ilk_wm_values *results)
3387 {
3388         struct intel_crtc *intel_crtc;
3389         int level, wm_lp;
3390
3391         results->enable_fbc_wm = merged->fbc_wm_enabled;
3392         results->partitioning = partitioning;
3393
3394         /* LP1+ register values */
3395         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3396                 const struct intel_wm_level *r;
3397
3398                 level = ilk_wm_lp_to_level(wm_lp, merged);
3399
3400                 r = &merged->wm[level];
3401
3402                 /*
3403                  * Maintain the watermark values even if the level is
3404                  * disabled. Doing otherwise could cause underruns.
3405                  */
3406                 results->wm_lp[wm_lp - 1] =
3407                         (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3408                         (r->pri_val << WM1_LP_SR_SHIFT) |
3409                         r->cur_val;
3410
3411                 if (r->enable)
3412                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3413
3414                 if (DISPLAY_VER(dev_priv) >= 8)
3415                         results->wm_lp[wm_lp - 1] |=
3416                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3417                 else
3418                         results->wm_lp[wm_lp - 1] |=
3419                                 r->fbc_val << WM1_LP_FBC_SHIFT;
3420
3421                 /*
3422                  * Always set WM1S_LP_EN when spr_val != 0, even if the
3423                  * level is disabled. Doing otherwise could cause underruns.
3424                  */
3425                 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3426                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3427                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3428                 } else
3429                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3430         }
3431
3432         /* LP0 register values */
3433         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3434                 enum pipe pipe = intel_crtc->pipe;
3435                 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3436                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3437
3438                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3439                         continue;
3440
3441                 results->wm_pipe[pipe] =
3442                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3443                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3444                         r->cur_val;
3445         }
3446 }
3447
3448 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3449  * case both are at the same level. Prefer r1 in case they're the same. */
3450 static struct intel_pipe_wm *
3451 ilk_find_best_result(struct drm_i915_private *dev_priv,
3452                      struct intel_pipe_wm *r1,
3453                      struct intel_pipe_wm *r2)
3454 {
3455         int level, max_level = ilk_wm_max_level(dev_priv);
3456         int level1 = 0, level2 = 0;
3457
3458         for (level = 1; level <= max_level; level++) {
3459                 if (r1->wm[level].enable)
3460                         level1 = level;
3461                 if (r2->wm[level].enable)
3462                         level2 = level;
3463         }
3464
3465         if (level1 == level2) {
3466                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3467                         return r2;
3468                 else
3469                         return r1;
3470         } else if (level1 > level2) {
3471                 return r1;
3472         } else {
3473                 return r2;
3474         }
3475 }
3476
3477 /* dirty bits used to track which watermarks need changes */
3478 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3479 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3480 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3481 #define WM_DIRTY_FBC (1 << 24)
3482 #define WM_DIRTY_DDB (1 << 25)
3483
3484 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3485                                          const struct ilk_wm_values *old,
3486                                          const struct ilk_wm_values *new)
3487 {
3488         unsigned int dirty = 0;
3489         enum pipe pipe;
3490         int wm_lp;
3491
3492         for_each_pipe(dev_priv, pipe) {
3493                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3494                         dirty |= WM_DIRTY_PIPE(pipe);
3495                         /* Must disable LP1+ watermarks too */
3496                         dirty |= WM_DIRTY_LP_ALL;
3497                 }
3498         }
3499
3500         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3501                 dirty |= WM_DIRTY_FBC;
3502                 /* Must disable LP1+ watermarks too */
3503                 dirty |= WM_DIRTY_LP_ALL;
3504         }
3505
3506         if (old->partitioning != new->partitioning) {
3507                 dirty |= WM_DIRTY_DDB;
3508                 /* Must disable LP1+ watermarks too */
3509                 dirty |= WM_DIRTY_LP_ALL;
3510         }
3511
3512         /* LP1+ watermarks already deemed dirty, no need to continue */
3513         if (dirty & WM_DIRTY_LP_ALL)
3514                 return dirty;
3515
3516         /* Find the lowest numbered LP1+ watermark in need of an update... */
3517         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3518                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3519                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3520                         break;
3521         }
3522
3523         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3524         for (; wm_lp <= 3; wm_lp++)
3525                 dirty |= WM_DIRTY_LP(wm_lp);
3526
3527         return dirty;
3528 }
3529
3530 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3531                                unsigned int dirty)
3532 {
3533         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3534         bool changed = false;
3535
3536         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3537                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3538                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3539                 changed = true;
3540         }
3541         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3542                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3543                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3544                 changed = true;
3545         }
3546         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3547                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3548                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3549                 changed = true;
3550         }
3551
3552         /*
3553          * Don't touch WM1S_LP_EN here.
3554          * Doing so could cause underruns.
3555          */
3556
3557         return changed;
3558 }
3559
3560 /*
3561  * The spec says we shouldn't write when we don't need, because every write
3562  * causes WMs to be re-evaluated, expending some power.
3563  */
3564 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3565                                 struct ilk_wm_values *results)
3566 {
3567         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3568         unsigned int dirty;
3569         u32 val;
3570
3571         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3572         if (!dirty)
3573                 return;
3574
3575         _ilk_disable_lp_wm(dev_priv, dirty);
3576
3577         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3578                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3579         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3580                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3581         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3582                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3583
3584         if (dirty & WM_DIRTY_DDB) {
3585                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3586                         val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3587                         if (results->partitioning == INTEL_DDB_PART_1_2)
3588                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3589                         else
3590                                 val |= WM_MISC_DATA_PARTITION_5_6;
3591                         intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3592                 } else {
3593                         val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3594                         if (results->partitioning == INTEL_DDB_PART_1_2)
3595                                 val &= ~DISP_DATA_PARTITION_5_6;
3596                         else
3597                                 val |= DISP_DATA_PARTITION_5_6;
3598                         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3599                 }
3600         }
3601
3602         if (dirty & WM_DIRTY_FBC) {
3603                 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3604                 if (results->enable_fbc_wm)
3605                         val &= ~DISP_FBC_WM_DIS;
3606                 else
3607                         val |= DISP_FBC_WM_DIS;
3608                 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3609         }
3610
3611         if (dirty & WM_DIRTY_LP(1) &&
3612             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3613                 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3614
3615         if (DISPLAY_VER(dev_priv) >= 7) {
3616                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3617                         intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3618                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3619                         intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3620         }
3621
3622         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3623                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3624         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3625                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3626         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3627                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3628
3629         dev_priv->wm.hw = *results;
3630 }
3631
3632 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3633 {
3634         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3635 }
3636
3637 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3638 {
3639         int i;
3640         int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3641         u8 enabled_slices_mask = 0;
3642
3643         for (i = 0; i < max_slices; i++) {
3644                 if (intel_uncore_read(&dev_priv->uncore, DBUF_CTL_S(i)) & DBUF_POWER_STATE)
3645                         enabled_slices_mask |= BIT(i);
3646         }
3647
3648         return enabled_slices_mask;
3649 }
3650
3651 /*
3652  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3653  * so assume we'll always need it in order to avoid underruns.
3654  */
3655 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3656 {
3657         return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3658 }
3659
3660 static bool
3661 intel_has_sagv(struct drm_i915_private *dev_priv)
3662 {
3663         return (IS_GEN9_BC(dev_priv) || DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) &&
3664                 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3665 }
3666
3667 static void
3668 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3669 {
3670         if (DISPLAY_VER(dev_priv) >= 12) {
3671                 u32 val = 0;
3672                 int ret;
3673
3674                 ret = sandybridge_pcode_read(dev_priv,
3675                                              GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3676                                              &val, NULL);
3677                 if (!ret) {
3678                         dev_priv->sagv_block_time_us = val;
3679                         return;
3680                 }
3681
3682                 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3683         } else if (IS_DISPLAY_VER(dev_priv, 11)) {
3684                 dev_priv->sagv_block_time_us = 10;
3685                 return;
3686         } else if (IS_DISPLAY_VER(dev_priv, 10)) {
3687                 dev_priv->sagv_block_time_us = 20;
3688                 return;
3689         } else if (IS_DISPLAY_VER(dev_priv, 9)) {
3690                 dev_priv->sagv_block_time_us = 30;
3691                 return;
3692         } else {
3693                 MISSING_CASE(DISPLAY_VER(dev_priv));
3694         }
3695
3696         /* Default to an unusable block time */
3697         dev_priv->sagv_block_time_us = -1;
3698 }
3699
3700 /*
3701  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3702  * depending on power and performance requirements. The display engine access
3703  * to system memory is blocked during the adjustment time. Because of the
3704  * blocking time, having this enabled can cause full system hangs and/or pipe
3705  * underruns if we don't meet all of the following requirements:
3706  *
3707  *  - <= 1 pipe enabled
3708  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3709  *  - We're not using an interlaced display configuration
3710  */
3711 static int
3712 intel_enable_sagv(struct drm_i915_private *dev_priv)
3713 {
3714         int ret;
3715
3716         if (!intel_has_sagv(dev_priv))
3717                 return 0;
3718
3719         if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3720                 return 0;
3721
3722         drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3723         ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3724                                       GEN9_SAGV_ENABLE);
3725
3726         /* We don't need to wait for SAGV when enabling */
3727
3728         /*
3729          * Some skl systems, pre-release machines in particular,
3730          * don't actually have SAGV.
3731          */
3732         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3733                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3734                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3735                 return 0;
3736         } else if (ret < 0) {
3737                 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3738                 return ret;
3739         }
3740
3741         dev_priv->sagv_status = I915_SAGV_ENABLED;
3742         return 0;
3743 }
3744
3745 static int
3746 intel_disable_sagv(struct drm_i915_private *dev_priv)
3747 {
3748         int ret;
3749
3750         if (!intel_has_sagv(dev_priv))
3751                 return 0;
3752
3753         if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3754                 return 0;
3755
3756         drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3757         /* bspec says to keep retrying for at least 1 ms */
3758         ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3759                                 GEN9_SAGV_DISABLE,
3760                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3761                                 1);
3762         /*
3763          * Some skl systems, pre-release machines in particular,
3764          * don't actually have SAGV.
3765          */
3766         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3767                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3768                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3769                 return 0;
3770         } else if (ret < 0) {
3771                 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3772                 return ret;
3773         }
3774
3775         dev_priv->sagv_status = I915_SAGV_DISABLED;
3776         return 0;
3777 }
3778
3779 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3780 {
3781         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3782         const struct intel_bw_state *new_bw_state;
3783         const struct intel_bw_state *old_bw_state;
3784         u32 new_mask = 0;
3785
3786         /*
3787          * Just return if we can't control SAGV or don't have it.
3788          * This is different from situation when we have SAGV but just can't
3789          * afford it due to DBuf limitation - in case if SAGV is completely
3790          * disabled in a BIOS, we are not even allowed to send a PCode request,
3791          * as it will throw an error. So have to check it here.
3792          */
3793         if (!intel_has_sagv(dev_priv))
3794                 return;
3795
3796         new_bw_state = intel_atomic_get_new_bw_state(state);
3797         if (!new_bw_state)
3798                 return;
3799
3800         if (DISPLAY_VER(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3801                 intel_disable_sagv(dev_priv);
3802                 return;
3803         }
3804
3805         old_bw_state = intel_atomic_get_old_bw_state(state);
3806         /*
3807          * Nothing to mask
3808          */
3809         if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3810                 return;
3811
3812         new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3813
3814         /*
3815          * If new mask is zero - means there is nothing to mask,
3816          * we can only unmask, which should be done in unmask.
3817          */
3818         if (!new_mask)
3819                 return;
3820
3821         /*
3822          * Restrict required qgv points before updating the configuration.
3823          * According to BSpec we can't mask and unmask qgv points at the same
3824          * time. Also masking should be done before updating the configuration
3825          * and unmasking afterwards.
3826          */
3827         icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3828 }
3829
3830 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3831 {
3832         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3833         const struct intel_bw_state *new_bw_state;
3834         const struct intel_bw_state *old_bw_state;
3835         u32 new_mask = 0;
3836
3837         /*
3838          * Just return if we can't control SAGV or don't have it.
3839          * This is different from situation when we have SAGV but just can't
3840          * afford it due to DBuf limitation - in case if SAGV is completely
3841          * disabled in a BIOS, we are not even allowed to send a PCode request,
3842          * as it will throw an error. So have to check it here.
3843          */
3844         if (!intel_has_sagv(dev_priv))
3845                 return;
3846
3847         new_bw_state = intel_atomic_get_new_bw_state(state);
3848         if (!new_bw_state)
3849                 return;
3850
3851         if (DISPLAY_VER(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3852                 intel_enable_sagv(dev_priv);
3853                 return;
3854         }
3855
3856         old_bw_state = intel_atomic_get_old_bw_state(state);
3857         /*
3858          * Nothing to unmask
3859          */
3860         if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3861                 return;
3862
3863         new_mask = new_bw_state->qgv_points_mask;
3864
3865         /*
3866          * Allow required qgv points after updating the configuration.
3867          * According to BSpec we can't mask and unmask qgv points at the same
3868          * time. Also masking should be done before updating the configuration
3869          * and unmasking afterwards.
3870          */
3871         icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3872 }
3873
3874 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3875 {
3876         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3877         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3878         enum plane_id plane_id;
3879         int max_level = INT_MAX;
3880
3881         if (!intel_has_sagv(dev_priv))
3882                 return false;
3883
3884         if (!crtc_state->hw.active)
3885                 return true;
3886
3887         if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3888                 return false;
3889
3890         for_each_plane_id_on_crtc(crtc, plane_id) {
3891                 const struct skl_plane_wm *wm =
3892                         &crtc_state->wm.skl.optimal.planes[plane_id];
3893                 int level;
3894
3895                 /* Skip this plane if it's not enabled */
3896                 if (!wm->wm[0].enable)
3897                         continue;
3898
3899                 /* Find the highest enabled wm level for this plane */
3900                 for (level = ilk_wm_max_level(dev_priv);
3901                      !wm->wm[level].enable; --level)
3902                      { }
3903
3904                 /* Highest common enabled wm level for all planes */
3905                 max_level = min(level, max_level);
3906         }
3907
3908         /* No enabled planes? */
3909         if (max_level == INT_MAX)
3910                 return true;
3911
3912         for_each_plane_id_on_crtc(crtc, plane_id) {
3913                 const struct skl_plane_wm *wm =
3914                         &crtc_state->wm.skl.optimal.planes[plane_id];
3915
3916                 /*
3917                  * All enabled planes must have enabled a common wm level that
3918                  * can tolerate memory latencies higher than sagv_block_time_us
3919                  */
3920                 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
3921                         return false;
3922         }
3923
3924         return true;
3925 }
3926
3927 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3928 {
3929         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3930         enum plane_id plane_id;
3931
3932         if (!crtc_state->hw.active)
3933                 return true;
3934
3935         for_each_plane_id_on_crtc(crtc, plane_id) {
3936                 const struct skl_plane_wm *wm =
3937                         &crtc_state->wm.skl.optimal.planes[plane_id];
3938
3939                 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
3940                         return false;
3941         }
3942
3943         return true;
3944 }
3945
3946 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3947 {
3948         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3949         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3950
3951         if (DISPLAY_VER(dev_priv) >= 12)
3952                 return tgl_crtc_can_enable_sagv(crtc_state);
3953         else
3954                 return skl_crtc_can_enable_sagv(crtc_state);
3955 }
3956
3957 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3958                            const struct intel_bw_state *bw_state)
3959 {
3960         if (DISPLAY_VER(dev_priv) < 11 &&
3961             bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3962                 return false;
3963
3964         return bw_state->pipe_sagv_reject == 0;
3965 }
3966
3967 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3968 {
3969         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3970         int ret;
3971         struct intel_crtc *crtc;
3972         struct intel_crtc_state *new_crtc_state;
3973         struct intel_bw_state *new_bw_state = NULL;
3974         const struct intel_bw_state *old_bw_state = NULL;
3975         int i;
3976
3977         for_each_new_intel_crtc_in_state(state, crtc,
3978                                          new_crtc_state, i) {
3979                 new_bw_state = intel_atomic_get_bw_state(state);
3980                 if (IS_ERR(new_bw_state))
3981                         return PTR_ERR(new_bw_state);
3982
3983                 old_bw_state = intel_atomic_get_old_bw_state(state);
3984
3985                 if (intel_crtc_can_enable_sagv(new_crtc_state))
3986                         new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
3987                 else
3988                         new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
3989         }
3990
3991         if (!new_bw_state)
3992                 return 0;
3993
3994         new_bw_state->active_pipes =
3995                 intel_calc_active_pipes(state, old_bw_state->active_pipes);
3996
3997         if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
3998                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
3999                 if (ret)
4000                         return ret;
4001         }
4002
4003         for_each_new_intel_crtc_in_state(state, crtc,
4004                                          new_crtc_state, i) {
4005                 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4006
4007                 /*
4008                  * We store use_sagv_wm in the crtc state rather than relying on
4009                  * that bw state since we have no convenient way to get at the
4010                  * latter from the plane commit hooks (especially in the legacy
4011                  * cursor case)
4012                  */
4013                 pipe_wm->use_sagv_wm = DISPLAY_VER(dev_priv) >= 12 &&
4014                                        intel_can_enable_sagv(dev_priv, new_bw_state);
4015         }
4016
4017         if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4018             intel_can_enable_sagv(dev_priv, old_bw_state)) {
4019                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4020                 if (ret)
4021                         return ret;
4022         } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4023                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4024                 if (ret)
4025                         return ret;
4026         }
4027
4028         return 0;
4029 }
4030
4031 static int intel_dbuf_size(struct drm_i915_private *dev_priv)
4032 {
4033         int ddb_size = INTEL_INFO(dev_priv)->ddb_size;
4034
4035         drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4036
4037         if (DISPLAY_VER(dev_priv) < 11)
4038                 return ddb_size - 4; /* 4 blocks for bypass path allocation */
4039
4040         return ddb_size;
4041 }
4042
4043 static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4044 {
4045         return intel_dbuf_size(dev_priv) /
4046                 INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4047 }
4048
4049 static void
4050 skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4051                          struct skl_ddb_entry *ddb)
4052 {
4053         int slice_size = intel_dbuf_slice_size(dev_priv);
4054
4055         if (!slice_mask) {
4056                 ddb->start = 0;
4057                 ddb->end = 0;
4058                 return;
4059         }
4060
4061         ddb->start = (ffs(slice_mask) - 1) * slice_size;
4062         ddb->end = fls(slice_mask) * slice_size;
4063
4064         WARN_ON(ddb->start >= ddb->end);
4065         WARN_ON(ddb->end > intel_dbuf_size(dev_priv));
4066 }
4067
4068 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4069                             const struct skl_ddb_entry *entry)
4070 {
4071         u32 slice_mask = 0;
4072         u16 ddb_size = intel_dbuf_size(dev_priv);
4073         u16 num_supported_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4074         u16 slice_size = ddb_size / num_supported_slices;
4075         u16 start_slice;
4076         u16 end_slice;
4077
4078         if (!skl_ddb_entry_size(entry))
4079                 return 0;
4080
4081         start_slice = entry->start / slice_size;
4082         end_slice = (entry->end - 1) / slice_size;
4083
4084         /*
4085          * Per plane DDB entry can in a really worst case be on multiple slices
4086          * but single entry is anyway contigious.
4087          */
4088         while (start_slice <= end_slice) {
4089                 slice_mask |= BIT(start_slice);
4090                 start_slice++;
4091         }
4092
4093         return slice_mask;
4094 }
4095
4096 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4097 {
4098         const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4099         int hdisplay, vdisplay;
4100
4101         if (!crtc_state->hw.active)
4102                 return 0;
4103
4104         /*
4105          * Watermark/ddb requirement highly depends upon width of the
4106          * framebuffer, So instead of allocating DDB equally among pipes
4107          * distribute DDB based on resolution/width of the display.
4108          */
4109         drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4110
4111         return hdisplay;
4112 }
4113
4114 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4115                                     enum pipe for_pipe,
4116                                     unsigned int *weight_start,
4117                                     unsigned int *weight_end,
4118                                     unsigned int *weight_total)
4119 {
4120         struct drm_i915_private *dev_priv =
4121                 to_i915(dbuf_state->base.state->base.dev);
4122         enum pipe pipe;
4123
4124         *weight_start = 0;
4125         *weight_end = 0;
4126         *weight_total = 0;
4127
4128         for_each_pipe(dev_priv, pipe) {
4129                 int weight = dbuf_state->weight[pipe];
4130
4131                 /*
4132                  * Do not account pipes using other slice sets
4133                  * luckily as of current BSpec slice sets do not partially
4134                  * intersect(pipes share either same one slice or same slice set
4135                  * i.e no partial intersection), so it is enough to check for
4136                  * equality for now.
4137                  */
4138                 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4139                         continue;
4140
4141                 *weight_total += weight;
4142                 if (pipe < for_pipe) {
4143                         *weight_start += weight;
4144                         *weight_end += weight;
4145                 } else if (pipe == for_pipe) {
4146                         *weight_end += weight;
4147                 }
4148         }
4149 }
4150
4151 static int
4152 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4153 {
4154         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4155         unsigned int weight_total, weight_start, weight_end;
4156         const struct intel_dbuf_state *old_dbuf_state =
4157                 intel_atomic_get_old_dbuf_state(state);
4158         struct intel_dbuf_state *new_dbuf_state =
4159                 intel_atomic_get_new_dbuf_state(state);
4160         struct intel_crtc_state *crtc_state;
4161         struct skl_ddb_entry ddb_slices;
4162         enum pipe pipe = crtc->pipe;
4163         u32 ddb_range_size;
4164         u32 dbuf_slice_mask;
4165         u32 start, end;
4166         int ret;
4167
4168         if (new_dbuf_state->weight[pipe] == 0) {
4169                 new_dbuf_state->ddb[pipe].start = 0;
4170                 new_dbuf_state->ddb[pipe].end = 0;
4171                 goto out;
4172         }
4173
4174         dbuf_slice_mask = new_dbuf_state->slices[pipe];
4175
4176         skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4177         ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4178
4179         intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4180                                 &weight_start, &weight_end, &weight_total);
4181
4182         start = ddb_range_size * weight_start / weight_total;
4183         end = ddb_range_size * weight_end / weight_total;
4184
4185         new_dbuf_state->ddb[pipe].start = ddb_slices.start + start;
4186         new_dbuf_state->ddb[pipe].end = ddb_slices.start + end;
4187
4188 out:
4189         if (skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4190                                 &new_dbuf_state->ddb[pipe]))
4191                 return 0;
4192
4193         ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4194         if (ret)
4195                 return ret;
4196
4197         crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4198         if (IS_ERR(crtc_state))
4199                 return PTR_ERR(crtc_state);
4200
4201         crtc_state->wm.skl.ddb = new_dbuf_state->ddb[pipe];
4202
4203         drm_dbg_kms(&dev_priv->drm,
4204                     "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4205                     crtc->base.base.id, crtc->base.name,
4206                     old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4207                     old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4208                     new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4209                     old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4210
4211         return 0;
4212 }
4213
4214 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4215                                  int width, const struct drm_format_info *format,
4216                                  u64 modifier, unsigned int rotation,
4217                                  u32 plane_pixel_rate, struct skl_wm_params *wp,
4218                                  int color_plane);
4219 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4220                                  int level,
4221                                  unsigned int latency,
4222                                  const struct skl_wm_params *wp,
4223                                  const struct skl_wm_level *result_prev,
4224                                  struct skl_wm_level *result /* out */);
4225
4226 static unsigned int
4227 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4228                       int num_active)
4229 {
4230         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4231         int level, max_level = ilk_wm_max_level(dev_priv);
4232         struct skl_wm_level wm = {};
4233         int ret, min_ddb_alloc = 0;
4234         struct skl_wm_params wp;
4235
4236         ret = skl_compute_wm_params(crtc_state, 256,
4237                                     drm_format_info(DRM_FORMAT_ARGB8888),
4238                                     DRM_FORMAT_MOD_LINEAR,
4239                                     DRM_MODE_ROTATE_0,
4240                                     crtc_state->pixel_rate, &wp, 0);
4241         drm_WARN_ON(&dev_priv->drm, ret);
4242
4243         for (level = 0; level <= max_level; level++) {
4244                 unsigned int latency = dev_priv->wm.skl_latency[level];
4245
4246                 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4247                 if (wm.min_ddb_alloc == U16_MAX)
4248                         break;
4249
4250                 min_ddb_alloc = wm.min_ddb_alloc;
4251         }
4252
4253         return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4254 }
4255
4256 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4257                                        struct skl_ddb_entry *entry, u32 reg)
4258 {
4259
4260         entry->start = reg & DDB_ENTRY_MASK;
4261         entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4262
4263         if (entry->end)
4264                 entry->end += 1;
4265 }
4266
4267 static void
4268 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4269                            const enum pipe pipe,
4270                            const enum plane_id plane_id,
4271                            struct skl_ddb_entry *ddb_y,
4272                            struct skl_ddb_entry *ddb_uv)
4273 {
4274         u32 val, val2;
4275         u32 fourcc = 0;
4276
4277         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4278         if (plane_id == PLANE_CURSOR) {
4279                 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4280                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4281                 return;
4282         }
4283
4284         val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id));
4285
4286         /* No DDB allocated for disabled planes */
4287         if (val & PLANE_CTL_ENABLE)
4288                 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4289                                               val & PLANE_CTL_ORDER_RGBX,
4290                                               val & PLANE_CTL_ALPHA_MASK);
4291
4292         if (DISPLAY_VER(dev_priv) >= 11) {
4293                 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4294                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4295         } else {
4296                 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4297                 val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4298
4299                 if (fourcc &&
4300                     drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4301                         swap(val, val2);
4302
4303                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4304                 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4305         }
4306 }
4307
4308 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4309                                struct skl_ddb_entry *ddb_y,
4310                                struct skl_ddb_entry *ddb_uv)
4311 {
4312         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4313         enum intel_display_power_domain power_domain;
4314         enum pipe pipe = crtc->pipe;
4315         intel_wakeref_t wakeref;
4316         enum plane_id plane_id;
4317
4318         power_domain = POWER_DOMAIN_PIPE(pipe);
4319         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4320         if (!wakeref)
4321                 return;
4322
4323         for_each_plane_id_on_crtc(crtc, plane_id)
4324                 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4325                                            plane_id,
4326                                            &ddb_y[plane_id],
4327                                            &ddb_uv[plane_id]);
4328
4329         intel_display_power_put(dev_priv, power_domain, wakeref);
4330 }
4331
4332 /*
4333  * Determines the downscale amount of a plane for the purposes of watermark calculations.
4334  * The bspec defines downscale amount as:
4335  *
4336  * """
4337  * Horizontal down scale amount = maximum[1, Horizontal source size /
4338  *                                           Horizontal destination size]
4339  * Vertical down scale amount = maximum[1, Vertical source size /
4340  *                                         Vertical destination size]
4341  * Total down scale amount = Horizontal down scale amount *
4342  *                           Vertical down scale amount
4343  * """
4344  *
4345  * Return value is provided in 16.16 fixed point form to retain fractional part.
4346  * Caller should take care of dividing & rounding off the value.
4347  */
4348 static uint_fixed_16_16_t
4349 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4350                            const struct intel_plane_state *plane_state)
4351 {
4352         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4353         u32 src_w, src_h, dst_w, dst_h;
4354         uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4355         uint_fixed_16_16_t downscale_h, downscale_w;
4356
4357         if (drm_WARN_ON(&dev_priv->drm,
4358                         !intel_wm_plane_visible(crtc_state, plane_state)))
4359                 return u32_to_fixed16(0);
4360
4361         /*
4362          * Src coordinates are already rotated by 270 degrees for
4363          * the 90/270 degree plane rotation cases (to match the
4364          * GTT mapping), hence no need to account for rotation here.
4365          *
4366          * n.b., src is 16.16 fixed point, dst is whole integer.
4367          */
4368         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4369         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4370         dst_w = drm_rect_width(&plane_state->uapi.dst);
4371         dst_h = drm_rect_height(&plane_state->uapi.dst);
4372
4373         fp_w_ratio = div_fixed16(src_w, dst_w);
4374         fp_h_ratio = div_fixed16(src_h, dst_h);
4375         downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4376         downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4377
4378         return mul_fixed16(downscale_w, downscale_h);
4379 }
4380
4381 struct dbuf_slice_conf_entry {
4382         u8 active_pipes;
4383         u8 dbuf_mask[I915_MAX_PIPES];
4384 };
4385
4386 /*
4387  * Table taken from Bspec 12716
4388  * Pipes do have some preferred DBuf slice affinity,
4389  * plus there are some hardcoded requirements on how
4390  * those should be distributed for multipipe scenarios.
4391  * For more DBuf slices algorithm can get even more messy
4392  * and less readable, so decided to use a table almost
4393  * as is from BSpec itself - that way it is at least easier
4394  * to compare, change and check.
4395  */
4396 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4397 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4398 {
4399         {
4400                 .active_pipes = BIT(PIPE_A),
4401                 .dbuf_mask = {
4402                         [PIPE_A] = BIT(DBUF_S1),
4403                 },
4404         },
4405         {
4406                 .active_pipes = BIT(PIPE_B),
4407                 .dbuf_mask = {
4408                         [PIPE_B] = BIT(DBUF_S1),
4409                 },
4410         },
4411         {
4412                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4413                 .dbuf_mask = {
4414                         [PIPE_A] = BIT(DBUF_S1),
4415                         [PIPE_B] = BIT(DBUF_S2),
4416                 },
4417         },
4418         {
4419                 .active_pipes = BIT(PIPE_C),
4420                 .dbuf_mask = {
4421                         [PIPE_C] = BIT(DBUF_S2),
4422                 },
4423         },
4424         {
4425                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4426                 .dbuf_mask = {
4427                         [PIPE_A] = BIT(DBUF_S1),
4428                         [PIPE_C] = BIT(DBUF_S2),
4429                 },
4430         },
4431         {
4432                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4433                 .dbuf_mask = {
4434                         [PIPE_B] = BIT(DBUF_S1),
4435                         [PIPE_C] = BIT(DBUF_S2),
4436                 },
4437         },
4438         {
4439                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4440                 .dbuf_mask = {
4441                         [PIPE_A] = BIT(DBUF_S1),
4442                         [PIPE_B] = BIT(DBUF_S1),
4443                         [PIPE_C] = BIT(DBUF_S2),
4444                 },
4445         },
4446         {}
4447 };
4448
4449 /*
4450  * Table taken from Bspec 49255
4451  * Pipes do have some preferred DBuf slice affinity,
4452  * plus there are some hardcoded requirements on how
4453  * those should be distributed for multipipe scenarios.
4454  * For more DBuf slices algorithm can get even more messy
4455  * and less readable, so decided to use a table almost
4456  * as is from BSpec itself - that way it is at least easier
4457  * to compare, change and check.
4458  */
4459 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4460 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4461 {
4462         {
4463                 .active_pipes = BIT(PIPE_A),
4464                 .dbuf_mask = {
4465                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4466                 },
4467         },
4468         {
4469                 .active_pipes = BIT(PIPE_B),
4470                 .dbuf_mask = {
4471                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4472                 },
4473         },
4474         {
4475                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4476                 .dbuf_mask = {
4477                         [PIPE_A] = BIT(DBUF_S2),
4478                         [PIPE_B] = BIT(DBUF_S1),
4479                 },
4480         },
4481         {
4482                 .active_pipes = BIT(PIPE_C),
4483                 .dbuf_mask = {
4484                         [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4485                 },
4486         },
4487         {
4488                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4489                 .dbuf_mask = {
4490                         [PIPE_A] = BIT(DBUF_S1),
4491                         [PIPE_C] = BIT(DBUF_S2),
4492                 },
4493         },
4494         {
4495                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4496                 .dbuf_mask = {
4497                         [PIPE_B] = BIT(DBUF_S1),
4498                         [PIPE_C] = BIT(DBUF_S2),
4499                 },
4500         },
4501         {
4502                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4503                 .dbuf_mask = {
4504                         [PIPE_A] = BIT(DBUF_S1),
4505                         [PIPE_B] = BIT(DBUF_S1),
4506                         [PIPE_C] = BIT(DBUF_S2),
4507                 },
4508         },
4509         {
4510                 .active_pipes = BIT(PIPE_D),
4511                 .dbuf_mask = {
4512                         [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4513                 },
4514         },
4515         {
4516                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4517                 .dbuf_mask = {
4518                         [PIPE_A] = BIT(DBUF_S1),
4519                         [PIPE_D] = BIT(DBUF_S2),
4520                 },
4521         },
4522         {
4523                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4524                 .dbuf_mask = {
4525                         [PIPE_B] = BIT(DBUF_S1),
4526                         [PIPE_D] = BIT(DBUF_S2),
4527                 },
4528         },
4529         {
4530                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4531                 .dbuf_mask = {
4532                         [PIPE_A] = BIT(DBUF_S1),
4533                         [PIPE_B] = BIT(DBUF_S1),
4534                         [PIPE_D] = BIT(DBUF_S2),
4535                 },
4536         },
4537         {
4538                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4539                 .dbuf_mask = {
4540                         [PIPE_C] = BIT(DBUF_S1),
4541                         [PIPE_D] = BIT(DBUF_S2),
4542                 },
4543         },
4544         {
4545                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4546                 .dbuf_mask = {
4547                         [PIPE_A] = BIT(DBUF_S1),
4548                         [PIPE_C] = BIT(DBUF_S2),
4549                         [PIPE_D] = BIT(DBUF_S2),
4550                 },
4551         },
4552         {
4553                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4554                 .dbuf_mask = {
4555                         [PIPE_B] = BIT(DBUF_S1),
4556                         [PIPE_C] = BIT(DBUF_S2),
4557                         [PIPE_D] = BIT(DBUF_S2),
4558                 },
4559         },
4560         {
4561                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4562                 .dbuf_mask = {
4563                         [PIPE_A] = BIT(DBUF_S1),
4564                         [PIPE_B] = BIT(DBUF_S1),
4565                         [PIPE_C] = BIT(DBUF_S2),
4566                         [PIPE_D] = BIT(DBUF_S2),
4567                 },
4568         },
4569         {}
4570 };
4571
4572 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4573                               const struct dbuf_slice_conf_entry *dbuf_slices)
4574 {
4575         int i;
4576
4577         for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4578                 if (dbuf_slices[i].active_pipes == active_pipes)
4579                         return dbuf_slices[i].dbuf_mask[pipe];
4580         }
4581         return 0;
4582 }
4583
4584 /*
4585  * This function finds an entry with same enabled pipe configuration and
4586  * returns correspondent DBuf slice mask as stated in BSpec for particular
4587  * platform.
4588  */
4589 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4590 {
4591         /*
4592          * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4593          * required calculating "pipe ratio" in order to determine
4594          * if one or two slices can be used for single pipe configurations
4595          * as additional constraint to the existing table.
4596          * However based on recent info, it should be not "pipe ratio"
4597          * but rather ratio between pixel_rate and cdclk with additional
4598          * constants, so for now we are using only table until this is
4599          * clarified. Also this is the reason why crtc_state param is
4600          * still here - we will need it once those additional constraints
4601          * pop up.
4602          */
4603         return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4604 }
4605
4606 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4607 {
4608         return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4609 }
4610
4611 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes)
4612 {
4613         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4614         enum pipe pipe = crtc->pipe;
4615
4616         if (IS_DISPLAY_VER(dev_priv, 12))
4617                 return tgl_compute_dbuf_slices(pipe, active_pipes);
4618         else if (IS_DISPLAY_VER(dev_priv, 11))
4619                 return icl_compute_dbuf_slices(pipe, active_pipes);
4620         /*
4621          * For anything else just return one slice yet.
4622          * Should be extended for other platforms.
4623          */
4624         return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4625 }
4626
4627 static u64
4628 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4629                              const struct intel_plane_state *plane_state,
4630                              int color_plane)
4631 {
4632         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4633         const struct drm_framebuffer *fb = plane_state->hw.fb;
4634         u32 data_rate;
4635         u32 width = 0, height = 0;
4636         uint_fixed_16_16_t down_scale_amount;
4637         u64 rate;
4638
4639         if (!plane_state->uapi.visible)
4640                 return 0;
4641
4642         if (plane->id == PLANE_CURSOR)
4643                 return 0;
4644
4645         if (color_plane == 1 &&
4646             !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4647                 return 0;
4648
4649         /*
4650          * Src coordinates are already rotated by 270 degrees for
4651          * the 90/270 degree plane rotation cases (to match the
4652          * GTT mapping), hence no need to account for rotation here.
4653          */
4654         width = drm_rect_width(&plane_state->uapi.src) >> 16;
4655         height = drm_rect_height(&plane_state->uapi.src) >> 16;
4656
4657         /* UV plane does 1/2 pixel sub-sampling */
4658         if (color_plane == 1) {
4659                 width /= 2;
4660                 height /= 2;
4661         }
4662
4663         data_rate = width * height;
4664
4665         down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4666
4667         rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4668
4669         rate *= fb->format->cpp[color_plane];
4670         return rate;
4671 }
4672
4673 static u64
4674 skl_get_total_relative_data_rate(struct intel_atomic_state *state,
4675                                  struct intel_crtc *crtc)
4676 {
4677         struct intel_crtc_state *crtc_state =
4678                 intel_atomic_get_new_crtc_state(state, crtc);
4679         const struct intel_plane_state *plane_state;
4680         struct intel_plane *plane;
4681         u64 total_data_rate = 0;
4682         enum plane_id plane_id;
4683         int i;
4684
4685         /* Calculate and cache data rate for each plane */
4686         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4687                 if (plane->pipe != crtc->pipe)
4688                         continue;
4689
4690                 plane_id = plane->id;
4691
4692                 /* packed/y */
4693                 crtc_state->plane_data_rate[plane_id] =
4694                         skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4695
4696                 /* uv-plane */
4697                 crtc_state->uv_plane_data_rate[plane_id] =
4698                         skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4699         }
4700
4701         for_each_plane_id_on_crtc(crtc, plane_id) {
4702                 total_data_rate += crtc_state->plane_data_rate[plane_id];
4703                 total_data_rate += crtc_state->uv_plane_data_rate[plane_id];
4704         }
4705
4706         return total_data_rate;
4707 }
4708
4709 static u64
4710 icl_get_total_relative_data_rate(struct intel_atomic_state *state,
4711                                  struct intel_crtc *crtc)
4712 {
4713         struct intel_crtc_state *crtc_state =
4714                 intel_atomic_get_new_crtc_state(state, crtc);
4715         const struct intel_plane_state *plane_state;
4716         struct intel_plane *plane;
4717         u64 total_data_rate = 0;
4718         enum plane_id plane_id;
4719         int i;
4720
4721         /* Calculate and cache data rate for each plane */
4722         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4723                 if (plane->pipe != crtc->pipe)
4724                         continue;
4725
4726                 plane_id = plane->id;
4727
4728                 if (!plane_state->planar_linked_plane) {
4729                         crtc_state->plane_data_rate[plane_id] =
4730                                 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4731                 } else {
4732                         enum plane_id y_plane_id;
4733
4734                         /*
4735                          * The slave plane might not iterate in
4736                          * intel_atomic_crtc_state_for_each_plane_state(),
4737                          * and needs the master plane state which may be
4738                          * NULL if we try get_new_plane_state(), so we
4739                          * always calculate from the master.
4740                          */
4741                         if (plane_state->planar_slave)
4742                                 continue;
4743
4744                         /* Y plane rate is calculated on the slave */
4745                         y_plane_id = plane_state->planar_linked_plane->id;
4746                         crtc_state->plane_data_rate[y_plane_id] =
4747                                 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4748
4749                         crtc_state->plane_data_rate[plane_id] =
4750                                 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4751                 }
4752         }
4753
4754         for_each_plane_id_on_crtc(crtc, plane_id)
4755                 total_data_rate += crtc_state->plane_data_rate[plane_id];
4756
4757         return total_data_rate;
4758 }
4759
4760 const struct skl_wm_level *
4761 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
4762                    enum plane_id plane_id,
4763                    int level)
4764 {
4765         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4766
4767         if (level == 0 && pipe_wm->use_sagv_wm)
4768                 return &wm->sagv.wm0;
4769
4770         return &wm->wm[level];
4771 }
4772
4773 const struct skl_wm_level *
4774 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
4775                    enum plane_id plane_id)
4776 {
4777         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4778
4779         if (pipe_wm->use_sagv_wm)
4780                 return &wm->sagv.trans_wm;
4781
4782         return &wm->trans_wm;
4783 }
4784
4785 /*
4786  * We only disable the watermarks for each plane if
4787  * they exceed the ddb allocation of said plane. This
4788  * is done so that we don't end up touching cursor
4789  * watermarks needlessly when some other plane reduces
4790  * our max possible watermark level.
4791  *
4792  * Bspec has this to say about the PLANE_WM enable bit:
4793  * "All the watermarks at this level for all enabled
4794  *  planes must be enabled before the level will be used."
4795  * So this is actually safe to do.
4796  */
4797 static void
4798 skl_check_wm_level(struct skl_wm_level *wm, u64 total)
4799 {
4800         if (wm->min_ddb_alloc > total)
4801                 memset(wm, 0, sizeof(*wm));
4802 }
4803
4804 static void
4805 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
4806                         u64 total, u64 uv_total)
4807 {
4808         if (wm->min_ddb_alloc > total ||
4809             uv_wm->min_ddb_alloc > uv_total) {
4810                 memset(wm, 0, sizeof(*wm));
4811                 memset(uv_wm, 0, sizeof(*uv_wm));
4812         }
4813 }
4814
4815 static int
4816 skl_allocate_plane_ddb(struct intel_atomic_state *state,
4817                        struct intel_crtc *crtc)
4818 {
4819         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4820         struct intel_crtc_state *crtc_state =
4821                 intel_atomic_get_new_crtc_state(state, crtc);
4822         const struct intel_dbuf_state *dbuf_state =
4823                 intel_atomic_get_new_dbuf_state(state);
4824         const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
4825         int num_active = hweight8(dbuf_state->active_pipes);
4826         u16 alloc_size, start = 0;
4827         u16 total[I915_MAX_PLANES] = {};
4828         u16 uv_total[I915_MAX_PLANES] = {};
4829         u64 total_data_rate;
4830         enum plane_id plane_id;
4831         u32 blocks;
4832         int level;
4833
4834         /* Clear the partitioning for disabled planes. */
4835         memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4836         memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4837
4838         if (!crtc_state->hw.active)
4839                 return 0;
4840
4841         if (DISPLAY_VER(dev_priv) >= 11)
4842                 total_data_rate =
4843                         icl_get_total_relative_data_rate(state, crtc);
4844         else
4845                 total_data_rate =
4846                         skl_get_total_relative_data_rate(state, crtc);
4847
4848         alloc_size = skl_ddb_entry_size(alloc);
4849         if (alloc_size == 0)
4850                 return 0;
4851
4852         /* Allocate fixed number of blocks for cursor. */
4853         total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4854         alloc_size -= total[PLANE_CURSOR];
4855         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4856                 alloc->end - total[PLANE_CURSOR];
4857         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4858
4859         if (total_data_rate == 0)
4860                 return 0;
4861
4862         /*
4863          * Find the highest watermark level for which we can satisfy the block
4864          * requirement of active planes.
4865          */
4866         for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4867                 blocks = 0;
4868                 for_each_plane_id_on_crtc(crtc, plane_id) {
4869                         const struct skl_plane_wm *wm =
4870                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4871
4872                         if (plane_id == PLANE_CURSOR) {
4873                                 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4874                                         drm_WARN_ON(&dev_priv->drm,
4875                                                     wm->wm[level].min_ddb_alloc != U16_MAX);
4876                                         blocks = U32_MAX;
4877                                         break;
4878                                 }
4879                                 continue;
4880                         }
4881
4882                         blocks += wm->wm[level].min_ddb_alloc;
4883                         blocks += wm->uv_wm[level].min_ddb_alloc;
4884                 }
4885
4886                 if (blocks <= alloc_size) {
4887                         alloc_size -= blocks;
4888                         break;
4889                 }
4890         }
4891
4892         if (level < 0) {
4893                 drm_dbg_kms(&dev_priv->drm,
4894                             "Requested display configuration exceeds system DDB limitations");
4895                 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4896                             blocks, alloc_size);
4897                 return -EINVAL;
4898         }
4899
4900         /*
4901          * Grant each plane the blocks it requires at the highest achievable
4902          * watermark level, plus an extra share of the leftover blocks
4903          * proportional to its relative data rate.
4904          */
4905         for_each_plane_id_on_crtc(crtc, plane_id) {
4906                 const struct skl_plane_wm *wm =
4907                         &crtc_state->wm.skl.optimal.planes[plane_id];
4908                 u64 rate;
4909                 u16 extra;
4910
4911                 if (plane_id == PLANE_CURSOR)
4912                         continue;
4913
4914                 /*
4915                  * We've accounted for all active planes; remaining planes are
4916                  * all disabled.
4917                  */
4918                 if (total_data_rate == 0)
4919                         break;
4920
4921                 rate = crtc_state->plane_data_rate[plane_id];
4922                 extra = min_t(u16, alloc_size,
4923                               DIV64_U64_ROUND_UP(alloc_size * rate,
4924                                                  total_data_rate));
4925                 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4926                 alloc_size -= extra;
4927                 total_data_rate -= rate;
4928
4929                 if (total_data_rate == 0)
4930                         break;
4931
4932                 rate = crtc_state->uv_plane_data_rate[plane_id];
4933                 extra = min_t(u16, alloc_size,
4934                               DIV64_U64_ROUND_UP(alloc_size * rate,
4935                                                  total_data_rate));
4936                 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4937                 alloc_size -= extra;
4938                 total_data_rate -= rate;
4939         }
4940         drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4941
4942         /* Set the actual DDB start/end points for each plane */
4943         start = alloc->start;
4944         for_each_plane_id_on_crtc(crtc, plane_id) {
4945                 struct skl_ddb_entry *plane_alloc =
4946                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
4947                 struct skl_ddb_entry *uv_plane_alloc =
4948                         &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4949
4950                 if (plane_id == PLANE_CURSOR)
4951                         continue;
4952
4953                 /* Gen11+ uses a separate plane for UV watermarks */
4954                 drm_WARN_ON(&dev_priv->drm,
4955                             DISPLAY_VER(dev_priv) >= 11 && uv_total[plane_id]);
4956
4957                 /* Leave disabled planes at (0,0) */
4958                 if (total[plane_id]) {
4959                         plane_alloc->start = start;
4960                         start += total[plane_id];
4961                         plane_alloc->end = start;
4962                 }
4963
4964                 if (uv_total[plane_id]) {
4965                         uv_plane_alloc->start = start;
4966                         start += uv_total[plane_id];
4967                         uv_plane_alloc->end = start;
4968                 }
4969         }
4970
4971         /*
4972          * When we calculated watermark values we didn't know how high
4973          * of a level we'd actually be able to hit, so we just marked
4974          * all levels as "enabled."  Go back now and disable the ones
4975          * that aren't actually possible.
4976          */
4977         for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4978                 for_each_plane_id_on_crtc(crtc, plane_id) {
4979                         struct skl_plane_wm *wm =
4980                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4981
4982                         skl_check_nv12_wm_level(&wm->wm[level], &wm->uv_wm[level],
4983                                                 total[plane_id], uv_total[plane_id]);
4984
4985                         /*
4986                          * Wa_1408961008:icl, ehl
4987                          * Underruns with WM1+ disabled
4988                          */
4989                         if (IS_DISPLAY_VER(dev_priv, 11) &&
4990                             level == 1 && wm->wm[0].enable) {
4991                                 wm->wm[level].blocks = wm->wm[0].blocks;
4992                                 wm->wm[level].lines = wm->wm[0].lines;
4993                                 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
4994                         }
4995                 }
4996         }
4997
4998         /*
4999          * Go back and disable the transition and SAGV watermarks
5000          * if it turns out we don't have enough DDB blocks for them.
5001          */
5002         for_each_plane_id_on_crtc(crtc, plane_id) {
5003                 struct skl_plane_wm *wm =
5004                         &crtc_state->wm.skl.optimal.planes[plane_id];
5005
5006                 skl_check_wm_level(&wm->trans_wm, total[plane_id]);
5007                 skl_check_wm_level(&wm->sagv.wm0, total[plane_id]);
5008                 skl_check_wm_level(&wm->sagv.trans_wm, total[plane_id]);
5009         }
5010
5011         return 0;
5012 }
5013
5014 /*
5015  * The max latency should be 257 (max the punit can code is 255 and we add 2us
5016  * for the read latency) and cpp should always be <= 8, so that
5017  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5018  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5019 */
5020 static uint_fixed_16_16_t
5021 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5022                u8 cpp, u32 latency, u32 dbuf_block_size)
5023 {
5024         u32 wm_intermediate_val;
5025         uint_fixed_16_16_t ret;
5026
5027         if (latency == 0)
5028                 return FP_16_16_MAX;
5029
5030         wm_intermediate_val = latency * pixel_rate * cpp;
5031         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5032
5033         if (DISPLAY_VER(dev_priv) >= 10)
5034                 ret = add_fixed16_u32(ret, 1);
5035
5036         return ret;
5037 }
5038
5039 static uint_fixed_16_16_t
5040 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5041                uint_fixed_16_16_t plane_blocks_per_line)
5042 {
5043         u32 wm_intermediate_val;
5044         uint_fixed_16_16_t ret;
5045
5046         if (latency == 0)
5047                 return FP_16_16_MAX;
5048
5049         wm_intermediate_val = latency * pixel_rate;
5050         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5051                                            pipe_htotal * 1000);
5052         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5053         return ret;
5054 }
5055
5056 static uint_fixed_16_16_t
5057 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5058 {
5059         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5060         u32 pixel_rate;
5061         u32 crtc_htotal;
5062         uint_fixed_16_16_t linetime_us;
5063
5064         if (!crtc_state->hw.active)
5065                 return u32_to_fixed16(0);
5066
5067         pixel_rate = crtc_state->pixel_rate;
5068
5069         if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5070                 return u32_to_fixed16(0);
5071
5072         crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5073         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5074
5075         return linetime_us;
5076 }
5077
5078 static int
5079 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5080                       int width, const struct drm_format_info *format,
5081                       u64 modifier, unsigned int rotation,
5082                       u32 plane_pixel_rate, struct skl_wm_params *wp,
5083                       int color_plane)
5084 {
5085         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5086         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5087         u32 interm_pbpl;
5088
5089         /* only planar format has two planes */
5090         if (color_plane == 1 &&
5091             !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5092                 drm_dbg_kms(&dev_priv->drm,
5093                             "Non planar format have single plane\n");
5094                 return -EINVAL;
5095         }
5096
5097         wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5098                       modifier == I915_FORMAT_MOD_Yf_TILED ||
5099                       modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5100                       modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5101         wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5102         wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5103                          modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5104         wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5105
5106         wp->width = width;
5107         if (color_plane == 1 && wp->is_planar)
5108                 wp->width /= 2;
5109
5110         wp->cpp = format->cpp[color_plane];
5111         wp->plane_pixel_rate = plane_pixel_rate;
5112
5113         if (DISPLAY_VER(dev_priv) >= 11 &&
5114             modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
5115                 wp->dbuf_block_size = 256;
5116         else
5117                 wp->dbuf_block_size = 512;
5118
5119         if (drm_rotation_90_or_270(rotation)) {
5120                 switch (wp->cpp) {
5121                 case 1:
5122                         wp->y_min_scanlines = 16;
5123                         break;
5124                 case 2:
5125                         wp->y_min_scanlines = 8;
5126                         break;
5127                 case 4:
5128                         wp->y_min_scanlines = 4;
5129                         break;
5130                 default:
5131                         MISSING_CASE(wp->cpp);
5132                         return -EINVAL;
5133                 }
5134         } else {
5135                 wp->y_min_scanlines = 4;
5136         }
5137
5138         if (skl_needs_memory_bw_wa(dev_priv))
5139                 wp->y_min_scanlines *= 2;
5140
5141         wp->plane_bytes_per_line = wp->width * wp->cpp;
5142         if (wp->y_tiled) {
5143                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5144                                            wp->y_min_scanlines,
5145                                            wp->dbuf_block_size);
5146
5147                 if (DISPLAY_VER(dev_priv) >= 10)
5148                         interm_pbpl++;
5149
5150                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5151                                                         wp->y_min_scanlines);
5152         } else {
5153                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5154                                            wp->dbuf_block_size);
5155
5156                 if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5157                         interm_pbpl++;
5158
5159                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5160         }
5161
5162         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5163                                              wp->plane_blocks_per_line);
5164
5165         wp->linetime_us = fixed16_to_u32_round_up(
5166                                         intel_get_linetime_us(crtc_state));
5167
5168         return 0;
5169 }
5170
5171 static int
5172 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5173                             const struct intel_plane_state *plane_state,
5174                             struct skl_wm_params *wp, int color_plane)
5175 {
5176         const struct drm_framebuffer *fb = plane_state->hw.fb;
5177         int width;
5178
5179         /*
5180          * Src coordinates are already rotated by 270 degrees for
5181          * the 90/270 degree plane rotation cases (to match the
5182          * GTT mapping), hence no need to account for rotation here.
5183          */
5184         width = drm_rect_width(&plane_state->uapi.src) >> 16;
5185
5186         return skl_compute_wm_params(crtc_state, width,
5187                                      fb->format, fb->modifier,
5188                                      plane_state->hw.rotation,
5189                                      intel_plane_pixel_rate(crtc_state, plane_state),
5190                                      wp, color_plane);
5191 }
5192
5193 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5194 {
5195         if (DISPLAY_VER(dev_priv) >= 10)
5196                 return true;
5197
5198         /* The number of lines are ignored for the level 0 watermark. */
5199         return level > 0;
5200 }
5201
5202 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5203                                  int level,
5204                                  unsigned int latency,
5205                                  const struct skl_wm_params *wp,
5206                                  const struct skl_wm_level *result_prev,
5207                                  struct skl_wm_level *result /* out */)
5208 {
5209         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5210         uint_fixed_16_16_t method1, method2;
5211         uint_fixed_16_16_t selected_result;
5212         u32 blocks, lines, min_ddb_alloc = 0;
5213
5214         if (latency == 0) {
5215                 /* reject it */
5216                 result->min_ddb_alloc = U16_MAX;
5217                 return;
5218         }
5219
5220         /*
5221          * WaIncreaseLatencyIPCEnabled: kbl,cfl
5222          * Display WA #1141: kbl,cfl
5223          */
5224         if ((IS_KABYLAKE(dev_priv) ||
5225              IS_COFFEELAKE(dev_priv) ||
5226              IS_COMETLAKE(dev_priv)) &&
5227             dev_priv->ipc_enabled)
5228                 latency += 4;
5229
5230         if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5231                 latency += 15;
5232
5233         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5234                                  wp->cpp, latency, wp->dbuf_block_size);
5235         method2 = skl_wm_method2(wp->plane_pixel_rate,
5236                                  crtc_state->hw.pipe_mode.crtc_htotal,
5237                                  latency,
5238                                  wp->plane_blocks_per_line);
5239
5240         if (wp->y_tiled) {
5241                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5242         } else {
5243                 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5244                      wp->dbuf_block_size < 1) &&
5245                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5246                         selected_result = method2;
5247                 } else if (latency >= wp->linetime_us) {
5248                         if (IS_DISPLAY_VER(dev_priv, 9))
5249                                 selected_result = min_fixed16(method1, method2);
5250                         else
5251                                 selected_result = method2;
5252                 } else {
5253                         selected_result = method1;
5254                 }
5255         }
5256
5257         blocks = fixed16_to_u32_round_up(selected_result) + 1;
5258         lines = div_round_up_fixed16(selected_result,
5259                                      wp->plane_blocks_per_line);
5260
5261         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5262                 /* Display WA #1125: skl,bxt,kbl */
5263                 if (level == 0 && wp->rc_surface)
5264                         blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5265
5266                 /* Display WA #1126: skl,bxt,kbl */
5267                 if (level >= 1 && level <= 7) {
5268                         if (wp->y_tiled) {
5269                                 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5270                                 lines += wp->y_min_scanlines;
5271                         } else {
5272                                 blocks++;
5273                         }
5274
5275                         /*
5276                          * Make sure result blocks for higher latency levels are
5277                          * atleast as high as level below the current level.
5278                          * Assumption in DDB algorithm optimization for special
5279                          * cases. Also covers Display WA #1125 for RC.
5280                          */
5281                         if (result_prev->blocks > blocks)
5282                                 blocks = result_prev->blocks;
5283                 }
5284         }
5285
5286         if (DISPLAY_VER(dev_priv) >= 11) {
5287                 if (wp->y_tiled) {
5288                         int extra_lines;
5289
5290                         if (lines % wp->y_min_scanlines == 0)
5291                                 extra_lines = wp->y_min_scanlines;
5292                         else
5293                                 extra_lines = wp->y_min_scanlines * 2 -
5294                                         lines % wp->y_min_scanlines;
5295
5296                         min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5297                                                                  wp->plane_blocks_per_line);
5298                 } else {
5299                         min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5300                 }
5301         }
5302
5303         if (!skl_wm_has_lines(dev_priv, level))
5304                 lines = 0;
5305
5306         if (lines > 31) {
5307                 /* reject it */
5308                 result->min_ddb_alloc = U16_MAX;
5309                 return;
5310         }
5311
5312         /*
5313          * If lines is valid, assume we can use this watermark level
5314          * for now.  We'll come back and disable it after we calculate the
5315          * DDB allocation if it turns out we don't actually have enough
5316          * blocks to satisfy it.
5317          */
5318         result->blocks = blocks;
5319         result->lines = lines;
5320         /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5321         result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
5322         result->enable = true;
5323
5324         if (DISPLAY_VER(dev_priv) < 12)
5325                 result->can_sagv = latency >= dev_priv->sagv_block_time_us;
5326 }
5327
5328 static void
5329 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5330                       const struct skl_wm_params *wm_params,
5331                       struct skl_wm_level *levels)
5332 {
5333         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5334         int level, max_level = ilk_wm_max_level(dev_priv);
5335         struct skl_wm_level *result_prev = &levels[0];
5336
5337         for (level = 0; level <= max_level; level++) {
5338                 struct skl_wm_level *result = &levels[level];
5339                 unsigned int latency = dev_priv->wm.skl_latency[level];
5340
5341                 skl_compute_plane_wm(crtc_state, level, latency,
5342                                      wm_params, result_prev, result);
5343
5344                 result_prev = result;
5345         }
5346 }
5347
5348 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5349                                 const struct skl_wm_params *wm_params,
5350                                 struct skl_plane_wm *plane_wm)
5351 {
5352         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5353         struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5354         struct skl_wm_level *levels = plane_wm->wm;
5355         unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5356
5357         skl_compute_plane_wm(crtc_state, 0, latency,
5358                              wm_params, &levels[0],
5359                              sagv_wm);
5360 }
5361
5362 static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5363                                       struct skl_wm_level *trans_wm,
5364                                       const struct skl_wm_level *wm0,
5365                                       const struct skl_wm_params *wp)
5366 {
5367         u16 trans_min, trans_amount, trans_y_tile_min;
5368         u16 wm0_blocks, trans_offset, blocks;
5369
5370         /* Transition WM don't make any sense if ipc is disabled */
5371         if (!dev_priv->ipc_enabled)
5372                 return;
5373
5374         /*
5375          * WaDisableTWM:skl,kbl,cfl,bxt
5376          * Transition WM are not recommended by HW team for GEN9
5377          */
5378         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
5379                 return;
5380
5381         if (DISPLAY_VER(dev_priv) >= 11)
5382                 trans_min = 4;
5383         else
5384                 trans_min = 14;
5385
5386         /* Display WA #1140: glk,cnl */
5387         if (IS_DISPLAY_VER(dev_priv, 10))
5388                 trans_amount = 0;
5389         else
5390                 trans_amount = 10; /* This is configurable amount */
5391
5392         trans_offset = trans_min + trans_amount;
5393
5394         /*
5395          * The spec asks for Selected Result Blocks for wm0 (the real value),
5396          * not Result Blocks (the integer value). Pay attention to the capital
5397          * letters. The value wm_l0->blocks is actually Result Blocks, but
5398          * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5399          * and since we later will have to get the ceiling of the sum in the
5400          * transition watermarks calculation, we can just pretend Selected
5401          * Result Blocks is Result Blocks minus 1 and it should work for the
5402          * current platforms.
5403          */
5404         wm0_blocks = wm0->blocks - 1;
5405
5406         if (wp->y_tiled) {
5407                 trans_y_tile_min =
5408                         (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5409                 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5410         } else {
5411                 blocks = wm0_blocks + trans_offset;
5412         }
5413         blocks++;
5414
5415         /*
5416          * Just assume we can enable the transition watermark.  After
5417          * computing the DDB we'll come back and disable it if that
5418          * assumption turns out to be false.
5419          */
5420         trans_wm->blocks = blocks;
5421         trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
5422         trans_wm->enable = true;
5423 }
5424
5425 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5426                                      const struct intel_plane_state *plane_state,
5427                                      enum plane_id plane_id, int color_plane)
5428 {
5429         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5430         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5431         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5432         struct skl_wm_params wm_params;
5433         int ret;
5434
5435         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5436                                           &wm_params, color_plane);
5437         if (ret)
5438                 return ret;
5439
5440         skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5441
5442         skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5443                                   &wm->wm[0], &wm_params);
5444
5445         if (DISPLAY_VER(dev_priv) >= 12) {
5446                 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5447
5448                 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5449                                           &wm->sagv.wm0, &wm_params);
5450         }
5451
5452         return 0;
5453 }
5454
5455 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5456                                  const struct intel_plane_state *plane_state,
5457                                  enum plane_id plane_id)
5458 {
5459         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5460         struct skl_wm_params wm_params;
5461         int ret;
5462
5463         wm->is_planar = true;
5464
5465         /* uv plane watermarks must also be validated for NV12/Planar */
5466         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5467                                           &wm_params, 1);
5468         if (ret)
5469                 return ret;
5470
5471         skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5472
5473         return 0;
5474 }
5475
5476 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5477                               const struct intel_plane_state *plane_state)
5478 {
5479         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5480         enum plane_id plane_id = plane->id;
5481         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5482         const struct drm_framebuffer *fb = plane_state->hw.fb;
5483         int ret;
5484
5485         memset(wm, 0, sizeof(*wm));
5486
5487         if (!intel_wm_plane_visible(crtc_state, plane_state))
5488                 return 0;
5489
5490         ret = skl_build_plane_wm_single(crtc_state, plane_state,
5491                                         plane_id, 0);
5492         if (ret)
5493                 return ret;
5494
5495         if (fb->format->is_yuv && fb->format->num_planes > 1) {
5496                 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5497                                             plane_id);
5498                 if (ret)
5499                         return ret;
5500         }
5501
5502         return 0;
5503 }
5504
5505 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5506                               const struct intel_plane_state *plane_state)
5507 {
5508         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5509         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5510         enum plane_id plane_id = plane->id;
5511         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5512         int ret;
5513
5514         /* Watermarks calculated in master */
5515         if (plane_state->planar_slave)
5516                 return 0;
5517
5518         memset(wm, 0, sizeof(*wm));
5519
5520         if (plane_state->planar_linked_plane) {
5521                 const struct drm_framebuffer *fb = plane_state->hw.fb;
5522                 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5523
5524                 drm_WARN_ON(&dev_priv->drm,
5525                             !intel_wm_plane_visible(crtc_state, plane_state));
5526                 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5527                             fb->format->num_planes == 1);
5528
5529                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5530                                                 y_plane_id, 0);
5531                 if (ret)
5532                         return ret;
5533
5534                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5535                                                 plane_id, 1);
5536                 if (ret)
5537                         return ret;
5538         } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5539                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5540                                                 plane_id, 0);
5541                 if (ret)
5542                         return ret;
5543         }
5544
5545         return 0;
5546 }
5547
5548 static int skl_build_pipe_wm(struct intel_atomic_state *state,
5549                              struct intel_crtc *crtc)
5550 {
5551         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5552         struct intel_crtc_state *crtc_state =
5553                 intel_atomic_get_new_crtc_state(state, crtc);
5554         const struct intel_plane_state *plane_state;
5555         struct intel_plane *plane;
5556         int ret, i;
5557
5558         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5559                 /*
5560                  * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
5561                  * instead but we don't populate that correctly for NV12 Y
5562                  * planes so for now hack this.
5563                  */
5564                 if (plane->pipe != crtc->pipe)
5565                         continue;
5566
5567                 if (DISPLAY_VER(dev_priv) >= 11)
5568                         ret = icl_build_plane_wm(crtc_state, plane_state);
5569                 else
5570                         ret = skl_build_plane_wm(crtc_state, plane_state);
5571                 if (ret)
5572                         return ret;
5573         }
5574
5575         crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5576
5577         return 0;
5578 }
5579
5580 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5581                                 i915_reg_t reg,
5582                                 const struct skl_ddb_entry *entry)
5583 {
5584         if (entry->end)
5585                 intel_de_write_fw(dev_priv, reg,
5586                                   (entry->end - 1) << 16 | entry->start);
5587         else
5588                 intel_de_write_fw(dev_priv, reg, 0);
5589 }
5590
5591 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5592                                i915_reg_t reg,
5593                                const struct skl_wm_level *level)
5594 {
5595         u32 val = 0;
5596
5597         if (level->enable)
5598                 val |= PLANE_WM_EN;
5599         if (level->ignore_lines)
5600                 val |= PLANE_WM_IGNORE_LINES;
5601         val |= level->blocks;
5602         val |= level->lines << PLANE_WM_LINES_SHIFT;
5603
5604         intel_de_write_fw(dev_priv, reg, val);
5605 }
5606
5607 void skl_write_plane_wm(struct intel_plane *plane,
5608                         const struct intel_crtc_state *crtc_state)
5609 {
5610         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5611         int level, max_level = ilk_wm_max_level(dev_priv);
5612         enum plane_id plane_id = plane->id;
5613         enum pipe pipe = plane->pipe;
5614         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5615         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5616         const struct skl_ddb_entry *ddb_y =
5617                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5618         const struct skl_ddb_entry *ddb_uv =
5619                 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5620
5621         for (level = 0; level <= max_level; level++)
5622                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5623                                    skl_plane_wm_level(pipe_wm, plane_id, level));
5624
5625         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5626                            skl_plane_trans_wm(pipe_wm, plane_id));
5627
5628         if (DISPLAY_VER(dev_priv) >= 11) {
5629                 skl_ddb_entry_write(dev_priv,
5630                                     PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5631                 return;
5632         }
5633
5634         if (wm->is_planar)
5635                 swap(ddb_y, ddb_uv);
5636
5637         skl_ddb_entry_write(dev_priv,
5638                             PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5639         skl_ddb_entry_write(dev_priv,
5640                             PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5641 }
5642
5643 void skl_write_cursor_wm(struct intel_plane *plane,
5644                          const struct intel_crtc_state *crtc_state)
5645 {
5646         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5647         int level, max_level = ilk_wm_max_level(dev_priv);
5648         enum plane_id plane_id = plane->id;
5649         enum pipe pipe = plane->pipe;
5650         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5651         const struct skl_ddb_entry *ddb =
5652                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5653
5654         for (level = 0; level <= max_level; level++)
5655                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5656                                    skl_plane_wm_level(pipe_wm, plane_id, level));
5657
5658         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5659                            skl_plane_trans_wm(pipe_wm, plane_id));
5660
5661         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5662 }
5663
5664 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5665                          const struct skl_wm_level *l2)
5666 {
5667         return l1->enable == l2->enable &&
5668                 l1->ignore_lines == l2->ignore_lines &&
5669                 l1->lines == l2->lines &&
5670                 l1->blocks == l2->blocks;
5671 }
5672
5673 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5674                                 const struct skl_plane_wm *wm1,
5675                                 const struct skl_plane_wm *wm2)
5676 {
5677         int level, max_level = ilk_wm_max_level(dev_priv);
5678
5679         for (level = 0; level <= max_level; level++) {
5680                 /*
5681                  * We don't check uv_wm as the hardware doesn't actually
5682                  * use it. It only gets used for calculating the required
5683                  * ddb allocation.
5684                  */
5685                 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5686                         return false;
5687         }
5688
5689         return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
5690                 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
5691                 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
5692 }
5693
5694 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5695                                     const struct skl_ddb_entry *b)
5696 {
5697         return a->start < b->end && b->start < a->end;
5698 }
5699
5700 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
5701                                 const struct skl_ddb_entry *b)
5702 {
5703         if (a->end && b->end) {
5704                 a->start = min(a->start, b->start);
5705                 a->end = max(a->end, b->end);
5706         } else if (b->end) {
5707                 a->start = b->start;
5708                 a->end = b->end;
5709         }
5710 }
5711
5712 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5713                                  const struct skl_ddb_entry *entries,
5714                                  int num_entries, int ignore_idx)
5715 {
5716         int i;
5717
5718         for (i = 0; i < num_entries; i++) {
5719                 if (i != ignore_idx &&
5720                     skl_ddb_entries_overlap(ddb, &entries[i]))
5721                         return true;
5722         }
5723
5724         return false;
5725 }
5726
5727 static int
5728 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5729                             struct intel_crtc_state *new_crtc_state)
5730 {
5731         struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5732         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5733         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5734         struct intel_plane *plane;
5735
5736         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5737                 struct intel_plane_state *plane_state;
5738                 enum plane_id plane_id = plane->id;
5739
5740                 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5741                                         &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5742                     skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5743                                         &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5744                         continue;
5745
5746                 plane_state = intel_atomic_get_plane_state(state, plane);
5747                 if (IS_ERR(plane_state))
5748                         return PTR_ERR(plane_state);
5749
5750                 new_crtc_state->update_planes |= BIT(plane_id);
5751         }
5752
5753         return 0;
5754 }
5755
5756 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
5757 {
5758         struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
5759         u8 enabled_slices;
5760         enum pipe pipe;
5761
5762         /*
5763          * FIXME: For now we always enable slice S1 as per
5764          * the Bspec display initialization sequence.
5765          */
5766         enabled_slices = BIT(DBUF_S1);
5767
5768         for_each_pipe(dev_priv, pipe)
5769                 enabled_slices |= dbuf_state->slices[pipe];
5770
5771         return enabled_slices;
5772 }
5773
5774 static int
5775 skl_compute_ddb(struct intel_atomic_state *state)
5776 {
5777         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5778         const struct intel_dbuf_state *old_dbuf_state;
5779         struct intel_dbuf_state *new_dbuf_state = NULL;
5780         const struct intel_crtc_state *old_crtc_state;
5781         struct intel_crtc_state *new_crtc_state;
5782         struct intel_crtc *crtc;
5783         int ret, i;
5784
5785         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
5786                 new_dbuf_state = intel_atomic_get_dbuf_state(state);
5787                 if (IS_ERR(new_dbuf_state))
5788                         return PTR_ERR(new_dbuf_state);
5789
5790                 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5791                 break;
5792         }
5793
5794         if (!new_dbuf_state)
5795                 return 0;
5796
5797         new_dbuf_state->active_pipes =
5798                 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
5799
5800         if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
5801                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5802                 if (ret)
5803                         return ret;
5804         }
5805
5806         for_each_intel_crtc(&dev_priv->drm, crtc) {
5807                 enum pipe pipe = crtc->pipe;
5808
5809                 new_dbuf_state->slices[pipe] =
5810                         skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes);
5811
5812                 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
5813                         continue;
5814
5815                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5816                 if (ret)
5817                         return ret;
5818         }
5819
5820         new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
5821
5822         if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
5823                 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
5824                 if (ret)
5825                         return ret;
5826
5827                 drm_dbg_kms(&dev_priv->drm,
5828                             "Enabled dbuf slices 0x%x -> 0x%x (out of %d dbuf slices)\n",
5829                             old_dbuf_state->enabled_slices,
5830                             new_dbuf_state->enabled_slices,
5831                             INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
5832         }
5833
5834         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
5835                 enum pipe pipe = crtc->pipe;
5836
5837                 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
5838
5839                 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
5840                         continue;
5841
5842                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5843                 if (ret)
5844                         return ret;
5845         }
5846
5847         for_each_intel_crtc(&dev_priv->drm, crtc) {
5848                 ret = skl_crtc_allocate_ddb(state, crtc);
5849                 if (ret)
5850                         return ret;
5851         }
5852
5853         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5854                                             new_crtc_state, i) {
5855                 ret = skl_allocate_plane_ddb(state, crtc);
5856                 if (ret)
5857                         return ret;
5858
5859                 ret = skl_ddb_add_affected_planes(old_crtc_state,
5860                                                   new_crtc_state);
5861                 if (ret)
5862                         return ret;
5863         }
5864
5865         return 0;
5866 }
5867
5868 static char enast(bool enable)
5869 {
5870         return enable ? '*' : ' ';
5871 }
5872
5873 static void
5874 skl_print_wm_changes(struct intel_atomic_state *state)
5875 {
5876         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5877         const struct intel_crtc_state *old_crtc_state;
5878         const struct intel_crtc_state *new_crtc_state;
5879         struct intel_plane *plane;
5880         struct intel_crtc *crtc;
5881         int i;
5882
5883         if (!drm_debug_enabled(DRM_UT_KMS))
5884                 return;
5885
5886         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5887                                             new_crtc_state, i) {
5888                 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5889
5890                 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5891                 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5892
5893                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5894                         enum plane_id plane_id = plane->id;
5895                         const struct skl_ddb_entry *old, *new;
5896
5897                         old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5898                         new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5899
5900                         if (skl_ddb_entry_equal(old, new))
5901                                 continue;
5902
5903                         drm_dbg_kms(&dev_priv->drm,
5904                                     "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5905                                     plane->base.base.id, plane->base.name,
5906                                     old->start, old->end, new->start, new->end,
5907                                     skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5908                 }
5909
5910                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5911                         enum plane_id plane_id = plane->id;
5912                         const struct skl_plane_wm *old_wm, *new_wm;
5913
5914                         old_wm = &old_pipe_wm->planes[plane_id];
5915                         new_wm = &new_pipe_wm->planes[plane_id];
5916
5917                         if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5918                                 continue;
5919
5920                         drm_dbg_kms(&dev_priv->drm,
5921                                     "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
5922                                     " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
5923                                     plane->base.base.id, plane->base.name,
5924                                     enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
5925                                     enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
5926                                     enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
5927                                     enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
5928                                     enast(old_wm->trans_wm.enable),
5929                                     enast(old_wm->sagv.wm0.enable),
5930                                     enast(old_wm->sagv.trans_wm.enable),
5931                                     enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
5932                                     enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
5933                                     enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
5934                                     enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
5935                                     enast(new_wm->trans_wm.enable),
5936                                     enast(new_wm->sagv.wm0.enable),
5937                                     enast(new_wm->sagv.trans_wm.enable));
5938
5939                         drm_dbg_kms(&dev_priv->drm,
5940                                     "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
5941                                       " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
5942                                     plane->base.base.id, plane->base.name,
5943                                     enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
5944                                     enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
5945                                     enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
5946                                     enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
5947                                     enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
5948                                     enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
5949                                     enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
5950                                     enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
5951                                     enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
5952                                     enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
5953                                     enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
5954                                     enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
5955                                     enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
5956                                     enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
5957                                     enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
5958                                     enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
5959                                     enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
5960                                     enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
5961                                     enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
5962                                     enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
5963                                     enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
5964                                     enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
5965
5966                         drm_dbg_kms(&dev_priv->drm,
5967                                     "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
5968                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
5969                                     plane->base.base.id, plane->base.name,
5970                                     old_wm->wm[0].blocks, old_wm->wm[1].blocks,
5971                                     old_wm->wm[2].blocks, old_wm->wm[3].blocks,
5972                                     old_wm->wm[4].blocks, old_wm->wm[5].blocks,
5973                                     old_wm->wm[6].blocks, old_wm->wm[7].blocks,
5974                                     old_wm->trans_wm.blocks,
5975                                     old_wm->sagv.wm0.blocks,
5976                                     old_wm->sagv.trans_wm.blocks,
5977                                     new_wm->wm[0].blocks, new_wm->wm[1].blocks,
5978                                     new_wm->wm[2].blocks, new_wm->wm[3].blocks,
5979                                     new_wm->wm[4].blocks, new_wm->wm[5].blocks,
5980                                     new_wm->wm[6].blocks, new_wm->wm[7].blocks,
5981                                     new_wm->trans_wm.blocks,
5982                                     new_wm->sagv.wm0.blocks,
5983                                     new_wm->sagv.trans_wm.blocks);
5984
5985                         drm_dbg_kms(&dev_priv->drm,
5986                                     "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
5987                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
5988                                     plane->base.base.id, plane->base.name,
5989                                     old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5990                                     old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5991                                     old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5992                                     old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5993                                     old_wm->trans_wm.min_ddb_alloc,
5994                                     old_wm->sagv.wm0.min_ddb_alloc,
5995                                     old_wm->sagv.trans_wm.min_ddb_alloc,
5996                                     new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5997                                     new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5998                                     new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5999                                     new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6000                                     new_wm->trans_wm.min_ddb_alloc,
6001                                     new_wm->sagv.wm0.min_ddb_alloc,
6002                                     new_wm->sagv.trans_wm.min_ddb_alloc);
6003                 }
6004         }
6005 }
6006
6007 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
6008                                          const struct skl_pipe_wm *old_pipe_wm,
6009                                          const struct skl_pipe_wm *new_pipe_wm)
6010 {
6011         struct drm_i915_private *i915 = to_i915(plane->base.dev);
6012         int level, max_level = ilk_wm_max_level(i915);
6013
6014         for (level = 0; level <= max_level; level++) {
6015                 /*
6016                  * We don't check uv_wm as the hardware doesn't actually
6017                  * use it. It only gets used for calculating the required
6018                  * ddb allocation.
6019                  */
6020                 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
6021                                          skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6022                         return false;
6023         }
6024
6025         return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6026                                    skl_plane_trans_wm(new_pipe_wm, plane->id));
6027 }
6028
6029 /*
6030  * To make sure the cursor watermark registers are always consistent
6031  * with our computed state the following scenario needs special
6032  * treatment:
6033  *
6034  * 1. enable cursor
6035  * 2. move cursor entirely offscreen
6036  * 3. disable cursor
6037  *
6038  * Step 2. does call .disable_plane() but does not zero the watermarks
6039  * (since we consider an offscreen cursor still active for the purposes
6040  * of watermarks). Step 3. would not normally call .disable_plane()
6041  * because the actual plane visibility isn't changing, and we don't
6042  * deallocate the cursor ddb until the pipe gets disabled. So we must
6043  * force step 3. to call .disable_plane() to update the watermark
6044  * registers properly.
6045  *
6046  * Other planes do not suffer from this issues as their watermarks are
6047  * calculated based on the actual plane visibility. The only time this
6048  * can trigger for the other planes is during the initial readout as the
6049  * default value of the watermarks registers is not zero.
6050  */
6051 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6052                                       struct intel_crtc *crtc)
6053 {
6054         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6055         const struct intel_crtc_state *old_crtc_state =
6056                 intel_atomic_get_old_crtc_state(state, crtc);
6057         struct intel_crtc_state *new_crtc_state =
6058                 intel_atomic_get_new_crtc_state(state, crtc);
6059         struct intel_plane *plane;
6060
6061         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6062                 struct intel_plane_state *plane_state;
6063                 enum plane_id plane_id = plane->id;
6064
6065                 /*
6066                  * Force a full wm update for every plane on modeset.
6067                  * Required because the reset value of the wm registers
6068                  * is non-zero, whereas we want all disabled planes to
6069                  * have zero watermarks. So if we turn off the relevant
6070                  * power well the hardware state will go out of sync
6071                  * with the software state.
6072                  */
6073                 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6074                     skl_plane_selected_wm_equals(plane,
6075                                                  &old_crtc_state->wm.skl.optimal,
6076                                                  &new_crtc_state->wm.skl.optimal))
6077                         continue;
6078
6079                 plane_state = intel_atomic_get_plane_state(state, plane);
6080                 if (IS_ERR(plane_state))
6081                         return PTR_ERR(plane_state);
6082
6083                 new_crtc_state->update_planes |= BIT(plane_id);
6084         }
6085
6086         return 0;
6087 }
6088
6089 static int
6090 skl_compute_wm(struct intel_atomic_state *state)
6091 {
6092         struct intel_crtc *crtc;
6093         struct intel_crtc_state *new_crtc_state;
6094         int ret, i;
6095
6096         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6097                 ret = skl_build_pipe_wm(state, crtc);
6098                 if (ret)
6099                         return ret;
6100         }
6101
6102         ret = skl_compute_ddb(state);
6103         if (ret)
6104                 return ret;
6105
6106         ret = intel_compute_sagv_mask(state);
6107         if (ret)
6108                 return ret;
6109
6110         /*
6111          * skl_compute_ddb() will have adjusted the final watermarks
6112          * based on how much ddb is available. Now we can actually
6113          * check if the final watermarks changed.
6114          */
6115         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6116                 ret = skl_wm_add_affected_planes(state, crtc);
6117                 if (ret)
6118                         return ret;
6119         }
6120
6121         skl_print_wm_changes(state);
6122
6123         return 0;
6124 }
6125
6126 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6127                                   struct intel_wm_config *config)
6128 {
6129         struct intel_crtc *crtc;
6130
6131         /* Compute the currently _active_ config */
6132         for_each_intel_crtc(&dev_priv->drm, crtc) {
6133                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6134
6135                 if (!wm->pipe_enabled)
6136                         continue;
6137
6138                 config->sprites_enabled |= wm->sprites_enabled;
6139                 config->sprites_scaled |= wm->sprites_scaled;
6140                 config->num_pipes_active++;
6141         }
6142 }
6143
6144 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6145 {
6146         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6147         struct ilk_wm_maximums max;
6148         struct intel_wm_config config = {};
6149         struct ilk_wm_values results = {};
6150         enum intel_ddb_partitioning partitioning;
6151
6152         ilk_compute_wm_config(dev_priv, &config);
6153
6154         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6155         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6156
6157         /* 5/6 split only in single pipe config on IVB+ */
6158         if (DISPLAY_VER(dev_priv) >= 7 &&
6159             config.num_pipes_active == 1 && config.sprites_enabled) {
6160                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6161                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6162
6163                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6164         } else {
6165                 best_lp_wm = &lp_wm_1_2;
6166         }
6167
6168         partitioning = (best_lp_wm == &lp_wm_1_2) ?
6169                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6170
6171         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6172
6173         ilk_write_wm_values(dev_priv, &results);
6174 }
6175
6176 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6177                                    struct intel_crtc *crtc)
6178 {
6179         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6180         const struct intel_crtc_state *crtc_state =
6181                 intel_atomic_get_new_crtc_state(state, crtc);
6182
6183         mutex_lock(&dev_priv->wm.wm_mutex);
6184         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6185         ilk_program_watermarks(dev_priv);
6186         mutex_unlock(&dev_priv->wm.wm_mutex);
6187 }
6188
6189 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6190                                     struct intel_crtc *crtc)
6191 {
6192         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6193         const struct intel_crtc_state *crtc_state =
6194                 intel_atomic_get_new_crtc_state(state, crtc);
6195
6196         if (!crtc_state->wm.need_postvbl_update)
6197                 return;
6198
6199         mutex_lock(&dev_priv->wm.wm_mutex);
6200         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6201         ilk_program_watermarks(dev_priv);
6202         mutex_unlock(&dev_priv->wm.wm_mutex);
6203 }
6204
6205 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6206 {
6207         level->enable = val & PLANE_WM_EN;
6208         level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6209         level->blocks = val & PLANE_WM_BLOCKS_MASK;
6210         level->lines = (val >> PLANE_WM_LINES_SHIFT) &
6211                 PLANE_WM_LINES_MASK;
6212 }
6213
6214 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6215                               struct skl_pipe_wm *out)
6216 {
6217         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6218         enum pipe pipe = crtc->pipe;
6219         int level, max_level;
6220         enum plane_id plane_id;
6221         u32 val;
6222
6223         max_level = ilk_wm_max_level(dev_priv);
6224
6225         for_each_plane_id_on_crtc(crtc, plane_id) {
6226                 struct skl_plane_wm *wm = &out->planes[plane_id];
6227
6228                 for (level = 0; level <= max_level; level++) {
6229                         if (plane_id != PLANE_CURSOR)
6230                                 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6231                         else
6232                                 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6233
6234                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
6235                 }
6236
6237                 if (plane_id != PLANE_CURSOR)
6238                         val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6239                 else
6240                         val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6241
6242                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6243
6244                 if (DISPLAY_VER(dev_priv) >= 12) {
6245                         wm->sagv.wm0 = wm->wm[0];
6246                         wm->sagv.trans_wm = wm->trans_wm;
6247                 }
6248         }
6249 }
6250
6251 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6252 {
6253         struct intel_dbuf_state *dbuf_state =
6254                 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6255         struct intel_crtc *crtc;
6256
6257         for_each_intel_crtc(&dev_priv->drm, crtc) {
6258                 struct intel_crtc_state *crtc_state =
6259                         to_intel_crtc_state(crtc->base.state);
6260                 enum pipe pipe = crtc->pipe;
6261                 enum plane_id plane_id;
6262
6263                 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6264                 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6265
6266                 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6267
6268                 for_each_plane_id_on_crtc(crtc, plane_id) {
6269                         struct skl_ddb_entry *ddb_y =
6270                                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6271                         struct skl_ddb_entry *ddb_uv =
6272                                 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
6273
6274                         skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6275                                                    plane_id, ddb_y, ddb_uv);
6276
6277                         skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6278                         skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
6279                 }
6280
6281                 dbuf_state->slices[pipe] =
6282                         skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes);
6283
6284                 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6285
6286                 crtc_state->wm.skl.ddb = dbuf_state->ddb[pipe];
6287
6288                 drm_dbg_kms(&dev_priv->drm,
6289                             "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n",
6290                             crtc->base.base.id, crtc->base.name,
6291                             dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6292                             dbuf_state->ddb[pipe].end, dbuf_state->active_pipes);
6293         }
6294
6295         dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6296 }
6297
6298 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6299 {
6300         struct drm_device *dev = crtc->base.dev;
6301         struct drm_i915_private *dev_priv = to_i915(dev);
6302         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6303         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6304         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6305         enum pipe pipe = crtc->pipe;
6306
6307         hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6308
6309         memset(active, 0, sizeof(*active));
6310
6311         active->pipe_enabled = crtc->active;
6312
6313         if (active->pipe_enabled) {
6314                 u32 tmp = hw->wm_pipe[pipe];
6315
6316                 /*
6317                  * For active pipes LP0 watermark is marked as
6318                  * enabled, and LP1+ watermaks as disabled since
6319                  * we can't really reverse compute them in case
6320                  * multiple pipes are active.
6321                  */
6322                 active->wm[0].enable = true;
6323                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6324                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6325                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6326         } else {
6327                 int level, max_level = ilk_wm_max_level(dev_priv);
6328
6329                 /*
6330                  * For inactive pipes, all watermark levels
6331                  * should be marked as enabled but zeroed,
6332                  * which is what we'd compute them to.
6333                  */
6334                 for (level = 0; level <= max_level; level++)
6335                         active->wm[level].enable = true;
6336         }
6337
6338         crtc->wm.active.ilk = *active;
6339 }
6340
6341 #define _FW_WM(value, plane) \
6342         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6343 #define _FW_WM_VLV(value, plane) \
6344         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6345
6346 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6347                                struct g4x_wm_values *wm)
6348 {
6349         u32 tmp;
6350
6351         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6352         wm->sr.plane = _FW_WM(tmp, SR);
6353         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6354         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6355         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6356
6357         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6358         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6359         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6360         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6361         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6362         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6363         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6364
6365         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6366         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6367         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6368         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6369         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6370 }
6371
6372 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6373                                struct vlv_wm_values *wm)
6374 {
6375         enum pipe pipe;
6376         u32 tmp;
6377
6378         for_each_pipe(dev_priv, pipe) {
6379                 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6380
6381                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6382                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6383                 wm->ddl[pipe].plane[PLANE_CURSOR] =
6384                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6385                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6386                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6387                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6388                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6389         }
6390
6391         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6392         wm->sr.plane = _FW_WM(tmp, SR);
6393         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6394         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6395         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6396
6397         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6398         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6399         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6400         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6401
6402         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6403         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6404
6405         if (IS_CHERRYVIEW(dev_priv)) {
6406                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6407                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6408                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6409
6410                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6411                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6412                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6413
6414                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6415                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6416                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6417
6418                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6419                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6420                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6421                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6422                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6423                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6424                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6425                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6426                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6427                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6428                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6429         } else {
6430                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6431                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6432                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6433
6434                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6435                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6436                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6437                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6438                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6439                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6440                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6441                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6442         }
6443 }
6444
6445 #undef _FW_WM
6446 #undef _FW_WM_VLV
6447
6448 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6449 {
6450         struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6451         struct intel_crtc *crtc;
6452
6453         g4x_read_wm_values(dev_priv, wm);
6454
6455         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6456
6457         for_each_intel_crtc(&dev_priv->drm, crtc) {
6458                 struct intel_crtc_state *crtc_state =
6459                         to_intel_crtc_state(crtc->base.state);
6460                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6461                 struct g4x_pipe_wm *raw;
6462                 enum pipe pipe = crtc->pipe;
6463                 enum plane_id plane_id;
6464                 int level, max_level;
6465
6466                 active->cxsr = wm->cxsr;
6467                 active->hpll_en = wm->hpll_en;
6468                 active->fbc_en = wm->fbc_en;
6469
6470                 active->sr = wm->sr;
6471                 active->hpll = wm->hpll;
6472
6473                 for_each_plane_id_on_crtc(crtc, plane_id) {
6474                         active->wm.plane[plane_id] =
6475                                 wm->pipe[pipe].plane[plane_id];
6476                 }
6477
6478                 if (wm->cxsr && wm->hpll_en)
6479                         max_level = G4X_WM_LEVEL_HPLL;
6480                 else if (wm->cxsr)
6481                         max_level = G4X_WM_LEVEL_SR;
6482                 else
6483                         max_level = G4X_WM_LEVEL_NORMAL;
6484
6485                 level = G4X_WM_LEVEL_NORMAL;
6486                 raw = &crtc_state->wm.g4x.raw[level];
6487                 for_each_plane_id_on_crtc(crtc, plane_id)
6488                         raw->plane[plane_id] = active->wm.plane[plane_id];
6489
6490                 if (++level > max_level)
6491                         goto out;
6492
6493                 raw = &crtc_state->wm.g4x.raw[level];
6494                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6495                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6496                 raw->plane[PLANE_SPRITE0] = 0;
6497                 raw->fbc = active->sr.fbc;
6498
6499                 if (++level > max_level)
6500                         goto out;
6501
6502                 raw = &crtc_state->wm.g4x.raw[level];
6503                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6504                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6505                 raw->plane[PLANE_SPRITE0] = 0;
6506                 raw->fbc = active->hpll.fbc;
6507
6508         out:
6509                 for_each_plane_id_on_crtc(crtc, plane_id)
6510                         g4x_raw_plane_wm_set(crtc_state, level,
6511                                              plane_id, USHRT_MAX);
6512                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6513
6514                 crtc_state->wm.g4x.optimal = *active;
6515                 crtc_state->wm.g4x.intermediate = *active;
6516
6517                 drm_dbg_kms(&dev_priv->drm,
6518                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6519                             pipe_name(pipe),
6520                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6521                             wm->pipe[pipe].plane[PLANE_CURSOR],
6522                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
6523         }
6524
6525         drm_dbg_kms(&dev_priv->drm,
6526                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6527                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6528         drm_dbg_kms(&dev_priv->drm,
6529                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6530                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6531         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6532                     yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6533 }
6534
6535 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6536 {
6537         struct intel_plane *plane;
6538         struct intel_crtc *crtc;
6539
6540         mutex_lock(&dev_priv->wm.wm_mutex);
6541
6542         for_each_intel_plane(&dev_priv->drm, plane) {
6543                 struct intel_crtc *crtc =
6544                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6545                 struct intel_crtc_state *crtc_state =
6546                         to_intel_crtc_state(crtc->base.state);
6547                 struct intel_plane_state *plane_state =
6548                         to_intel_plane_state(plane->base.state);
6549                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6550                 enum plane_id plane_id = plane->id;
6551                 int level;
6552
6553                 if (plane_state->uapi.visible)
6554                         continue;
6555
6556                 for (level = 0; level < 3; level++) {
6557                         struct g4x_pipe_wm *raw =
6558                                 &crtc_state->wm.g4x.raw[level];
6559
6560                         raw->plane[plane_id] = 0;
6561                         wm_state->wm.plane[plane_id] = 0;
6562                 }
6563
6564                 if (plane_id == PLANE_PRIMARY) {
6565                         for (level = 0; level < 3; level++) {
6566                                 struct g4x_pipe_wm *raw =
6567                                         &crtc_state->wm.g4x.raw[level];
6568                                 raw->fbc = 0;
6569                         }
6570
6571                         wm_state->sr.fbc = 0;
6572                         wm_state->hpll.fbc = 0;
6573                         wm_state->fbc_en = false;
6574                 }
6575         }
6576
6577         for_each_intel_crtc(&dev_priv->drm, crtc) {
6578                 struct intel_crtc_state *crtc_state =
6579                         to_intel_crtc_state(crtc->base.state);
6580
6581                 crtc_state->wm.g4x.intermediate =
6582                         crtc_state->wm.g4x.optimal;
6583                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6584         }
6585
6586         g4x_program_watermarks(dev_priv);
6587
6588         mutex_unlock(&dev_priv->wm.wm_mutex);
6589 }
6590
6591 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6592 {
6593         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6594         struct intel_crtc *crtc;
6595         u32 val;
6596
6597         vlv_read_wm_values(dev_priv, wm);
6598
6599         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6600         wm->level = VLV_WM_LEVEL_PM2;
6601
6602         if (IS_CHERRYVIEW(dev_priv)) {
6603                 vlv_punit_get(dev_priv);
6604
6605                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6606                 if (val & DSP_MAXFIFO_PM5_ENABLE)
6607                         wm->level = VLV_WM_LEVEL_PM5;
6608
6609                 /*
6610                  * If DDR DVFS is disabled in the BIOS, Punit
6611                  * will never ack the request. So if that happens
6612                  * assume we don't have to enable/disable DDR DVFS
6613                  * dynamically. To test that just set the REQ_ACK
6614                  * bit to poke the Punit, but don't change the
6615                  * HIGH/LOW bits so that we don't actually change
6616                  * the current state.
6617                  */
6618                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6619                 val |= FORCE_DDR_FREQ_REQ_ACK;
6620                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6621
6622                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6623                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6624                         drm_dbg_kms(&dev_priv->drm,
6625                                     "Punit not acking DDR DVFS request, "
6626                                     "assuming DDR DVFS is disabled\n");
6627                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6628                 } else {
6629                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6630                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6631                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6632                 }
6633
6634                 vlv_punit_put(dev_priv);
6635         }
6636
6637         for_each_intel_crtc(&dev_priv->drm, crtc) {
6638                 struct intel_crtc_state *crtc_state =
6639                         to_intel_crtc_state(crtc->base.state);
6640                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6641                 const struct vlv_fifo_state *fifo_state =
6642                         &crtc_state->wm.vlv.fifo_state;
6643                 enum pipe pipe = crtc->pipe;
6644                 enum plane_id plane_id;
6645                 int level;
6646
6647                 vlv_get_fifo_size(crtc_state);
6648
6649                 active->num_levels = wm->level + 1;
6650                 active->cxsr = wm->cxsr;
6651
6652                 for (level = 0; level < active->num_levels; level++) {
6653                         struct g4x_pipe_wm *raw =
6654                                 &crtc_state->wm.vlv.raw[level];
6655
6656                         active->sr[level].plane = wm->sr.plane;
6657                         active->sr[level].cursor = wm->sr.cursor;
6658
6659                         for_each_plane_id_on_crtc(crtc, plane_id) {
6660                                 active->wm[level].plane[plane_id] =
6661                                         wm->pipe[pipe].plane[plane_id];
6662
6663                                 raw->plane[plane_id] =
6664                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
6665                                                             fifo_state->plane[plane_id]);
6666                         }
6667                 }
6668
6669                 for_each_plane_id_on_crtc(crtc, plane_id)
6670                         vlv_raw_plane_wm_set(crtc_state, level,
6671                                              plane_id, USHRT_MAX);
6672                 vlv_invalidate_wms(crtc, active, level);
6673
6674                 crtc_state->wm.vlv.optimal = *active;
6675                 crtc_state->wm.vlv.intermediate = *active;
6676
6677                 drm_dbg_kms(&dev_priv->drm,
6678                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6679                             pipe_name(pipe),
6680                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6681                             wm->pipe[pipe].plane[PLANE_CURSOR],
6682                             wm->pipe[pipe].plane[PLANE_SPRITE0],
6683                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
6684         }
6685
6686         drm_dbg_kms(&dev_priv->drm,
6687                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6688                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6689 }
6690
6691 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6692 {
6693         struct intel_plane *plane;
6694         struct intel_crtc *crtc;
6695
6696         mutex_lock(&dev_priv->wm.wm_mutex);
6697
6698         for_each_intel_plane(&dev_priv->drm, plane) {
6699                 struct intel_crtc *crtc =
6700                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6701                 struct intel_crtc_state *crtc_state =
6702                         to_intel_crtc_state(crtc->base.state);
6703                 struct intel_plane_state *plane_state =
6704                         to_intel_plane_state(plane->base.state);
6705                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6706                 const struct vlv_fifo_state *fifo_state =
6707                         &crtc_state->wm.vlv.fifo_state;
6708                 enum plane_id plane_id = plane->id;
6709                 int level;
6710
6711                 if (plane_state->uapi.visible)
6712                         continue;
6713
6714                 for (level = 0; level < wm_state->num_levels; level++) {
6715                         struct g4x_pipe_wm *raw =
6716                                 &crtc_state->wm.vlv.raw[level];
6717
6718                         raw->plane[plane_id] = 0;
6719
6720                         wm_state->wm[level].plane[plane_id] =
6721                                 vlv_invert_wm_value(raw->plane[plane_id],
6722                                                     fifo_state->plane[plane_id]);
6723                 }
6724         }
6725
6726         for_each_intel_crtc(&dev_priv->drm, crtc) {
6727                 struct intel_crtc_state *crtc_state =
6728                         to_intel_crtc_state(crtc->base.state);
6729
6730                 crtc_state->wm.vlv.intermediate =
6731                         crtc_state->wm.vlv.optimal;
6732                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6733         }
6734
6735         vlv_program_watermarks(dev_priv);
6736
6737         mutex_unlock(&dev_priv->wm.wm_mutex);
6738 }
6739
6740 /*
6741  * FIXME should probably kill this and improve
6742  * the real watermark readout/sanitation instead
6743  */
6744 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6745 {
6746         intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM1_LP_SR_EN);
6747         intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM1_LP_SR_EN);
6748         intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM1_LP_SR_EN);
6749
6750         /*
6751          * Don't touch WM1S_LP_EN here.
6752          * Doing so could cause underruns.
6753          */
6754 }
6755
6756 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6757 {
6758         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6759         struct intel_crtc *crtc;
6760
6761         ilk_init_lp_watermarks(dev_priv);
6762
6763         for_each_intel_crtc(&dev_priv->drm, crtc)
6764                 ilk_pipe_wm_get_hw_state(crtc);
6765
6766         hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
6767         hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
6768         hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
6769
6770         hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
6771         if (DISPLAY_VER(dev_priv) >= 7) {
6772                 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
6773                 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
6774         }
6775
6776         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6777                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6778                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6779         else if (IS_IVYBRIDGE(dev_priv))
6780                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6781                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6782
6783         hw->enable_fbc_wm =
6784                 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6785 }
6786
6787 /**
6788  * intel_update_watermarks - update FIFO watermark values based on current modes
6789  * @crtc: the #intel_crtc on which to compute the WM
6790  *
6791  * Calculate watermark values for the various WM regs based on current mode
6792  * and plane configuration.
6793  *
6794  * There are several cases to deal with here:
6795  *   - normal (i.e. non-self-refresh)
6796  *   - self-refresh (SR) mode
6797  *   - lines are large relative to FIFO size (buffer can hold up to 2)
6798  *   - lines are small relative to FIFO size (buffer can hold more than 2
6799  *     lines), so need to account for TLB latency
6800  *
6801  *   The normal calculation is:
6802  *     watermark = dotclock * bytes per pixel * latency
6803  *   where latency is platform & configuration dependent (we assume pessimal
6804  *   values here).
6805  *
6806  *   The SR calculation is:
6807  *     watermark = (trunc(latency/line time)+1) * surface width *
6808  *       bytes per pixel
6809  *   where
6810  *     line time = htotal / dotclock
6811  *     surface width = hdisplay for normal plane and 64 for cursor
6812  *   and latency is assumed to be high, as above.
6813  *
6814  * The final value programmed to the register should always be rounded up,
6815  * and include an extra 2 entries to account for clock crossings.
6816  *
6817  * We don't use the sprite, so we can ignore that.  And on Crestline we have
6818  * to set the non-SR watermarks to 8.
6819  */
6820 void intel_update_watermarks(struct intel_crtc *crtc)
6821 {
6822         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6823
6824         if (dev_priv->display.update_wm)
6825                 dev_priv->display.update_wm(crtc);
6826 }
6827
6828 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6829 {
6830         u32 val;
6831
6832         if (!HAS_IPC(dev_priv))
6833                 return;
6834
6835         val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
6836
6837         if (dev_priv->ipc_enabled)
6838                 val |= DISP_IPC_ENABLE;
6839         else
6840                 val &= ~DISP_IPC_ENABLE;
6841
6842         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
6843 }
6844
6845 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6846 {
6847         /* Display WA #0477 WaDisableIPC: skl */
6848         if (IS_SKYLAKE(dev_priv))
6849                 return false;
6850
6851         /* Display WA #1141: SKL:all KBL:all CFL */
6852         if (IS_KABYLAKE(dev_priv) ||
6853             IS_COFFEELAKE(dev_priv) ||
6854             IS_COMETLAKE(dev_priv))
6855                 return dev_priv->dram_info.symmetric_memory;
6856
6857         return true;
6858 }
6859
6860 void intel_init_ipc(struct drm_i915_private *dev_priv)
6861 {
6862         if (!HAS_IPC(dev_priv))
6863                 return;
6864
6865         dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6866
6867         intel_enable_ipc(dev_priv);
6868 }
6869
6870 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6871 {
6872         /*
6873          * On Ibex Peak and Cougar Point, we need to disable clock
6874          * gating for the panel power sequencer or it will fail to
6875          * start up when no ports are active.
6876          */
6877         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6878 }
6879
6880 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6881 {
6882         enum pipe pipe;
6883
6884         for_each_pipe(dev_priv, pipe) {
6885                 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
6886                            intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
6887                            DISPPLANE_TRICKLE_FEED_DISABLE);
6888
6889                 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
6890                 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
6891         }
6892 }
6893
6894 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6895 {
6896         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6897
6898         /*
6899          * Required for FBC
6900          * WaFbcDisableDpfcClockGating:ilk
6901          */
6902         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6903                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6904                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6905
6906         intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
6907                    MARIUNIT_CLOCK_GATE_DISABLE |
6908                    SVSMUNIT_CLOCK_GATE_DISABLE);
6909         intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
6910                    VFMUNIT_CLOCK_GATE_DISABLE);
6911
6912         /*
6913          * According to the spec the following bits should be set in
6914          * order to enable memory self-refresh
6915          * The bit 22/21 of 0x42004
6916          * The bit 5 of 0x42020
6917          * The bit 15 of 0x45000
6918          */
6919         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
6920                    (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
6921                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6922         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6923         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
6924                    (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
6925                     DISP_FBC_WM_DIS));
6926
6927         /*
6928          * Based on the document from hardware guys the following bits
6929          * should be set unconditionally in order to enable FBC.
6930          * The bit 22 of 0x42000
6931          * The bit 22 of 0x42004
6932          * The bit 7,8,9 of 0x42020.
6933          */
6934         if (IS_IRONLAKE_M(dev_priv)) {
6935                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6936                 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
6937                            intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
6938                            ILK_FBCQ_DIS);
6939                 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
6940                            intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
6941                            ILK_DPARB_GATE);
6942         }
6943
6944         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
6945
6946         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
6947                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
6948                    ILK_ELPIN_409_SELECT);
6949
6950         g4x_disable_trickle_feed(dev_priv);
6951
6952         ibx_init_clock_gating(dev_priv);
6953 }
6954
6955 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6956 {
6957         enum pipe pipe;
6958         u32 val;
6959
6960         /*
6961          * On Ibex Peak and Cougar Point, we need to disable clock
6962          * gating for the panel power sequencer or it will fail to
6963          * start up when no ports are active.
6964          */
6965         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6966                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6967                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
6968         intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
6969                    DPLS_EDP_PPS_FIX_DIS);
6970         /* The below fixes the weird display corruption, a few pixels shifted
6971          * downward, on (only) LVDS of some HP laptops with IVY.
6972          */
6973         for_each_pipe(dev_priv, pipe) {
6974                 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
6975                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6976                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6977                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6978                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6979                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6980                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6981                 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
6982         }
6983         /* WADP0ClockGatingDisable */
6984         for_each_pipe(dev_priv, pipe) {
6985                 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
6986                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6987         }
6988 }
6989
6990 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6991 {
6992         u32 tmp;
6993
6994         tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
6995         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6996                 drm_dbg_kms(&dev_priv->drm,
6997                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6998                             tmp);
6999 }
7000
7001 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7002 {
7003         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7004
7005         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7006
7007         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7008                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7009                    ILK_ELPIN_409_SELECT);
7010
7011         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7012                    intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7013                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7014                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7015
7016         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7017          * gating disable must be set.  Failure to set it results in
7018          * flickering pixels due to Z write ordering failures after
7019          * some amount of runtime in the Mesa "fire" demo, and Unigine
7020          * Sanctuary and Tropics, and apparently anything else with
7021          * alpha test or pixel discard.
7022          *
7023          * According to the spec, bit 11 (RCCUNIT) must also be set,
7024          * but we didn't debug actual testcases to find it out.
7025          *
7026          * WaDisableRCCUnitClockGating:snb
7027          * WaDisableRCPBUnitClockGating:snb
7028          */
7029         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7030                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7031                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7032
7033         /*
7034          * According to the spec the following bits should be
7035          * set in order to enable memory self-refresh and fbc:
7036          * The bit21 and bit22 of 0x42000
7037          * The bit21 and bit22 of 0x42004
7038          * The bit5 and bit7 of 0x42020
7039          * The bit14 of 0x70180
7040          * The bit14 of 0x71180
7041          *
7042          * WaFbcAsynchFlipDisableFbcQueue:snb
7043          */
7044         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7045                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7046                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7047         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7048                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7049                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7050         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7051                    intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7052                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
7053                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7054
7055         g4x_disable_trickle_feed(dev_priv);
7056
7057         cpt_init_clock_gating(dev_priv);
7058
7059         gen6_check_mch_setup(dev_priv);
7060 }
7061
7062 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7063 {
7064         /*
7065          * TODO: this bit should only be enabled when really needed, then
7066          * disabled when not needed anymore in order to save power.
7067          */
7068         if (HAS_PCH_LPT_LP(dev_priv))
7069                 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7070                            intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7071                            PCH_LP_PARTITION_LEVEL_DISABLE);
7072
7073         /* WADPOClockGatingDisable:hsw */
7074         intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7075                    intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7076                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7077 }
7078
7079 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7080 {
7081         if (HAS_PCH_LPT_LP(dev_priv)) {
7082                 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7083
7084                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7085                 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7086         }
7087 }
7088
7089 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7090                                    int general_prio_credits,
7091                                    int high_prio_credits)
7092 {
7093         u32 misccpctl;
7094         u32 val;
7095
7096         /* WaTempDisableDOPClkGating:bdw */
7097         misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7098         intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7099
7100         val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7101         val &= ~L3_PRIO_CREDITS_MASK;
7102         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7103         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7104         intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7105
7106         /*
7107          * Wait at least 100 clocks before re-enabling clock gating.
7108          * See the definition of L3SQCREG1 in BSpec.
7109          */
7110         intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7111         udelay(1);
7112         intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7113 }
7114
7115 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7116 {
7117         /* Wa_1409120013:icl,ehl */
7118         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7119                    ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7120
7121         /* This is not an Wa. Enable to reduce Sampler power */
7122         intel_uncore_write(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7123                    intel_uncore_read(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7124
7125         /*Wa_14010594013:icl, ehl */
7126         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7127                          0, CNL_DELAY_PMRSP);
7128 }
7129
7130 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7131 {
7132         /* Wa_1409120013:tgl,rkl,adl_s,dg1 */
7133         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7134                            ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7135
7136         /* Wa_1409825376:tgl (pre-prod)*/
7137         if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B1))
7138                 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7139                            TGL_VRH_GATING_DIS);
7140
7141         /* Wa_14011059788:tgl,rkl,adl_s,dg1 */
7142         intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7143                          0, DFR_DISABLE);
7144 }
7145
7146 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7147 {
7148         gen12lp_init_clock_gating(dev_priv);
7149
7150         /* Wa_1409836686:dg1[a0] */
7151         if (IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0))
7152                 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7153                            DPT_GATING_DIS);
7154 }
7155
7156 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7157 {
7158         if (!HAS_PCH_CNP(dev_priv))
7159                 return;
7160
7161         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7162         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7163                    CNP_PWM_CGE_GATING_DISABLE);
7164 }
7165
7166 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7167 {
7168         u32 val;
7169         cnp_init_clock_gating(dev_priv);
7170
7171         /* This is not an Wa. Enable for better image quality */
7172         intel_uncore_write(&dev_priv->uncore, _3D_CHICKEN3,
7173                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7174
7175         /* WaEnableChickenDCPR:cnl */
7176         intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7177                    intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7178
7179         /*
7180          * WaFbcWakeMemOn:cnl
7181          * Display WA #0859: cnl
7182          */
7183         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7184                    DISP_FBC_MEMORY_WAKE);
7185
7186         val = intel_uncore_read(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE);
7187         /* ReadHitWriteOnlyDisable:cnl */
7188         val |= RCCUNIT_CLKGATE_DIS;
7189         intel_uncore_write(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE, val);
7190
7191         /* Wa_2201832410:cnl */
7192         val = intel_uncore_read(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE);
7193         val |= GWUNIT_CLKGATE_DIS;
7194         intel_uncore_write(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7195
7196         /* WaDisableVFclkgate:cnl */
7197         /* WaVFUnitClockGatingDisable:cnl */
7198         val = intel_uncore_read(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE);
7199         val |= VFUNIT_CLKGATE_DIS;
7200         intel_uncore_write(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, val);
7201 }
7202
7203 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7204 {
7205         cnp_init_clock_gating(dev_priv);
7206         gen9_init_clock_gating(dev_priv);
7207
7208         /* WAC6entrylatency:cfl */
7209         intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7210                    FBC_LLC_FULLY_OPEN);
7211
7212         /*
7213          * WaFbcTurnOffFbcWatermark:cfl
7214          * Display WA #0562: cfl
7215          */
7216         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7217                    DISP_FBC_WM_DIS);
7218
7219         /*
7220          * WaFbcNukeOnHostModify:cfl
7221          * Display WA #0873: cfl
7222          */
7223         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7224                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7225 }
7226
7227 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7228 {
7229         gen9_init_clock_gating(dev_priv);
7230
7231         /* WAC6entrylatency:kbl */
7232         intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7233                    FBC_LLC_FULLY_OPEN);
7234
7235         /* WaDisableSDEUnitClockGating:kbl */
7236         if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0))
7237                 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7238                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7239
7240         /* WaDisableGamClockGating:kbl */
7241         if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0))
7242                 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7243                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7244
7245         /*
7246          * WaFbcTurnOffFbcWatermark:kbl
7247          * Display WA #0562: kbl
7248          */
7249         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7250                    DISP_FBC_WM_DIS);
7251
7252         /*
7253          * WaFbcNukeOnHostModify:kbl
7254          * Display WA #0873: kbl
7255          */
7256         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7257                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7258 }
7259
7260 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7261 {
7262         gen9_init_clock_gating(dev_priv);
7263
7264         /* WaDisableDopClockGating:skl */
7265         intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7266                    ~GEN7_DOP_CLOCK_GATE_ENABLE);
7267
7268         /* WAC6entrylatency:skl */
7269         intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7270                    FBC_LLC_FULLY_OPEN);
7271
7272         /*
7273          * WaFbcTurnOffFbcWatermark:skl
7274          * Display WA #0562: skl
7275          */
7276         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7277                    DISP_FBC_WM_DIS);
7278
7279         /*
7280          * WaFbcNukeOnHostModify:skl
7281          * Display WA #0873: skl
7282          */
7283         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7284                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7285
7286         /*
7287          * WaFbcHighMemBwCorruptionAvoidance:skl
7288          * Display WA #0883: skl
7289          */
7290         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7291                    ILK_DPFC_DISABLE_DUMMY0);
7292 }
7293
7294 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7295 {
7296         enum pipe pipe;
7297
7298         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7299         intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7300                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7301                    HSW_FBCQ_DIS);
7302
7303         /* WaSwitchSolVfFArbitrationPriority:bdw */
7304         intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7305
7306         /* WaPsrDPAMaskVBlankInSRD:bdw */
7307         intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7308                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7309
7310         for_each_pipe(dev_priv, pipe) {
7311                 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7312                 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7313                            intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7314                            BDW_DPRS_MASK_VBLANK_SRD);
7315
7316                 /* Undocumented but fixes async flip + VT-d corruption */
7317                 if (intel_vtd_active())
7318                         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7319                                          HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7320         }
7321
7322         /* WaVSRefCountFullforceMissDisable:bdw */
7323         /* WaDSRefCountFullforceMissDisable:bdw */
7324         intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7325                    intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7326                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7327
7328         intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7329                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7330
7331         /* WaDisableSDEUnitClockGating:bdw */
7332         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7333                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7334
7335         /* WaProgramL3SqcReg1Default:bdw */
7336         gen8_set_l3sqc_credits(dev_priv, 30, 2);
7337
7338         /* WaKVMNotificationOnConfigChange:bdw */
7339         intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7340                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7341
7342         lpt_init_clock_gating(dev_priv);
7343
7344         /* WaDisableDopClockGating:bdw
7345          *
7346          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7347          * clock gating.
7348          */
7349         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7350                    intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7351 }
7352
7353 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7354 {
7355         enum pipe pipe;
7356
7357         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7358         intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7359                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7360                    HSW_FBCQ_DIS);
7361
7362         for_each_pipe(dev_priv, pipe) {
7363                 /* Undocumented but fixes async flip + VT-d corruption */
7364                 if (intel_vtd_active())
7365                         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7366                                          HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7367         }
7368
7369         /* This is required by WaCatErrorRejectionIssue:hsw */
7370         intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7371                    intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7372                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7373
7374         /* WaSwitchSolVfFArbitrationPriority:hsw */
7375         intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7376
7377         lpt_init_clock_gating(dev_priv);
7378 }
7379
7380 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7381 {
7382         u32 snpcr;
7383
7384         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7385
7386         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7387         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7388                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7389                    ILK_FBCQ_DIS);
7390
7391         /* WaDisableBackToBackFlipFix:ivb */
7392         intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7393                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7394                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7395
7396         if (IS_IVB_GT1(dev_priv))
7397                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7398                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7399         else {
7400                 /* must write both registers */
7401                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7402                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7403                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7404                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7405         }
7406
7407         /*
7408          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7409          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7410          */
7411         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7412                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7413
7414         /* This is required by WaCatErrorRejectionIssue:ivb */
7415         intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7416                         intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7417                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7418
7419         g4x_disable_trickle_feed(dev_priv);
7420
7421         snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7422         snpcr &= ~GEN6_MBC_SNPCR_MASK;
7423         snpcr |= GEN6_MBC_SNPCR_MED;
7424         intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7425
7426         if (!HAS_PCH_NOP(dev_priv))
7427                 cpt_init_clock_gating(dev_priv);
7428
7429         gen6_check_mch_setup(dev_priv);
7430 }
7431
7432 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7433 {
7434         /* WaDisableBackToBackFlipFix:vlv */
7435         intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7436                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7437                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7438
7439         /* WaDisableDopClockGating:vlv */
7440         intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7441                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7442
7443         /* This is required by WaCatErrorRejectionIssue:vlv */
7444         intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7445                    intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7446                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7447
7448         /*
7449          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7450          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7451          */
7452         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7453                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7454
7455         /* WaDisableL3Bank2xClockGate:vlv
7456          * Disabling L3 clock gating- MMIO 940c[25] = 1
7457          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7458         intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7459                    intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7460
7461         /*
7462          * WaDisableVLVClockGating_VBIIssue:vlv
7463          * Disable clock gating on th GCFG unit to prevent a delay
7464          * in the reporting of vblank events.
7465          */
7466         intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7467 }
7468
7469 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7470 {
7471         /* WaVSRefCountFullforceMissDisable:chv */
7472         /* WaDSRefCountFullforceMissDisable:chv */
7473         intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7474                    intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7475                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7476
7477         /* WaDisableSemaphoreAndSyncFlipWait:chv */
7478         intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7479                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7480
7481         /* WaDisableCSUnitClockGating:chv */
7482         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7483                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7484
7485         /* WaDisableSDEUnitClockGating:chv */
7486         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7487                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7488
7489         /*
7490          * WaProgramL3SqcReg1Default:chv
7491          * See gfxspecs/Related Documents/Performance Guide/
7492          * LSQC Setting Recommendations.
7493          */
7494         gen8_set_l3sqc_credits(dev_priv, 38, 2);
7495 }
7496
7497 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7498 {
7499         u32 dspclk_gate;
7500
7501         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7502         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7503                    GS_UNIT_CLOCK_GATE_DISABLE |
7504                    CL_UNIT_CLOCK_GATE_DISABLE);
7505         intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7506         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7507                 OVRUNIT_CLOCK_GATE_DISABLE |
7508                 OVCUNIT_CLOCK_GATE_DISABLE;
7509         if (IS_GM45(dev_priv))
7510                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7511         intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7512
7513         g4x_disable_trickle_feed(dev_priv);
7514 }
7515
7516 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7517 {
7518         struct intel_uncore *uncore = &dev_priv->uncore;
7519
7520         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7521         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7522         intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7523         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7524         intel_uncore_write16(uncore, DEUC, 0);
7525         intel_uncore_write(uncore,
7526                            MI_ARB_STATE,
7527                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7528 }
7529
7530 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7531 {
7532         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7533                    I965_RCC_CLOCK_GATE_DISABLE |
7534                    I965_RCPB_CLOCK_GATE_DISABLE |
7535                    I965_ISC_CLOCK_GATE_DISABLE |
7536                    I965_FBC_CLOCK_GATE_DISABLE);
7537         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
7538         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7539                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7540 }
7541
7542 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7543 {
7544         u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
7545
7546         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7547                 DSTATE_DOT_CLOCK_GATING;
7548         intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
7549
7550         if (IS_PINEVIEW(dev_priv))
7551                 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7552
7553         /* IIR "flip pending" means done if this bit is set */
7554         intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7555
7556         /* interrupts should cause a wake up from C3 */
7557         intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7558
7559         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7560         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7561
7562         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7563                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7564 }
7565
7566 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7567 {
7568         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7569
7570         /* interrupts should cause a wake up from C3 */
7571         intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7572                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7573
7574         intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7575                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7576
7577         /*
7578          * Have FBC ignore 3D activity since we use software
7579          * render tracking, and otherwise a pure 3D workload
7580          * (even if it just renders a single frame and then does
7581          * abosultely nothing) would not allow FBC to recompress
7582          * until a 2D blit occurs.
7583          */
7584         intel_uncore_write(&dev_priv->uncore, SCPD0,
7585                    _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7586 }
7587
7588 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7589 {
7590         intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7591                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7592                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7593 }
7594
7595 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7596 {
7597         dev_priv->display.init_clock_gating(dev_priv);
7598 }
7599
7600 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7601 {
7602         if (HAS_PCH_LPT(dev_priv))
7603                 lpt_suspend_hw(dev_priv);
7604 }
7605
7606 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7607 {
7608         drm_dbg_kms(&dev_priv->drm,
7609                     "No clock gating settings or workarounds applied.\n");
7610 }
7611
7612 /**
7613  * intel_init_clock_gating_hooks - setup the clock gating hooks
7614  * @dev_priv: device private
7615  *
7616  * Setup the hooks that configure which clocks of a given platform can be
7617  * gated and also apply various GT and display specific workarounds for these
7618  * platforms. Note that some GT specific workarounds are applied separately
7619  * when GPU contexts or batchbuffers start their execution.
7620  */
7621 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7622 {
7623         if (IS_DG1(dev_priv))
7624                 dev_priv->display.init_clock_gating = dg1_init_clock_gating;
7625         else if (IS_GEN(dev_priv, 12))
7626                 dev_priv->display.init_clock_gating = gen12lp_init_clock_gating;
7627         else if (IS_GEN(dev_priv, 11))
7628                 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7629         else if (IS_CANNONLAKE(dev_priv))
7630                 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7631         else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7632                 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7633         else if (IS_SKYLAKE(dev_priv))
7634                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7635         else if (IS_KABYLAKE(dev_priv))
7636                 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7637         else if (IS_BROXTON(dev_priv))
7638                 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7639         else if (IS_GEMINILAKE(dev_priv))
7640                 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7641         else if (IS_BROADWELL(dev_priv))
7642                 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7643         else if (IS_CHERRYVIEW(dev_priv))
7644                 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7645         else if (IS_HASWELL(dev_priv))
7646                 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7647         else if (IS_IVYBRIDGE(dev_priv))
7648                 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7649         else if (IS_VALLEYVIEW(dev_priv))
7650                 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7651         else if (IS_GEN(dev_priv, 6))
7652                 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7653         else if (IS_GEN(dev_priv, 5))
7654                 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7655         else if (IS_G4X(dev_priv))
7656                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7657         else if (IS_I965GM(dev_priv))
7658                 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7659         else if (IS_I965G(dev_priv))
7660                 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7661         else if (IS_GEN(dev_priv, 3))
7662                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7663         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7664                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7665         else if (IS_GEN(dev_priv, 2))
7666                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7667         else {
7668                 MISSING_CASE(INTEL_DEVID(dev_priv));
7669                 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7670         }
7671 }
7672
7673 /* Set up chip specific power management-related functions */
7674 void intel_init_pm(struct drm_i915_private *dev_priv)
7675 {
7676         /* For cxsr */
7677         if (IS_PINEVIEW(dev_priv))
7678                 pnv_get_mem_freq(dev_priv);
7679         else if (IS_GEN(dev_priv, 5))
7680                 ilk_get_mem_freq(dev_priv);
7681
7682         if (intel_has_sagv(dev_priv))
7683                 skl_setup_sagv_block_time(dev_priv);
7684
7685         /* For FIFO watermark updates */
7686         if (DISPLAY_VER(dev_priv) >= 9) {
7687                 skl_setup_wm_latency(dev_priv);
7688                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7689         } else if (HAS_PCH_SPLIT(dev_priv)) {
7690                 ilk_setup_wm_latency(dev_priv);
7691
7692                 if ((IS_DISPLAY_VER(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7693                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7694                     (!IS_DISPLAY_VER(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7695                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7696                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7697                         dev_priv->display.compute_intermediate_wm =
7698                                 ilk_compute_intermediate_wm;
7699                         dev_priv->display.initial_watermarks =
7700                                 ilk_initial_watermarks;
7701                         dev_priv->display.optimize_watermarks =
7702                                 ilk_optimize_watermarks;
7703                 } else {
7704                         drm_dbg_kms(&dev_priv->drm,
7705                                     "Failed to read display plane latency. "
7706                                     "Disable CxSR\n");
7707                 }
7708         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7709                 vlv_setup_wm_latency(dev_priv);
7710                 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7711                 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7712                 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7713                 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7714                 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7715         } else if (IS_G4X(dev_priv)) {
7716                 g4x_setup_wm_latency(dev_priv);
7717                 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7718                 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7719                 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7720                 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7721         } else if (IS_PINEVIEW(dev_priv)) {
7722                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7723                                             dev_priv->is_ddr3,
7724                                             dev_priv->fsb_freq,
7725                                             dev_priv->mem_freq)) {
7726                         drm_info(&dev_priv->drm,
7727                                  "failed to find known CxSR latency "
7728                                  "(found ddr%s fsb freq %d, mem freq %d), "
7729                                  "disabling CxSR\n",
7730                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
7731                                  dev_priv->fsb_freq, dev_priv->mem_freq);
7732                         /* Disable CxSR and never update its watermark again */
7733                         intel_set_memory_cxsr(dev_priv, false);
7734                         dev_priv->display.update_wm = NULL;
7735                 } else
7736                         dev_priv->display.update_wm = pnv_update_wm;
7737         } else if (IS_DISPLAY_VER(dev_priv, 4)) {
7738                 dev_priv->display.update_wm = i965_update_wm;
7739         } else if (IS_DISPLAY_VER(dev_priv, 3)) {
7740                 dev_priv->display.update_wm = i9xx_update_wm;
7741                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7742         } else if (IS_DISPLAY_VER(dev_priv, 2)) {
7743                 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7744                         dev_priv->display.update_wm = i845_update_wm;
7745                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
7746                 } else {
7747                         dev_priv->display.update_wm = i9xx_update_wm;
7748                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
7749                 }
7750         } else {
7751                 drm_err(&dev_priv->drm,
7752                         "unexpected fall-through in %s\n", __func__);
7753         }
7754 }
7755
7756 void intel_pm_setup(struct drm_i915_private *dev_priv)
7757 {
7758         dev_priv->runtime_pm.suspended = false;
7759         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7760 }
7761
7762 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
7763 {
7764         struct intel_dbuf_state *dbuf_state;
7765
7766         dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
7767         if (!dbuf_state)
7768                 return NULL;
7769
7770         return &dbuf_state->base;
7771 }
7772
7773 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
7774                                      struct intel_global_state *state)
7775 {
7776         kfree(state);
7777 }
7778
7779 static const struct intel_global_state_funcs intel_dbuf_funcs = {
7780         .atomic_duplicate_state = intel_dbuf_duplicate_state,
7781         .atomic_destroy_state = intel_dbuf_destroy_state,
7782 };
7783
7784 struct intel_dbuf_state *
7785 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
7786 {
7787         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7788         struct intel_global_state *dbuf_state;
7789
7790         dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
7791         if (IS_ERR(dbuf_state))
7792                 return ERR_CAST(dbuf_state);
7793
7794         return to_intel_dbuf_state(dbuf_state);
7795 }
7796
7797 int intel_dbuf_init(struct drm_i915_private *dev_priv)
7798 {
7799         struct intel_dbuf_state *dbuf_state;
7800
7801         dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
7802         if (!dbuf_state)
7803                 return -ENOMEM;
7804
7805         intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
7806                                      &dbuf_state->base, &intel_dbuf_funcs);
7807
7808         return 0;
7809 }
7810
7811 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
7812 {
7813         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7814         const struct intel_dbuf_state *new_dbuf_state =
7815                 intel_atomic_get_new_dbuf_state(state);
7816         const struct intel_dbuf_state *old_dbuf_state =
7817                 intel_atomic_get_old_dbuf_state(state);
7818
7819         if (!new_dbuf_state ||
7820             new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7821                 return;
7822
7823         WARN_ON(!new_dbuf_state->base.changed);
7824
7825         gen9_dbuf_slices_update(dev_priv,
7826                                 old_dbuf_state->enabled_slices |
7827                                 new_dbuf_state->enabled_slices);
7828 }
7829
7830 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
7831 {
7832         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7833         const struct intel_dbuf_state *new_dbuf_state =
7834                 intel_atomic_get_new_dbuf_state(state);
7835         const struct intel_dbuf_state *old_dbuf_state =
7836                 intel_atomic_get_old_dbuf_state(state);
7837
7838         if (!new_dbuf_state ||
7839             new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7840                 return;
7841
7842         WARN_ON(!new_dbuf_state->base.changed);
7843
7844         gen9_dbuf_slices_update(dev_priv,
7845                                 new_dbuf_state->enabled_slices);
7846 }