drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/i2c.h>
  28 #include <linux/input.h>
  29 #include <linux/intel-iommu.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/dma-resv.h>
  33 #include <linux/slab.h>
  34 #include <linux/vgaarb.h>
  35
  36 #include <drm/drm_atomic.h>
  37 #include <drm/drm_atomic_helper.h>
  38 #include <drm/drm_atomic_uapi.h>
  39 #include <drm/drm_dp_helper.h>
  40 #include <drm/drm_edid.h>
  41 #include <drm/drm_fourcc.h>
  42 #include <drm/drm_plane_helper.h>
  43 #include <drm/drm_probe_helper.h>
  44 #include <drm/drm_rect.h>
  45 #include <drm/i915_drm.h>
  46
  47 #include "display/intel_crt.h"
  48 #include "display/intel_ddi.h"
  49 #include "display/intel_dp.h"
  50 #include "display/intel_dsi.h"
  51 #include "display/intel_dvo.h"
  52 #include "display/intel_gmbus.h"
  53 #include "display/intel_hdmi.h"
  54 #include "display/intel_lvds.h"
  55 #include "display/intel_sdvo.h"
  56 #include "display/intel_tv.h"
  57 #include "display/intel_vdsc.h"
  58
  59 #include "i915_drv.h"
  60 #include "i915_trace.h"
  61 #include "intel_acpi.h"
  62 #include "intel_atomic.h"
  63 #include "intel_atomic_plane.h"
  64 #include "intel_bw.h"
  65 #include "intel_cdclk.h"
  66 #include "intel_color.h"
  67 #include "intel_display_types.h"
  68 #include "intel_fbc.h"
  69 #include "intel_fbdev.h"
  70 #include "intel_fifo_underrun.h"
  71 #include "intel_frontbuffer.h"
  72 #include "intel_hdcp.h"
  73 #include "intel_hotplug.h"
  74 #include "intel_overlay.h"
  75 #include "intel_pipe_crc.h"
  76 #include "intel_pm.h"
  77 #include "intel_psr.h"
  78 #include "intel_quirks.h"
  79 #include "intel_sideband.h"
  80 #include "intel_sprite.h"
  81 #include "intel_tc.h"
  82
  83 /* Primary plane formats for gen <= 3 */
  84 static const u32 i8xx_primary_formats[] = {
  85         DRM_FORMAT_C8,
  86         DRM_FORMAT_RGB565,
  87         DRM_FORMAT_XRGB1555,
  88         DRM_FORMAT_XRGB8888,
  89 };
  90
  91 /* Primary plane formats for gen >= 4 */
  92 static const u32 i965_primary_formats[] = {
  93         DRM_FORMAT_C8,
  94         DRM_FORMAT_RGB565,
  95         DRM_FORMAT_XRGB8888,
  96         DRM_FORMAT_XBGR8888,
  97         DRM_FORMAT_XRGB2101010,
  98         DRM_FORMAT_XBGR2101010,
  99 };
 100
 101 static const u64 i9xx_format_modifiers[] = {
 102         I915_FORMAT_MOD_X_TILED,
 103         DRM_FORMAT_MOD_LINEAR,
 104         DRM_FORMAT_MOD_INVALID
 105 };
 106
 107 /* Cursor formats */
 108 static const u32 intel_cursor_formats[] = {
 109         DRM_FORMAT_ARGB8888,
 110 };
 111
 112 static const u64 cursor_format_modifiers[] = {
 113         DRM_FORMAT_MOD_LINEAR,
 114         DRM_FORMAT_MOD_INVALID
 115 };
 116
 117 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 118                                 struct intel_crtc_state *pipe_config);
 119 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
 120                                    struct intel_crtc_state *pipe_config);
 121
 122 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 123                                   struct drm_i915_gem_object *obj,
 124                                   struct drm_mode_fb_cmd2 *mode_cmd);
 125 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
 126 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 127 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 128                                          const struct intel_link_m_n *m_n,
 129                                          const struct intel_link_m_n *m2_n2);
 130 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 131 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state);
 132 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state);
 133 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 134 static void vlv_prepare_pll(struct intel_crtc *crtc,
 135                             const struct intel_crtc_state *pipe_config);
 136 static void chv_prepare_pll(struct intel_crtc *crtc,
 137                             const struct intel_crtc_state *pipe_config);
 138 static void intel_begin_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
 139 static void intel_finish_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
 140 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
 141                                     struct intel_crtc_state *crtc_state);
 142 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
 143 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state);
 144 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state);
 145 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 146                                          struct drm_modeset_acquire_ctx *ctx);
 147 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
 148
 149 struct intel_limit {
 150         struct {
 151                 int min, max;
 152         } dot, vco, n, m, m1, m2, p, p1;
 153
 154         struct {
 155                 int dot_limit;
 156                 int p2_slow, p2_fast;
 157         } p2;
 158 };
 159
 160 /* returns HPLL frequency in kHz */
 161 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 162 {
 163         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 164
 165         /* Obtain SKU information */
 166         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 167                 CCK_FUSE_HPLL_FREQ_MASK;
 168
 169         return vco_freq[hpll_freq] * 1000;
 170 }
 171
 172 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 173                       const char *name, u32 reg, int ref_freq)
 174 {
 175         u32 val;
 176         int divider;
 177
 178         val = vlv_cck_read(dev_priv, reg);
 179         divider = val & CCK_FREQUENCY_VALUES;
 180
 181         WARN((val & CCK_FREQUENCY_STATUS) !=
 182              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 183              "%s change in progress\n", name);
 184
 185         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 186 }
 187
 188 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 189                            const char *name, u32 reg)
 190 {
 191         int hpll;
 192
 193         vlv_cck_get(dev_priv);
 194
 195         if (dev_priv->hpll_freq == 0)
 196                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 197
 198         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 199
 200         vlv_cck_put(dev_priv);
 201
 202         return hpll;
 203 }
 204
 205 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 206 {
 207         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 208                 return;
 209
 210         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 211                                                       CCK_CZ_CLOCK_CONTROL);
 212
 213         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 214 }
 215
 216 static inline u32 /* units of 100MHz */
 217 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 218                     const struct intel_crtc_state *pipe_config)
 219 {
 220         if (HAS_DDI(dev_priv))
 221                 return pipe_config->port_clock; /* SPLL */
 222         else
 223                 return dev_priv->fdi_pll_freq;
 224 }
 225
 226 static const struct intel_limit intel_limits_i8xx_dac = {
 227         .dot = { .min = 25000, .max = 350000 },
 228         .vco = { .min = 908000, .max = 1512000 },
 229         .n = { .min = 2, .max = 16 },
 230         .m = { .min = 96, .max = 140 },
 231         .m1 = { .min = 18, .max = 26 },
 232         .m2 = { .min = 6, .max = 16 },
 233         .p = { .min = 4, .max = 128 },
 234         .p1 = { .min = 2, .max = 33 },
 235         .p2 = { .dot_limit = 165000,
 236                 .p2_slow = 4, .p2_fast = 2 },
 237 };
 238
 239 static const struct intel_limit intel_limits_i8xx_dvo = {
 240         .dot = { .min = 25000, .max = 350000 },
 241         .vco = { .min = 908000, .max = 1512000 },
 242         .n = { .min = 2, .max = 16 },
 243         .m = { .min = 96, .max = 140 },
 244         .m1 = { .min = 18, .max = 26 },
 245         .m2 = { .min = 6, .max = 16 },
 246         .p = { .min = 4, .max = 128 },
 247         .p1 = { .min = 2, .max = 33 },
 248         .p2 = { .dot_limit = 165000,
 249                 .p2_slow = 4, .p2_fast = 4 },
 250 };
 251
 252 static const struct intel_limit intel_limits_i8xx_lvds = {
 253         .dot = { .min = 25000, .max = 350000 },
 254         .vco = { .min = 908000, .max = 1512000 },
 255         .n = { .min = 2, .max = 16 },
 256         .m = { .min = 96, .max = 140 },
 257         .m1 = { .min = 18, .max = 26 },
 258         .m2 = { .min = 6, .max = 16 },
 259         .p = { .min = 4, .max = 128 },
 260         .p1 = { .min = 1, .max = 6 },
 261         .p2 = { .dot_limit = 165000,
 262                 .p2_slow = 14, .p2_fast = 7 },
 263 };
 264
 265 static const struct intel_limit intel_limits_i9xx_sdvo = {
 266         .dot = { .min = 20000, .max = 400000 },
 267         .vco = { .min = 1400000, .max = 2800000 },
 268         .n = { .min = 1, .max = 6 },
 269         .m = { .min = 70, .max = 120 },
 270         .m1 = { .min = 8, .max = 18 },
 271         .m2 = { .min = 3, .max = 7 },
 272         .p = { .min = 5, .max = 80 },
 273         .p1 = { .min = 1, .max = 8 },
 274         .p2 = { .dot_limit = 200000,
 275                 .p2_slow = 10, .p2_fast = 5 },
 276 };
 277
 278 static const struct intel_limit intel_limits_i9xx_lvds = {
 279         .dot = { .min = 20000, .max = 400000 },
 280         .vco = { .min = 1400000, .max = 2800000 },
 281         .n = { .min = 1, .max = 6 },
 282         .m = { .min = 70, .max = 120 },
 283         .m1 = { .min = 8, .max = 18 },
 284         .m2 = { .min = 3, .max = 7 },
 285         .p = { .min = 7, .max = 98 },
 286         .p1 = { .min = 1, .max = 8 },
 287         .p2 = { .dot_limit = 112000,
 288                 .p2_slow = 14, .p2_fast = 7 },
 289 };
 290
 291
 292 static const struct intel_limit intel_limits_g4x_sdvo = {
 293         .dot = { .min = 25000, .max = 270000 },
 294         .vco = { .min = 1750000, .max = 3500000},
 295         .n = { .min = 1, .max = 4 },
 296         .m = { .min = 104, .max = 138 },
 297         .m1 = { .min = 17, .max = 23 },
 298         .m2 = { .min = 5, .max = 11 },
 299         .p = { .min = 10, .max = 30 },
 300         .p1 = { .min = 1, .max = 3},
 301         .p2 = { .dot_limit = 270000,
 302                 .p2_slow = 10,
 303                 .p2_fast = 10
 304         },
 305 };
 306
 307 static const struct intel_limit intel_limits_g4x_hdmi = {
 308         .dot = { .min = 22000, .max = 400000 },
 309         .vco = { .min = 1750000, .max = 3500000},
 310         .n = { .min = 1, .max = 4 },
 311         .m = { .min = 104, .max = 138 },
 312         .m1 = { .min = 16, .max = 23 },
 313         .m2 = { .min = 5, .max = 11 },
 314         .p = { .min = 5, .max = 80 },
 315         .p1 = { .min = 1, .max = 8},
 316         .p2 = { .dot_limit = 165000,
 317                 .p2_slow = 10, .p2_fast = 5 },
 318 };
 319
 320 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 321         .dot = { .min = 20000, .max = 115000 },
 322         .vco = { .min = 1750000, .max = 3500000 },
 323         .n = { .min = 1, .max = 3 },
 324         .m = { .min = 104, .max = 138 },
 325         .m1 = { .min = 17, .max = 23 },
 326         .m2 = { .min = 5, .max = 11 },
 327         .p = { .min = 28, .max = 112 },
 328         .p1 = { .min = 2, .max = 8 },
 329         .p2 = { .dot_limit = 0,
 330                 .p2_slow = 14, .p2_fast = 14
 331         },
 332 };
 333
 334 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 335         .dot = { .min = 80000, .max = 224000 },
 336         .vco = { .min = 1750000, .max = 3500000 },
 337         .n = { .min = 1, .max = 3 },
 338         .m = { .min = 104, .max = 138 },
 339         .m1 = { .min = 17, .max = 23 },
 340         .m2 = { .min = 5, .max = 11 },
 341         .p = { .min = 14, .max = 42 },
 342         .p1 = { .min = 2, .max = 6 },
 343         .p2 = { .dot_limit = 0,
 344                 .p2_slow = 7, .p2_fast = 7
 345         },
 346 };
 347
 348 static const struct intel_limit intel_limits_pineview_sdvo = {
 349         .dot = { .min = 20000, .max = 400000},
 350         .vco = { .min = 1700000, .max = 3500000 },
 351         /* Pineview's Ncounter is a ring counter */
 352         .n = { .min = 3, .max = 6 },
 353         .m = { .min = 2, .max = 256 },
 354         /* Pineview only has one combined m divider, which we treat as m2. */
 355         .m1 = { .min = 0, .max = 0 },
 356         .m2 = { .min = 0, .max = 254 },
 357         .p = { .min = 5, .max = 80 },
 358         .p1 = { .min = 1, .max = 8 },
 359         .p2 = { .dot_limit = 200000,
 360                 .p2_slow = 10, .p2_fast = 5 },
 361 };
 362
 363 static const struct intel_limit intel_limits_pineview_lvds = {
 364         .dot = { .min = 20000, .max = 400000 },
 365         .vco = { .min = 1700000, .max = 3500000 },
 366         .n = { .min = 3, .max = 6 },
 367         .m = { .min = 2, .max = 256 },
 368         .m1 = { .min = 0, .max = 0 },
 369         .m2 = { .min = 0, .max = 254 },
 370         .p = { .min = 7, .max = 112 },
 371         .p1 = { .min = 1, .max = 8 },
 372         .p2 = { .dot_limit = 112000,
 373                 .p2_slow = 14, .p2_fast = 14 },
 374 };
 375
 376 /* Ironlake / Sandybridge
 377  *
 378  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 379  * the range value for them is (actual_value - 2).
 380  */
 381 static const struct intel_limit intel_limits_ironlake_dac = {
 382         .dot = { .min = 25000, .max = 350000 },
 383         .vco = { .min = 1760000, .max = 3510000 },
 384         .n = { .min = 1, .max = 5 },
 385         .m = { .min = 79, .max = 127 },
 386         .m1 = { .min = 12, .max = 22 },
 387         .m2 = { .min = 5, .max = 9 },
 388         .p = { .min = 5, .max = 80 },
 389         .p1 = { .min = 1, .max = 8 },
 390         .p2 = { .dot_limit = 225000,
 391                 .p2_slow = 10, .p2_fast = 5 },
 392 };
 393
 394 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 395         .dot = { .min = 25000, .max = 350000 },
 396         .vco = { .min = 1760000, .max = 3510000 },
 397         .n = { .min = 1, .max = 3 },
 398         .m = { .min = 79, .max = 118 },
 399         .m1 = { .min = 12, .max = 22 },
 400         .m2 = { .min = 5, .max = 9 },
 401         .p = { .min = 28, .max = 112 },
 402         .p1 = { .min = 2, .max = 8 },
 403         .p2 = { .dot_limit = 225000,
 404                 .p2_slow = 14, .p2_fast = 14 },
 405 };
 406
 407 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 408         .dot = { .min = 25000, .max = 350000 },
 409         .vco = { .min = 1760000, .max = 3510000 },
 410         .n = { .min = 1, .max = 3 },
 411         .m = { .min = 79, .max = 127 },
 412         .m1 = { .min = 12, .max = 22 },
 413         .m2 = { .min = 5, .max = 9 },
 414         .p = { .min = 14, .max = 56 },
 415         .p1 = { .min = 2, .max = 8 },
 416         .p2 = { .dot_limit = 225000,
 417                 .p2_slow = 7, .p2_fast = 7 },
 418 };
 419
 420 /* LVDS 100mhz refclk limits. */
 421 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 422         .dot = { .min = 25000, .max = 350000 },
 423         .vco = { .min = 1760000, .max = 3510000 },
 424         .n = { .min = 1, .max = 2 },
 425         .m = { .min = 79, .max = 126 },
 426         .m1 = { .min = 12, .max = 22 },
 427         .m2 = { .min = 5, .max = 9 },
 428         .p = { .min = 28, .max = 112 },
 429         .p1 = { .min = 2, .max = 8 },
 430         .p2 = { .dot_limit = 225000,
 431                 .p2_slow = 14, .p2_fast = 14 },
 432 };
 433
 434 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 435         .dot = { .min = 25000, .max = 350000 },
 436         .vco = { .min = 1760000, .max = 3510000 },
 437         .n = { .min = 1, .max = 3 },
 438         .m = { .min = 79, .max = 126 },
 439         .m1 = { .min = 12, .max = 22 },
 440         .m2 = { .min = 5, .max = 9 },
 441         .p = { .min = 14, .max = 42 },
 442         .p1 = { .min = 2, .max = 6 },
 443         .p2 = { .dot_limit = 225000,
 444                 .p2_slow = 7, .p2_fast = 7 },
 445 };
 446
 447 static const struct intel_limit intel_limits_vlv = {
 448          /*
 449           * These are the data rate limits (measured in fast clocks)
 450           * since those are the strictest limits we have. The fast
 451           * clock and actual rate limits are more relaxed, so checking
 452           * them would make no difference.
 453           */
 454         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 455         .vco = { .min = 4000000, .max = 6000000 },
 456         .n = { .min = 1, .max = 7 },
 457         .m1 = { .min = 2, .max = 3 },
 458         .m2 = { .min = 11, .max = 156 },
 459         .p1 = { .min = 2, .max = 3 },
 460         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 461 };
 462
 463 static const struct intel_limit intel_limits_chv = {
 464         /*
 465          * These are the data rate limits (measured in fast clocks)
 466          * since those are the strictest limits we have.  The fast
 467          * clock and actual rate limits are more relaxed, so checking
 468          * them would make no difference.
 469          */
 470         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 471         .vco = { .min = 4800000, .max = 6480000 },
 472         .n = { .min = 1, .max = 1 },
 473         .m1 = { .min = 2, .max = 2 },
 474         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 475         .p1 = { .min = 2, .max = 4 },
 476         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 477 };
 478
 479 static const struct intel_limit intel_limits_bxt = {
 480         /* FIXME: find real dot limits */
 481         .dot = { .min = 0, .max = INT_MAX },
 482         .vco = { .min = 4800000, .max = 6700000 },
 483         .n = { .min = 1, .max = 1 },
 484         .m1 = { .min = 2, .max = 2 },
 485         /* FIXME: find real m2 limits */
 486         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 487         .p1 = { .min = 2, .max = 4 },
 488         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 489 };
 490
 491 /* WA Display #0827: Gen9:all */
 492 static void
 493 skl_wa_827(struct drm_i915_private *dev_priv, int pipe, bool enable)
 494 {
 495         if (enable)
 496                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 497                            I915_READ(CLKGATE_DIS_PSL(pipe)) |
 498                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 499         else
 500                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 501                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 502                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 503 }
 504
 505 /* Wa_2006604312:icl */
 506 static void
 507 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 508                        bool enable)
 509 {
 510         if (enable)
 511                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 512                            I915_READ(CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 513         else
 514                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 515                            I915_READ(CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 516 }
 517
 518 static bool
 519 needs_modeset(const struct intel_crtc_state *state)
 520 {
 521         return drm_atomic_crtc_needs_modeset(&state->base);
 522 }
 523
 524 /*
 525  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 526  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 527  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 528  * The helpers' return value is the rate of the clock that is fed to the
 529  * display engine's pipe which can be the above fast dot clock rate or a
 530  * divided-down version of it.
 531  */
 532 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 533 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 534 {
 535         clock->m = clock->m2 + 2;
 536         clock->p = clock->p1 * clock->p2;
 537         if (WARN_ON(clock->n == 0 || clock->p == 0))
 538                 return 0;
 539         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 540         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 541
 542         return clock->dot;
 543 }
 544
 545 static u32 i9xx_dpll_compute_m(struct dpll *dpll)
 546 {
 547         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 548 }
 549
 550 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 551 {
 552         clock->m = i9xx_dpll_compute_m(clock);
 553         clock->p = clock->p1 * clock->p2;
 554         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 555                 return 0;
 556         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 557         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 558
 559         return clock->dot;
 560 }
 561
 562 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 563 {
 564         clock->m = clock->m1 * clock->m2;
 565         clock->p = clock->p1 * clock->p2;
 566         if (WARN_ON(clock->n == 0 || clock->p == 0))
 567                 return 0;
 568         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 569         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 570
 571         return clock->dot / 5;
 572 }
 573
 574 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 575 {
 576         clock->m = clock->m1 * clock->m2;
 577         clock->p = clock->p1 * clock->p2;
 578         if (WARN_ON(clock->n == 0 || clock->p == 0))
 579                 return 0;
 580         clock->vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock->m),
 581                                            clock->n << 22);
 582         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 583
 584         return clock->dot / 5;
 585 }
 586
 587 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 588
 589 /*
 590  * Returns whether the given set of divisors are valid for a given refclk with
 591  * the given connectors.
 592  */
 593 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 594                                const struct intel_limit *limit,
 595                                const struct dpll *clock)
 596 {
 597         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 598                 INTELPllInvalid("n out of range\n");
 599         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 600                 INTELPllInvalid("p1 out of range\n");
 601         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 602                 INTELPllInvalid("m2 out of range\n");
 603         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 604                 INTELPllInvalid("m1 out of range\n");
 605
 606         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 607             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 608                 if (clock->m1 <= clock->m2)
 609                         INTELPllInvalid("m1 <= m2\n");
 610
 611         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 612             !IS_GEN9_LP(dev_priv)) {
 613                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 614                         INTELPllInvalid("p out of range\n");
 615                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 616                         INTELPllInvalid("m out of range\n");
 617         }
 618
 619         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 620                 INTELPllInvalid("vco out of range\n");
 621         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 622          * connector, etc., rather than just a single range.
 623          */
 624         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 625                 INTELPllInvalid("dot out of range\n");
 626
 627         return true;
 628 }
 629
 630 static int
 631 i9xx_select_p2_div(const struct intel_limit *limit,
 632                    const struct intel_crtc_state *crtc_state,
 633                    int target)
 634 {
 635         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 636
 637         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 638                 /*
 639                  * For LVDS just rely on its current settings for dual-channel.
 640                  * We haven't figured out how to reliably set up different
 641                  * single/dual channel state, if we even can.
 642                  */
 643                 if (intel_is_dual_link_lvds(dev_priv))
 644                         return limit->p2.p2_fast;
 645                 else
 646                         return limit->p2.p2_slow;
 647         } else {
 648                 if (target < limit->p2.dot_limit)
 649                         return limit->p2.p2_slow;
 650                 else
 651                         return limit->p2.p2_fast;
 652         }
 653 }
 654
 655 /*
 656  * Returns a set of divisors for the desired target clock with the given
 657  * refclk, or FALSE.  The returned values represent the clock equation:
 658  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 659  *
 660  * Target and reference clocks are specified in kHz.
 661  *
 662  * If match_clock is provided, then best_clock P divider must match the P
 663  * divider from @match_clock used for LVDS downclocking.
 664  */
 665 static bool
 666 i9xx_find_best_dpll(const struct intel_limit *limit,
 667                     struct intel_crtc_state *crtc_state,
 668                     int target, int refclk, struct dpll *match_clock,
 669                     struct dpll *best_clock)
 670 {
 671         struct drm_device *dev = crtc_state->base.crtc->dev;
 672         struct dpll clock;
 673         int err = target;
 674
 675         memset(best_clock, 0, sizeof(*best_clock));
 676
 677         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 678
 679         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 680              clock.m1++) {
 681                 for (clock.m2 = limit->m2.min;
 682                      clock.m2 <= limit->m2.max; clock.m2++) {
 683                         if (clock.m2 >= clock.m1)
 684                                 break;
 685                         for (clock.n = limit->n.min;
 686                              clock.n <= limit->n.max; clock.n++) {
 687                                 for (clock.p1 = limit->p1.min;
 688                                         clock.p1 <= limit->p1.max; clock.p1++) {
 689                                         int this_err;
 690
 691                                         i9xx_calc_dpll_params(refclk, &clock);
 692                                         if (!intel_PLL_is_valid(to_i915(dev),
 693                                                                 limit,
 694                                                                 &clock))
 695                                                 continue;
 696                                         if (match_clock &&
 697                                             clock.p != match_clock->p)
 698                                                 continue;
 699
 700                                         this_err = abs(clock.dot - target);
 701                                         if (this_err < err) {
 702                                                 *best_clock = clock;
 703                                                 err = this_err;
 704                                         }
 705                                 }
 706                         }
 707                 }
 708         }
 709
 710         return (err != target);
 711 }
 712
 713 /*
 714  * Returns a set of divisors for the desired target clock with the given
 715  * refclk, or FALSE.  The returned values represent the clock equation:
 716  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 717  *
 718  * Target and reference clocks are specified in kHz.
 719  *
 720  * If match_clock is provided, then best_clock P divider must match the P
 721  * divider from @match_clock used for LVDS downclocking.
 722  */
 723 static bool
 724 pnv_find_best_dpll(const struct intel_limit *limit,
 725                    struct intel_crtc_state *crtc_state,
 726                    int target, int refclk, struct dpll *match_clock,
 727                    struct dpll *best_clock)
 728 {
 729         struct drm_device *dev = crtc_state->base.crtc->dev;
 730         struct dpll clock;
 731         int err = target;
 732
 733         memset(best_clock, 0, sizeof(*best_clock));
 734
 735         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 736
 737         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 738              clock.m1++) {
 739                 for (clock.m2 = limit->m2.min;
 740                      clock.m2 <= limit->m2.max; clock.m2++) {
 741                         for (clock.n = limit->n.min;
 742                              clock.n <= limit->n.max; clock.n++) {
 743                                 for (clock.p1 = limit->p1.min;
 744                                         clock.p1 <= limit->p1.max; clock.p1++) {
 745                                         int this_err;
 746
 747                                         pnv_calc_dpll_params(refclk, &clock);
 748                                         if (!intel_PLL_is_valid(to_i915(dev),
 749                                                                 limit,
 750                                                                 &clock))
 751                                                 continue;
 752                                         if (match_clock &&
 753                                             clock.p != match_clock->p)
 754                                                 continue;
 755
 756                                         this_err = abs(clock.dot - target);
 757                                         if (this_err < err) {
 758                                                 *best_clock = clock;
 759                                                 err = this_err;
 760                                         }
 761                                 }
 762                         }
 763                 }
 764         }
 765
 766         return (err != target);
 767 }
 768
 769 /*
 770  * Returns a set of divisors for the desired target clock with the given
 771  * refclk, or FALSE.  The returned values represent the clock equation:
 772  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 773  *
 774  * Target and reference clocks are specified in kHz.
 775  *
 776  * If match_clock is provided, then best_clock P divider must match the P
 777  * divider from @match_clock used for LVDS downclocking.
 778  */
 779 static bool
 780 g4x_find_best_dpll(const struct intel_limit *limit,
 781                    struct intel_crtc_state *crtc_state,
 782                    int target, int refclk, struct dpll *match_clock,
 783                    struct dpll *best_clock)
 784 {
 785         struct drm_device *dev = crtc_state->base.crtc->dev;
 786         struct dpll clock;
 787         int max_n;
 788         bool found = false;
 789         /* approximately equals target * 0.00585 */
 790         int err_most = (target >> 8) + (target >> 9);
 791
 792         memset(best_clock, 0, sizeof(*best_clock));
 793
 794         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 795
 796         max_n = limit->n.max;
 797         /* based on hardware requirement, prefer smaller n to precision */
 798         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 799                 /* based on hardware requirement, prefere larger m1,m2 */
 800                 for (clock.m1 = limit->m1.max;
 801                      clock.m1 >= limit->m1.min; clock.m1--) {
 802                         for (clock.m2 = limit->m2.max;
 803                              clock.m2 >= limit->m2.min; clock.m2--) {
 804                                 for (clock.p1 = limit->p1.max;
 805                                      clock.p1 >= limit->p1.min; clock.p1--) {
 806                                         int this_err;
 807
 808                                         i9xx_calc_dpll_params(refclk, &clock);
 809                                         if (!intel_PLL_is_valid(to_i915(dev),
 810                                                                 limit,
 811                                                                 &clock))
 812                                                 continue;
 813
 814                                         this_err = abs(clock.dot - target);
 815                                         if (this_err < err_most) {
 816                                                 *best_clock = clock;
 817                                                 err_most = this_err;
 818                                                 max_n = clock.n;
 819                                                 found = true;
 820                                         }
 821                                 }
 822                         }
 823                 }
 824         }
 825         return found;
 826 }
 827
 828 /*
 829  * Check if the calculated PLL configuration is more optimal compared to the
 830  * best configuration and error found so far. Return the calculated error.
 831  */
 832 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 833                                const struct dpll *calculated_clock,
 834                                const struct dpll *best_clock,
 835                                unsigned int best_error_ppm,
 836                                unsigned int *error_ppm)
 837 {
 838         /*
 839          * For CHV ignore the error and consider only the P value.
 840          * Prefer a bigger P value based on HW requirements.
 841          */
 842         if (IS_CHERRYVIEW(to_i915(dev))) {
 843                 *error_ppm = 0;
 844
 845                 return calculated_clock->p > best_clock->p;
 846         }
 847
 848         if (WARN_ON_ONCE(!target_freq))
 849                 return false;
 850
 851         *error_ppm = div_u64(1000000ULL *
 852                                 abs(target_freq - calculated_clock->dot),
 853                              target_freq);
 854         /*
 855          * Prefer a better P value over a better (smaller) error if the error
 856          * is small. Ensure this preference for future configurations too by
 857          * setting the error to 0.
 858          */
 859         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 860                 *error_ppm = 0;
 861
 862                 return true;
 863         }
 864
 865         return *error_ppm + 10 < best_error_ppm;
 866 }
 867
 868 /*
 869  * Returns a set of divisors for the desired target clock with the given
 870  * refclk, or FALSE.  The returned values represent the clock equation:
 871  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 872  */
 873 static bool
 874 vlv_find_best_dpll(const struct intel_limit *limit,
 875                    struct intel_crtc_state *crtc_state,
 876                    int target, int refclk, struct dpll *match_clock,
 877                    struct dpll *best_clock)
 878 {
 879         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 880         struct drm_device *dev = crtc->base.dev;
 881         struct dpll clock;
 882         unsigned int bestppm = 1000000;
 883         /* min update 19.2 MHz */
 884         int max_n = min(limit->n.max, refclk / 19200);
 885         bool found = false;
 886
 887         target *= 5; /* fast clock */
 888
 889         memset(best_clock, 0, sizeof(*best_clock));
 890
 891         /* based on hardware requirement, prefer smaller n to precision */
 892         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 893                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 894                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 895                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 896                                 clock.p = clock.p1 * clock.p2;
 897                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 898                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 899                                         unsigned int ppm;
 900
 901                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 902                                                                      refclk * clock.m1);
 903
 904                                         vlv_calc_dpll_params(refclk, &clock);
 905
 906                                         if (!intel_PLL_is_valid(to_i915(dev),
 907                                                                 limit,
 908                                                                 &clock))
 909                                                 continue;
 910
 911                                         if (!vlv_PLL_is_optimal(dev, target,
 912                                                                 &clock,
 913                                                                 best_clock,
 914                                                                 bestppm, &ppm))
 915                                                 continue;
 916
 917                                         *best_clock = clock;
 918                                         bestppm = ppm;
 919                                         found = true;
 920                                 }
 921                         }
 922                 }
 923         }
 924
 925         return found;
 926 }
 927
 928 /*
 929  * Returns a set of divisors for the desired target clock with the given
 930  * refclk, or FALSE.  The returned values represent the clock equation:
 931  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 932  */
 933 static bool
 934 chv_find_best_dpll(const struct intel_limit *limit,
 935                    struct intel_crtc_state *crtc_state,
 936                    int target, int refclk, struct dpll *match_clock,
 937                    struct dpll *best_clock)
 938 {
 939         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 940         struct drm_device *dev = crtc->base.dev;
 941         unsigned int best_error_ppm;
 942         struct dpll clock;
 943         u64 m2;
 944         int found = false;
 945
 946         memset(best_clock, 0, sizeof(*best_clock));
 947         best_error_ppm = 1000000;
 948
 949         /*
 950          * Based on hardware doc, the n always set to 1, and m1 always
 951          * set to 2.  If requires to support 200Mhz refclk, we need to
 952          * revisit this because n may not 1 anymore.
 953          */
 954         clock.n = 1, clock.m1 = 2;
 955         target *= 5;    /* fast clock */
 956
 957         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 958                 for (clock.p2 = limit->p2.p2_fast;
 959                                 clock.p2 >= limit->p2.p2_slow;
 960                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 961                         unsigned int error_ppm;
 962
 963                         clock.p = clock.p1 * clock.p2;
 964
 965                         m2 = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(target, clock.p * clock.n) << 22,
 966                                                    refclk * clock.m1);
 967
 968                         if (m2 > INT_MAX/clock.m1)
 969                                 continue;
 970
 971                         clock.m2 = m2;
 972
 973                         chv_calc_dpll_params(refclk, &clock);
 974
 975                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
 976                                 continue;
 977
 978                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
 979                                                 best_error_ppm, &error_ppm))
 980                                 continue;
 981
 982                         *best_clock = clock;
 983                         best_error_ppm = error_ppm;
 984                         found = true;
 985                 }
 986         }
 987
 988         return found;
 989 }
 990
 991 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
 992                         struct dpll *best_clock)
 993 {
 994         int refclk = 100000;
 995         const struct intel_limit *limit = &intel_limits_bxt;
 996
 997         return chv_find_best_dpll(limit, crtc_state,
 998                                   crtc_state->port_clock, refclk,
 999                                   NULL, best_clock);
1000 }
1001
1002 bool intel_crtc_active(struct intel_crtc *crtc)
1003 {
1004         /* Be paranoid as we can arrive here with only partial
1005          * state retrieved from the hardware during setup.
1006          *
1007          * We can ditch the adjusted_mode.crtc_clock check as soon
1008          * as Haswell has gained clock readout/fastboot support.
1009          *
1010          * We can ditch the crtc->primary->state->fb check as soon as we can
1011          * properly reconstruct framebuffers.
1012          *
1013          * FIXME: The intel_crtc->active here should be switched to
1014          * crtc->state->active once we have proper CRTC states wired up
1015          * for atomic.
1016          */
1017         return crtc->active && crtc->base.primary->state->fb &&
1018                 crtc->config->base.adjusted_mode.crtc_clock;
1019 }
1020
1021 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1022                                              enum pipe pipe)
1023 {
1024         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1025
1026         return crtc->config->cpu_transcoder;
1027 }
1028
1029 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1030                                     enum pipe pipe)
1031 {
1032         i915_reg_t reg = PIPEDSL(pipe);
1033         u32 line1, line2;
1034         u32 line_mask;
1035
1036         if (IS_GEN(dev_priv, 2))
1037                 line_mask = DSL_LINEMASK_GEN2;
1038         else
1039                 line_mask = DSL_LINEMASK_GEN3;
1040
1041         line1 = I915_READ(reg) & line_mask;
1042         msleep(5);
1043         line2 = I915_READ(reg) & line_mask;
1044
1045         return line1 != line2;
1046 }
1047
1048 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1049 {
1050         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1051         enum pipe pipe = crtc->pipe;
1052
1053         /* Wait for the display line to settle/start moving */
1054         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1055                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1056                           pipe_name(pipe), onoff(state));
1057 }
1058
1059 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1060 {
1061         wait_for_pipe_scanline_moving(crtc, false);
1062 }
1063
1064 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1065 {
1066         wait_for_pipe_scanline_moving(crtc, true);
1067 }
1068
1069 static void
1070 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1071 {
1072         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1073         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1074
1075         if (INTEL_GEN(dev_priv) >= 4) {
1076                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1077                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1078
1079                 /* Wait for the Pipe State to go off */
1080                 if (intel_de_wait_for_clear(dev_priv, reg,
1081                                             I965_PIPECONF_ACTIVE, 100))
1082                         WARN(1, "pipe_off wait timed out\n");
1083         } else {
1084                 intel_wait_for_pipe_scanline_stopped(crtc);
1085         }
1086 }
1087
1088 /* Only for pre-ILK configs */
1089 void assert_pll(struct drm_i915_private *dev_priv,
1090                 enum pipe pipe, bool state)
1091 {
1092         u32 val;
1093         bool cur_state;
1094
1095         val = I915_READ(DPLL(pipe));
1096         cur_state = !!(val & DPLL_VCO_ENABLE);
1097         I915_STATE_WARN(cur_state != state,
1098              "PLL state assertion failure (expected %s, current %s)\n",
1099                         onoff(state), onoff(cur_state));
1100 }
1101
1102 /* XXX: the dsi pll is shared between MIPI DSI ports */
1103 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1104 {
1105         u32 val;
1106         bool cur_state;
1107
1108         vlv_cck_get(dev_priv);
1109         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1110         vlv_cck_put(dev_priv);
1111
1112         cur_state = val & DSI_PLL_VCO_EN;
1113         I915_STATE_WARN(cur_state != state,
1114              "DSI PLL state assertion failure (expected %s, current %s)\n",
1115                         onoff(state), onoff(cur_state));
1116 }
1117
1118 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1119                           enum pipe pipe, bool state)
1120 {
1121         bool cur_state;
1122         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1123                                                                       pipe);
1124
1125         if (HAS_DDI(dev_priv)) {
1126                 /* DDI does not have a specific FDI_TX register */
1127                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1128                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1129         } else {
1130                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1131                 cur_state = !!(val & FDI_TX_ENABLE);
1132         }
1133         I915_STATE_WARN(cur_state != state,
1134              "FDI TX state assertion failure (expected %s, current %s)\n",
1135                         onoff(state), onoff(cur_state));
1136 }
1137 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1138 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1139
1140 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1141                           enum pipe pipe, bool state)
1142 {
1143         u32 val;
1144         bool cur_state;
1145
1146         val = I915_READ(FDI_RX_CTL(pipe));
1147         cur_state = !!(val & FDI_RX_ENABLE);
1148         I915_STATE_WARN(cur_state != state,
1149              "FDI RX state assertion failure (expected %s, current %s)\n",
1150                         onoff(state), onoff(cur_state));
1151 }
1152 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1153 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1154
1155 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1156                                       enum pipe pipe)
1157 {
1158         u32 val;
1159
1160         /* ILK FDI PLL is always enabled */
1161         if (IS_GEN(dev_priv, 5))
1162                 return;
1163
1164         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1165         if (HAS_DDI(dev_priv))
1166                 return;
1167
1168         val = I915_READ(FDI_TX_CTL(pipe));
1169         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1170 }
1171
1172 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1173                        enum pipe pipe, bool state)
1174 {
1175         u32 val;
1176         bool cur_state;
1177
1178         val = I915_READ(FDI_RX_CTL(pipe));
1179         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1180         I915_STATE_WARN(cur_state != state,
1181              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1182                         onoff(state), onoff(cur_state));
1183 }
1184
1185 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1186 {
1187         i915_reg_t pp_reg;
1188         u32 val;
1189         enum pipe panel_pipe = INVALID_PIPE;
1190         bool locked = true;
1191
1192         if (WARN_ON(HAS_DDI(dev_priv)))
1193                 return;
1194
1195         if (HAS_PCH_SPLIT(dev_priv)) {
1196                 u32 port_sel;
1197
1198                 pp_reg = PP_CONTROL(0);
1199                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1200
1201                 switch (port_sel) {
1202                 case PANEL_PORT_SELECT_LVDS:
1203                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1204                         break;
1205                 case PANEL_PORT_SELECT_DPA:
1206                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1207                         break;
1208                 case PANEL_PORT_SELECT_DPC:
1209                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1210                         break;
1211                 case PANEL_PORT_SELECT_DPD:
1212                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1213                         break;
1214                 default:
1215                         MISSING_CASE(port_sel);
1216                         break;
1217                 }
1218         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1219                 /* presumably write lock depends on pipe, not port select */
1220                 pp_reg = PP_CONTROL(pipe);
1221                 panel_pipe = pipe;
1222         } else {
1223                 u32 port_sel;
1224
1225                 pp_reg = PP_CONTROL(0);
1226                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1227
1228                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1229                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1230         }
1231
1232         val = I915_READ(pp_reg);
1233         if (!(val & PANEL_POWER_ON) ||
1234             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1235                 locked = false;
1236
1237         I915_STATE_WARN(panel_pipe == pipe && locked,
1238              "panel assertion failure, pipe %c regs locked\n",
1239              pipe_name(pipe));
1240 }
1241
1242 void assert_pipe(struct drm_i915_private *dev_priv,
1243                  enum pipe pipe, bool state)
1244 {
1245         bool cur_state;
1246         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1247                                                                       pipe);
1248         enum intel_display_power_domain power_domain;
1249         intel_wakeref_t wakeref;
1250
1251         /* we keep both pipes enabled on 830 */
1252         if (IS_I830(dev_priv))
1253                 state = true;
1254
1255         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1256         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
1257         if (wakeref) {
1258                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1259                 cur_state = !!(val & PIPECONF_ENABLE);
1260
1261                 intel_display_power_put(dev_priv, power_domain, wakeref);
1262         } else {
1263                 cur_state = false;
1264         }
1265
1266         I915_STATE_WARN(cur_state != state,
1267              "pipe %c assertion failure (expected %s, current %s)\n",
1268                         pipe_name(pipe), onoff(state), onoff(cur_state));
1269 }
1270
1271 static void assert_plane(struct intel_plane *plane, bool state)
1272 {
1273         enum pipe pipe;
1274         bool cur_state;
1275
1276         cur_state = plane->get_hw_state(plane, &pipe);
1277
1278         I915_STATE_WARN(cur_state != state,
1279                         "%s assertion failure (expected %s, current %s)\n",
1280                         plane->base.name, onoff(state), onoff(cur_state));
1281 }
1282
1283 #define assert_plane_enabled(p) assert_plane(p, true)
1284 #define assert_plane_disabled(p) assert_plane(p, false)
1285
1286 static void assert_planes_disabled(struct intel_crtc *crtc)
1287 {
1288         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1289         struct intel_plane *plane;
1290
1291         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1292                 assert_plane_disabled(plane);
1293 }
1294
1295 static void assert_vblank_disabled(struct drm_crtc *crtc)
1296 {
1297         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1298                 drm_crtc_vblank_put(crtc);
1299 }
1300
1301 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1302                                     enum pipe pipe)
1303 {
1304         u32 val;
1305         bool enabled;
1306
1307         val = I915_READ(PCH_TRANSCONF(pipe));
1308         enabled = !!(val & TRANS_ENABLE);
1309         I915_STATE_WARN(enabled,
1310              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1311              pipe_name(pipe));
1312 }
1313
1314 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1315                                    enum pipe pipe, enum port port,
1316                                    i915_reg_t dp_reg)
1317 {
1318         enum pipe port_pipe;
1319         bool state;
1320
1321         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1322
1323         I915_STATE_WARN(state && port_pipe == pipe,
1324                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1325                         port_name(port), pipe_name(pipe));
1326
1327         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1328                         "IBX PCH DP %c still using transcoder B\n",
1329                         port_name(port));
1330 }
1331
1332 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1333                                      enum pipe pipe, enum port port,
1334                                      i915_reg_t hdmi_reg)
1335 {
1336         enum pipe port_pipe;
1337         bool state;
1338
1339         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1340
1341         I915_STATE_WARN(state && port_pipe == pipe,
1342                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1343                         port_name(port), pipe_name(pipe));
1344
1345         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1346                         "IBX PCH HDMI %c still using transcoder B\n",
1347                         port_name(port));
1348 }
1349
1350 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1351                                       enum pipe pipe)
1352 {
1353         enum pipe port_pipe;
1354
1355         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1356         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1357         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1358
1359         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1360                         port_pipe == pipe,
1361                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1362                         pipe_name(pipe));
1363
1364         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1365                         port_pipe == pipe,
1366                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1367                         pipe_name(pipe));
1368
1369         /* PCH SDVOB multiplex with HDMIB */
1370         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1371         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1372         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1373 }
1374
1375 static void _vlv_enable_pll(struct intel_crtc *crtc,
1376                             const struct intel_crtc_state *pipe_config)
1377 {
1378         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1379         enum pipe pipe = crtc->pipe;
1380
1381         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1382         POSTING_READ(DPLL(pipe));
1383         udelay(150);
1384
1385         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1386                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1387 }
1388
1389 static void vlv_enable_pll(struct intel_crtc *crtc,
1390                            const struct intel_crtc_state *pipe_config)
1391 {
1392         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1393         enum pipe pipe = crtc->pipe;
1394
1395         assert_pipe_disabled(dev_priv, pipe);
1396
1397         /* PLL is protected by panel, make sure we can write it */
1398         assert_panel_unlocked(dev_priv, pipe);
1399
1400         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1401                 _vlv_enable_pll(crtc, pipe_config);
1402
1403         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1404         POSTING_READ(DPLL_MD(pipe));
1405 }
1406
1407
1408 static void _chv_enable_pll(struct intel_crtc *crtc,
1409                             const struct intel_crtc_state *pipe_config)
1410 {
1411         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1412         enum pipe pipe = crtc->pipe;
1413         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1414         u32 tmp;
1415
1416         vlv_dpio_get(dev_priv);
1417
1418         /* Enable back the 10bit clock to display controller */
1419         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1420         tmp |= DPIO_DCLKP_EN;
1421         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1422
1423         vlv_dpio_put(dev_priv);
1424
1425         /*
1426          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1427          */
1428         udelay(1);
1429
1430         /* Enable PLL */
1431         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1432
1433         /* Check PLL is locked */
1434         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1435                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1436 }
1437
1438 static void chv_enable_pll(struct intel_crtc *crtc,
1439                            const struct intel_crtc_state *pipe_config)
1440 {
1441         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1442         enum pipe pipe = crtc->pipe;
1443
1444         assert_pipe_disabled(dev_priv, pipe);
1445
1446         /* PLL is protected by panel, make sure we can write it */
1447         assert_panel_unlocked(dev_priv, pipe);
1448
1449         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1450                 _chv_enable_pll(crtc, pipe_config);
1451
1452         if (pipe != PIPE_A) {
1453                 /*
1454                  * WaPixelRepeatModeFixForC0:chv
1455                  *
1456                  * DPLLCMD is AWOL. Use chicken bits to propagate
1457                  * the value from DPLLBMD to either pipe B or C.
1458                  */
1459                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1460                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1461                 I915_WRITE(CBR4_VLV, 0);
1462                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1463
1464                 /*
1465                  * DPLLB VGA mode also seems to cause problems.
1466                  * We should always have it disabled.
1467                  */
1468                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1469         } else {
1470                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1471                 POSTING_READ(DPLL_MD(pipe));
1472         }
1473 }
1474
1475 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
1476 {
1477         if (IS_I830(dev_priv))
1478                 return false;
1479
1480         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
1481 }
1482
1483 static void i9xx_enable_pll(struct intel_crtc *crtc,
1484                             const struct intel_crtc_state *crtc_state)
1485 {
1486         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1487         i915_reg_t reg = DPLL(crtc->pipe);
1488         u32 dpll = crtc_state->dpll_hw_state.dpll;
1489         int i;
1490
1491         assert_pipe_disabled(dev_priv, crtc->pipe);
1492
1493         /* PLL is protected by panel, make sure we can write it */
1494         if (i9xx_has_pps(dev_priv))
1495                 assert_panel_unlocked(dev_priv, crtc->pipe);
1496
1497         /*
1498          * Apparently we need to have VGA mode enabled prior to changing
1499          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1500          * dividers, even though the register value does change.
1501          */
1502         I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS);
1503         I915_WRITE(reg, dpll);
1504
1505         /* Wait for the clocks to stabilize. */
1506         POSTING_READ(reg);
1507         udelay(150);
1508
1509         if (INTEL_GEN(dev_priv) >= 4) {
1510                 I915_WRITE(DPLL_MD(crtc->pipe),
1511                            crtc_state->dpll_hw_state.dpll_md);
1512         } else {
1513                 /* The pixel multiplier can only be updated once the
1514                  * DPLL is enabled and the clocks are stable.
1515                  *
1516                  * So write it again.
1517                  */
1518                 I915_WRITE(reg, dpll);
1519         }
1520
1521         /* We do this three times for luck */
1522         for (i = 0; i < 3; i++) {
1523                 I915_WRITE(reg, dpll);
1524                 POSTING_READ(reg);
1525                 udelay(150); /* wait for warmup */
1526         }
1527 }
1528
1529 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1530 {
1531         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1532         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1533         enum pipe pipe = crtc->pipe;
1534
1535         /* Don't disable pipe or pipe PLLs if needed */
1536         if (IS_I830(dev_priv))
1537                 return;
1538
1539         /* Make sure the pipe isn't still relying on us */
1540         assert_pipe_disabled(dev_priv, pipe);
1541
1542         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1543         POSTING_READ(DPLL(pipe));
1544 }
1545
1546 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1547 {
1548         u32 val;
1549
1550         /* Make sure the pipe isn't still relying on us */
1551         assert_pipe_disabled(dev_priv, pipe);
1552
1553         val = DPLL_INTEGRATED_REF_CLK_VLV |
1554                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1555         if (pipe != PIPE_A)
1556                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1557
1558         I915_WRITE(DPLL(pipe), val);
1559         POSTING_READ(DPLL(pipe));
1560 }
1561
1562 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1563 {
1564         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1565         u32 val;
1566
1567         /* Make sure the pipe isn't still relying on us */
1568         assert_pipe_disabled(dev_priv, pipe);
1569
1570         val = DPLL_SSC_REF_CLK_CHV |
1571                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1572         if (pipe != PIPE_A)
1573                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1574
1575         I915_WRITE(DPLL(pipe), val);
1576         POSTING_READ(DPLL(pipe));
1577
1578         vlv_dpio_get(dev_priv);
1579
1580         /* Disable 10bit clock to display controller */
1581         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1582         val &= ~DPIO_DCLKP_EN;
1583         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1584
1585         vlv_dpio_put(dev_priv);
1586 }
1587
1588 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1589                          struct intel_digital_port *dport,
1590                          unsigned int expected_mask)
1591 {
1592         u32 port_mask;
1593         i915_reg_t dpll_reg;
1594
1595         switch (dport->base.port) {
1596         case PORT_B:
1597                 port_mask = DPLL_PORTB_READY_MASK;
1598                 dpll_reg = DPLL(0);
1599                 break;
1600         case PORT_C:
1601                 port_mask = DPLL_PORTC_READY_MASK;
1602                 dpll_reg = DPLL(0);
1603                 expected_mask <<= 4;
1604                 break;
1605         case PORT_D:
1606                 port_mask = DPLL_PORTD_READY_MASK;
1607                 dpll_reg = DPIO_PHY_STATUS;
1608                 break;
1609         default:
1610                 BUG();
1611         }
1612
1613         if (intel_de_wait_for_register(dev_priv, dpll_reg,
1614                                        port_mask, expected_mask, 1000))
1615                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1616                      port_name(dport->base.port),
1617                      I915_READ(dpll_reg) & port_mask, expected_mask);
1618 }
1619
1620 static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1621 {
1622         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1623         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1624         enum pipe pipe = crtc->pipe;
1625         i915_reg_t reg;
1626         u32 val, pipeconf_val;
1627
1628         /* Make sure PCH DPLL is enabled */
1629         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1630
1631         /* FDI must be feeding us bits for PCH ports */
1632         assert_fdi_tx_enabled(dev_priv, pipe);
1633         assert_fdi_rx_enabled(dev_priv, pipe);
1634
1635         if (HAS_PCH_CPT(dev_priv)) {
1636                 /* Workaround: Set the timing override bit before enabling the
1637                  * pch transcoder. */
1638                 reg = TRANS_CHICKEN2(pipe);
1639                 val = I915_READ(reg);
1640                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1641                 I915_WRITE(reg, val);
1642         }
1643
1644         reg = PCH_TRANSCONF(pipe);
1645         val = I915_READ(reg);
1646         pipeconf_val = I915_READ(PIPECONF(pipe));
1647
1648         if (HAS_PCH_IBX(dev_priv)) {
1649                 /*
1650                  * Make the BPC in transcoder be consistent with
1651                  * that in pipeconf reg. For HDMI we must use 8bpc
1652                  * here for both 8bpc and 12bpc.
1653                  */
1654                 val &= ~PIPECONF_BPC_MASK;
1655                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1656                         val |= PIPECONF_8BPC;
1657                 else
1658                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1659         }
1660
1661         val &= ~TRANS_INTERLACE_MASK;
1662         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
1663                 if (HAS_PCH_IBX(dev_priv) &&
1664                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1665                         val |= TRANS_LEGACY_INTERLACED_ILK;
1666                 else
1667                         val |= TRANS_INTERLACED;
1668         } else {
1669                 val |= TRANS_PROGRESSIVE;
1670         }
1671
1672         I915_WRITE(reg, val | TRANS_ENABLE);
1673         if (intel_de_wait_for_set(dev_priv, reg, TRANS_STATE_ENABLE, 100))
1674                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1675 }
1676
1677 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1678                                       enum transcoder cpu_transcoder)
1679 {
1680         u32 val, pipeconf_val;
1681
1682         /* FDI must be feeding us bits for PCH ports */
1683         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1684         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1685
1686         /* Workaround: set timing override bit. */
1687         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1688         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1689         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1690
1691         val = TRANS_ENABLE;
1692         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1693
1694         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1695             PIPECONF_INTERLACED_ILK)
1696                 val |= TRANS_INTERLACED;
1697         else
1698                 val |= TRANS_PROGRESSIVE;
1699
1700         I915_WRITE(LPT_TRANSCONF, val);
1701         if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF,
1702                                   TRANS_STATE_ENABLE, 100))
1703                 DRM_ERROR("Failed to enable PCH transcoder\n");
1704 }
1705
1706 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1707                                             enum pipe pipe)
1708 {
1709         i915_reg_t reg;
1710         u32 val;
1711
1712         /* FDI relies on the transcoder */
1713         assert_fdi_tx_disabled(dev_priv, pipe);
1714         assert_fdi_rx_disabled(dev_priv, pipe);
1715
1716         /* Ports must be off as well */
1717         assert_pch_ports_disabled(dev_priv, pipe);
1718
1719         reg = PCH_TRANSCONF(pipe);
1720         val = I915_READ(reg);
1721         val &= ~TRANS_ENABLE;
1722         I915_WRITE(reg, val);
1723         /* wait for PCH transcoder off, transcoder state */
1724         if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
1725                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1726
1727         if (HAS_PCH_CPT(dev_priv)) {
1728                 /* Workaround: Clear the timing override chicken bit again. */
1729                 reg = TRANS_CHICKEN2(pipe);
1730                 val = I915_READ(reg);
1731                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1732                 I915_WRITE(reg, val);
1733         }
1734 }
1735
1736 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1737 {
1738         u32 val;
1739
1740         val = I915_READ(LPT_TRANSCONF);
1741         val &= ~TRANS_ENABLE;
1742         I915_WRITE(LPT_TRANSCONF, val);
1743         /* wait for PCH transcoder off, transcoder state */
1744         if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
1745                                     TRANS_STATE_ENABLE, 50))
1746                 DRM_ERROR("Failed to disable PCH transcoder\n");
1747
1748         /* Workaround: clear timing override bit. */
1749         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1750         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1751         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1752 }
1753
1754 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1755 {
1756         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1757
1758         if (HAS_PCH_LPT(dev_priv))
1759                 return PIPE_A;
1760         else
1761                 return crtc->pipe;
1762 }
1763
1764 static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
1765 {
1766         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1767
1768         /*
1769          * On i965gm the hardware frame counter reads
1770          * zero when the TV encoder is enabled :(
1771          */
1772         if (IS_I965GM(dev_priv) &&
1773             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
1774                 return 0;
1775
1776         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1777                 return 0xffffffff; /* full 32 bit counter */
1778         else if (INTEL_GEN(dev_priv) >= 3)
1779                 return 0xffffff; /* only 24 bits of frame count */
1780         else
1781                 return 0; /* Gen2 doesn't have a hardware frame counter */
1782 }
1783
1784 static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
1785 {
1786         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1787
1788         drm_crtc_set_max_vblank_count(&crtc->base,
1789                                       intel_crtc_max_vblank_count(crtc_state));
1790         drm_crtc_vblank_on(&crtc->base);
1791 }
1792
1793 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1794 {
1795         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1796         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1797         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1798         enum pipe pipe = crtc->pipe;
1799         i915_reg_t reg;
1800         u32 val;
1801
1802         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1803
1804         assert_planes_disabled(crtc);
1805
1806         /*
1807          * A pipe without a PLL won't actually be able to drive bits from
1808          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1809          * need the check.
1810          */
1811         if (HAS_GMCH(dev_priv)) {
1812                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1813                         assert_dsi_pll_enabled(dev_priv);
1814                 else
1815                         assert_pll_enabled(dev_priv, pipe);
1816         } else {
1817                 if (new_crtc_state->has_pch_encoder) {
1818                         /* if driving the PCH, we need FDI enabled */
1819                         assert_fdi_rx_pll_enabled(dev_priv,
1820                                                   intel_crtc_pch_transcoder(crtc));
1821                         assert_fdi_tx_pll_enabled(dev_priv,
1822                                                   (enum pipe) cpu_transcoder);
1823                 }
1824                 /* FIXME: assert CPU port conditions for SNB+ */
1825         }
1826
1827         trace_intel_pipe_enable(crtc);
1828
1829         reg = PIPECONF(cpu_transcoder);
1830         val = I915_READ(reg);
1831         if (val & PIPECONF_ENABLE) {
1832                 /* we keep both pipes enabled on 830 */
1833                 WARN_ON(!IS_I830(dev_priv));
1834                 return;
1835         }
1836
1837         I915_WRITE(reg, val | PIPECONF_ENABLE);
1838         POSTING_READ(reg);
1839
1840         /*
1841          * Until the pipe starts PIPEDSL reads will return a stale value,
1842          * which causes an apparent vblank timestamp jump when PIPEDSL
1843          * resets to its proper value. That also messes up the frame count
1844          * when it's derived from the timestamps. So let's wait for the
1845          * pipe to start properly before we call drm_crtc_vblank_on()
1846          */
1847         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1848                 intel_wait_for_pipe_scanline_moving(crtc);
1849 }
1850
1851 static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1852 {
1853         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1854         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1855         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1856         enum pipe pipe = crtc->pipe;
1857         i915_reg_t reg;
1858         u32 val;
1859
1860         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1861
1862         /*
1863          * Make sure planes won't keep trying to pump pixels to us,
1864          * or we might hang the display.
1865          */
1866         assert_planes_disabled(crtc);
1867
1868         trace_intel_pipe_disable(crtc);
1869
1870         reg = PIPECONF(cpu_transcoder);
1871         val = I915_READ(reg);
1872         if ((val & PIPECONF_ENABLE) == 0)
1873                 return;
1874
1875         /*
1876          * Double wide has implications for planes
1877          * so best keep it disabled when not needed.
1878          */
1879         if (old_crtc_state->double_wide)
1880                 val &= ~PIPECONF_DOUBLE_WIDE;
1881
1882         /* Don't disable pipe or pipe PLLs if needed */
1883         if (!IS_I830(dev_priv))
1884                 val &= ~PIPECONF_ENABLE;
1885
1886         I915_WRITE(reg, val);
1887         if ((val & PIPECONF_ENABLE) == 0)
1888                 intel_wait_for_pipe_off(old_crtc_state);
1889 }
1890
1891 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1892 {
1893         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1894 }
1895
1896 static unsigned int
1897 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1898 {
1899         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1900         unsigned int cpp = fb->format->cpp[color_plane];
1901
1902         switch (fb->modifier) {
1903         case DRM_FORMAT_MOD_LINEAR:
1904                 return intel_tile_size(dev_priv);
1905         case I915_FORMAT_MOD_X_TILED:
1906                 if (IS_GEN(dev_priv, 2))
1907                         return 128;
1908                 else
1909                         return 512;
1910         case I915_FORMAT_MOD_Y_TILED_CCS:
1911                 if (color_plane == 1)
1912                         return 128;
1913                 /* fall through */
1914         case I915_FORMAT_MOD_Y_TILED:
1915                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
1916                         return 128;
1917                 else
1918                         return 512;
1919         case I915_FORMAT_MOD_Yf_TILED_CCS:
1920                 if (color_plane == 1)
1921                         return 128;
1922                 /* fall through */
1923         case I915_FORMAT_MOD_Yf_TILED:
1924                 switch (cpp) {
1925                 case 1:
1926                         return 64;
1927                 case 2:
1928                 case 4:
1929                         return 128;
1930                 case 8:
1931                 case 16:
1932                         return 256;
1933                 default:
1934                         MISSING_CASE(cpp);
1935                         return cpp;
1936                 }
1937                 break;
1938         default:
1939                 MISSING_CASE(fb->modifier);
1940                 return cpp;
1941         }
1942 }
1943
1944 static unsigned int
1945 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
1946 {
1947         return intel_tile_size(to_i915(fb->dev)) /
1948                 intel_tile_width_bytes(fb, color_plane);
1949 }
1950
1951 /* Return the tile dimensions in pixel units */
1952 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
1953                             unsigned int *tile_width,
1954                             unsigned int *tile_height)
1955 {
1956         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
1957         unsigned int cpp = fb->format->cpp[color_plane];
1958
1959         *tile_width = tile_width_bytes / cpp;
1960         *tile_height = intel_tile_size(to_i915(fb->dev)) / tile_width_bytes;
1961 }
1962
1963 unsigned int
1964 intel_fb_align_height(const struct drm_framebuffer *fb,
1965                       int color_plane, unsigned int height)
1966 {
1967         unsigned int tile_height = intel_tile_height(fb, color_plane);
1968
1969         return ALIGN(height, tile_height);
1970 }
1971
1972 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
1973 {
1974         unsigned int size = 0;
1975         int i;
1976
1977         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1978                 size += rot_info->plane[i].width * rot_info->plane[i].height;
1979
1980         return size;
1981 }
1982
1983 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
1984 {
1985         unsigned int size = 0;
1986         int i;
1987
1988         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
1989                 size += rem_info->plane[i].width * rem_info->plane[i].height;
1990
1991         return size;
1992 }
1993
1994 static void
1995 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
1996                         const struct drm_framebuffer *fb,
1997                         unsigned int rotation)
1998 {
1999         view->type = I915_GGTT_VIEW_NORMAL;
2000         if (drm_rotation_90_or_270(rotation)) {
2001                 view->type = I915_GGTT_VIEW_ROTATED;
2002                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2003         }
2004 }
2005
2006 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2007 {
2008         if (IS_I830(dev_priv))
2009                 return 16 * 1024;
2010         else if (IS_I85X(dev_priv))
2011                 return 256;
2012         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2013                 return 32;
2014         else
2015                 return 4 * 1024;
2016 }
2017
2018 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2019 {
2020         if (INTEL_GEN(dev_priv) >= 9)
2021                 return 256 * 1024;
2022         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2023                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2024                 return 128 * 1024;
2025         else if (INTEL_GEN(dev_priv) >= 4)
2026                 return 4 * 1024;
2027         else
2028                 return 0;
2029 }
2030
2031 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2032                                          int color_plane)
2033 {
2034         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2035
2036         /* AUX_DIST needs only 4K alignment */
2037         if (color_plane == 1)
2038                 return 4096;
2039
2040         switch (fb->modifier) {
2041         case DRM_FORMAT_MOD_LINEAR:
2042                 return intel_linear_alignment(dev_priv);
2043         case I915_FORMAT_MOD_X_TILED:
2044                 if (INTEL_GEN(dev_priv) >= 9)
2045                         return 256 * 1024;
2046                 return 0;
2047         case I915_FORMAT_MOD_Y_TILED_CCS:
2048         case I915_FORMAT_MOD_Yf_TILED_CCS:
2049         case I915_FORMAT_MOD_Y_TILED:
2050         case I915_FORMAT_MOD_Yf_TILED:
2051                 return 1 * 1024 * 1024;
2052         default:
2053                 MISSING_CASE(fb->modifier);
2054                 return 0;
2055         }
2056 }
2057
2058 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2059 {
2060         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2061         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2062
2063         return INTEL_GEN(dev_priv) < 4 ||
2064                 (plane->has_fbc &&
2065                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
2066 }
2067
2068 struct i915_vma *
2069 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2070                            const struct i915_ggtt_view *view,
2071                            bool uses_fence,
2072                            unsigned long *out_flags)
2073 {
2074         struct drm_device *dev = fb->dev;
2075         struct drm_i915_private *dev_priv = to_i915(dev);
2076         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2077         intel_wakeref_t wakeref;
2078         struct i915_vma *vma;
2079         unsigned int pinctl;
2080         u32 alignment;
2081
2082         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2083
2084         alignment = intel_surf_alignment(fb, 0);
2085
2086         /* Note that the w/a also requires 64 PTE of padding following the
2087          * bo. We currently fill all unused PTE with the shadow page and so
2088          * we should always have valid PTE following the scanout preventing
2089          * the VT-d warning.
2090          */
2091         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2092                 alignment = 256 * 1024;
2093
2094         /*
2095          * Global gtt pte registers are special registers which actually forward
2096          * writes to a chunk of system memory. Which means that there is no risk
2097          * that the register values disappear as soon as we call
2098          * intel_runtime_pm_put(), so it is correct to wrap only the
2099          * pin/unpin/fence and not more.
2100          */
2101         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2102         i915_gem_object_lock(obj);
2103
2104         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2105
2106         pinctl = 0;
2107
2108         /* Valleyview is definitely limited to scanning out the first
2109          * 512MiB. Lets presume this behaviour was inherited from the
2110          * g4x display engine and that all earlier gen are similarly
2111          * limited. Testing suggests that it is a little more
2112          * complicated than this. For example, Cherryview appears quite
2113          * happy to scanout from anywhere within its global aperture.
2114          */
2115         if (HAS_GMCH(dev_priv))
2116                 pinctl |= PIN_MAPPABLE;
2117
2118         vma = i915_gem_object_pin_to_display_plane(obj,
2119                                                    alignment, view, pinctl);
2120         if (IS_ERR(vma))
2121                 goto err;
2122
2123         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2124                 int ret;
2125
2126                 /* Install a fence for tiled scan-out. Pre-i965 always needs a
2127                  * fence, whereas 965+ only requires a fence if using
2128                  * framebuffer compression.  For simplicity, we always, when
2129                  * possible, install a fence as the cost is not that onerous.
2130                  *
2131                  * If we fail to fence the tiled scanout, then either the
2132                  * modeset will reject the change (which is highly unlikely as
2133                  * the affected systems, all but one, do not have unmappable
2134                  * space) or we will not be able to enable full powersaving
2135                  * techniques (also likely not to apply due to various limits
2136                  * FBC and the like impose on the size of the buffer, which
2137                  * presumably we violated anyway with this unmappable buffer).
2138                  * Anyway, it is presumably better to stumble onwards with
2139                  * something and try to run the system in a "less than optimal"
2140                  * mode that matches the user configuration.
2141                  */
2142                 ret = i915_vma_pin_fence(vma);
2143                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2144                         i915_gem_object_unpin_from_display_plane(vma);
2145                         vma = ERR_PTR(ret);
2146                         goto err;
2147                 }
2148
2149                 if (ret == 0 && vma->fence)
2150                         *out_flags |= PLANE_HAS_FENCE;
2151         }
2152
2153         i915_vma_get(vma);
2154 err:
2155         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2156
2157         i915_gem_object_unlock(obj);
2158         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2159         return vma;
2160 }
2161
2162 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2163 {
2164         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
2165
2166         i915_gem_object_lock(vma->obj);
2167         if (flags & PLANE_HAS_FENCE)
2168                 i915_vma_unpin_fence(vma);
2169         i915_gem_object_unpin_from_display_plane(vma);
2170         i915_gem_object_unlock(vma->obj);
2171
2172         i915_vma_put(vma);
2173 }
2174
2175 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2176                           unsigned int rotation)
2177 {
2178         if (drm_rotation_90_or_270(rotation))
2179                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2180         else
2181                 return fb->pitches[color_plane];
2182 }
2183
2184 /*
2185  * Convert the x/y offsets into a linear offset.
2186  * Only valid with 0/180 degree rotation, which is fine since linear
2187  * offset is only used with linear buffers on pre-hsw and tiled buffers
2188  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2189  */
2190 u32 intel_fb_xy_to_linear(int x, int y,
2191                           const struct intel_plane_state *state,
2192                           int color_plane)
2193 {
2194         const struct drm_framebuffer *fb = state->base.fb;
2195         unsigned int cpp = fb->format->cpp[color_plane];
2196         unsigned int pitch = state->color_plane[color_plane].stride;
2197
2198         return y * pitch + x * cpp;
2199 }
2200
2201 /*
2202  * Add the x/y offsets derived from fb->offsets[] to the user
2203  * specified plane src x/y offsets. The resulting x/y offsets
2204  * specify the start of scanout from the beginning of the gtt mapping.
2205  */
2206 void intel_add_fb_offsets(int *x, int *y,
2207                           const struct intel_plane_state *state,
2208                           int color_plane)
2209
2210 {
2211         *x += state->color_plane[color_plane].x;
2212         *y += state->color_plane[color_plane].y;
2213 }
2214
2215 static u32 intel_adjust_tile_offset(int *x, int *y,
2216                                     unsigned int tile_width,
2217                                     unsigned int tile_height,
2218                                     unsigned int tile_size,
2219                                     unsigned int pitch_tiles,
2220                                     u32 old_offset,
2221                                     u32 new_offset)
2222 {
2223         unsigned int pitch_pixels = pitch_tiles * tile_width;
2224         unsigned int tiles;
2225
2226         WARN_ON(old_offset & (tile_size - 1));
2227         WARN_ON(new_offset & (tile_size - 1));
2228         WARN_ON(new_offset > old_offset);
2229
2230         tiles = (old_offset - new_offset) / tile_size;
2231
2232         *y += tiles / pitch_tiles * tile_height;
2233         *x += tiles % pitch_tiles * tile_width;
2234
2235         /* minimize x in case it got needlessly big */
2236         *y += *x / pitch_pixels * tile_height;
2237         *x %= pitch_pixels;
2238
2239         return new_offset;
2240 }
2241
2242 static bool is_surface_linear(u64 modifier, int color_plane)
2243 {
2244         return modifier == DRM_FORMAT_MOD_LINEAR;
2245 }
2246
2247 static u32 intel_adjust_aligned_offset(int *x, int *y,
2248                                        const struct drm_framebuffer *fb,
2249                                        int color_plane,
2250                                        unsigned int rotation,
2251                                        unsigned int pitch,
2252                                        u32 old_offset, u32 new_offset)
2253 {
2254         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2255         unsigned int cpp = fb->format->cpp[color_plane];
2256
2257         WARN_ON(new_offset > old_offset);
2258
2259         if (!is_surface_linear(fb->modifier, color_plane)) {
2260                 unsigned int tile_size, tile_width, tile_height;
2261                 unsigned int pitch_tiles;
2262
2263                 tile_size = intel_tile_size(dev_priv);
2264                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2265
2266                 if (drm_rotation_90_or_270(rotation)) {
2267                         pitch_tiles = pitch / tile_height;
2268                         swap(tile_width, tile_height);
2269                 } else {
2270                         pitch_tiles = pitch / (tile_width * cpp);
2271                 }
2272
2273                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2274                                          tile_size, pitch_tiles,
2275                                          old_offset, new_offset);
2276         } else {
2277                 old_offset += *y * pitch + *x * cpp;
2278
2279                 *y = (old_offset - new_offset) / pitch;
2280                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2281         }
2282
2283         return new_offset;
2284 }
2285
2286 /*
2287  * Adjust the tile offset by moving the difference into
2288  * the x/y offsets.
2289  */
2290 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2291                                              const struct intel_plane_state *state,
2292                                              int color_plane,
2293                                              u32 old_offset, u32 new_offset)
2294 {
2295         return intel_adjust_aligned_offset(x, y, state->base.fb, color_plane,
2296                                            state->base.rotation,
2297                                            state->color_plane[color_plane].stride,
2298                                            old_offset, new_offset);
2299 }
2300
2301 /*
2302  * Computes the aligned offset to the base tile and adjusts
2303  * x, y. bytes per pixel is assumed to be a power-of-two.
2304  *
2305  * In the 90/270 rotated case, x and y are assumed
2306  * to be already rotated to match the rotated GTT view, and
2307  * pitch is the tile_height aligned framebuffer height.
2308  *
2309  * This function is used when computing the derived information
2310  * under intel_framebuffer, so using any of that information
2311  * here is not allowed. Anything under drm_framebuffer can be
2312  * used. This is why the user has to pass in the pitch since it
2313  * is specified in the rotated orientation.
2314  */
2315 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2316                                         int *x, int *y,
2317                                         const struct drm_framebuffer *fb,
2318                                         int color_plane,
2319                                         unsigned int pitch,
2320                                         unsigned int rotation,
2321                                         u32 alignment)
2322 {
2323         unsigned int cpp = fb->format->cpp[color_plane];
2324         u32 offset, offset_aligned;
2325
2326         if (alignment)
2327                 alignment--;
2328
2329         if (!is_surface_linear(fb->modifier, color_plane)) {
2330                 unsigned int tile_size, tile_width, tile_height;
2331                 unsigned int tile_rows, tiles, pitch_tiles;
2332
2333                 tile_size = intel_tile_size(dev_priv);
2334                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2335
2336                 if (drm_rotation_90_or_270(rotation)) {
2337                         pitch_tiles = pitch / tile_height;
2338                         swap(tile_width, tile_height);
2339                 } else {
2340                         pitch_tiles = pitch / (tile_width * cpp);
2341                 }
2342
2343                 tile_rows = *y / tile_height;
2344                 *y %= tile_height;
2345
2346                 tiles = *x / tile_width;
2347                 *x %= tile_width;
2348
2349                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2350                 offset_aligned = offset & ~alignment;
2351
2352                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2353                                          tile_size, pitch_tiles,
2354                                          offset, offset_aligned);
2355         } else {
2356                 offset = *y * pitch + *x * cpp;
2357                 offset_aligned = offset & ~alignment;
2358
2359                 *y = (offset & alignment) / pitch;
2360                 *x = ((offset & alignment) - *y * pitch) / cpp;
2361         }
2362
2363         return offset_aligned;
2364 }
2365
2366 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2367                                               const struct intel_plane_state *state,
2368                                               int color_plane)
2369 {
2370         struct intel_plane *intel_plane = to_intel_plane(state->base.plane);
2371         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2372         const struct drm_framebuffer *fb = state->base.fb;
2373         unsigned int rotation = state->base.rotation;
2374         int pitch = state->color_plane[color_plane].stride;
2375         u32 alignment;
2376
2377         if (intel_plane->id == PLANE_CURSOR)
2378                 alignment = intel_cursor_alignment(dev_priv);
2379         else
2380                 alignment = intel_surf_alignment(fb, color_plane);
2381
2382         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2383                                             pitch, rotation, alignment);
2384 }
2385
2386 /* Convert the fb->offset[] into x/y offsets */
2387 static int intel_fb_offset_to_xy(int *x, int *y,
2388                                  const struct drm_framebuffer *fb,
2389                                  int color_plane)
2390 {
2391         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2392         unsigned int height;
2393
2394         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2395             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2396                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2397                               fb->offsets[color_plane], color_plane);
2398                 return -EINVAL;
2399         }
2400
2401         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2402         height = ALIGN(height, intel_tile_height(fb, color_plane));
2403
2404         /* Catch potential overflows early */
2405         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2406                             fb->offsets[color_plane])) {
2407                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2408                               fb->offsets[color_plane], fb->pitches[color_plane],
2409                               color_plane);
2410                 return -ERANGE;
2411         }
2412
2413         *x = 0;
2414         *y = 0;
2415
2416         intel_adjust_aligned_offset(x, y,
2417                                     fb, color_plane, DRM_MODE_ROTATE_0,
2418                                     fb->pitches[color_plane],
2419                                     fb->offsets[color_plane], 0);
2420
2421         return 0;
2422 }
2423
2424 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
2425 {
2426         switch (fb_modifier) {
2427         case I915_FORMAT_MOD_X_TILED:
2428                 return I915_TILING_X;
2429         case I915_FORMAT_MOD_Y_TILED:
2430         case I915_FORMAT_MOD_Y_TILED_CCS:
2431                 return I915_TILING_Y;
2432         default:
2433                 return I915_TILING_NONE;
2434         }
2435 }
2436
2437 /*
2438  * From the Sky Lake PRM:
2439  * "The Color Control Surface (CCS) contains the compression status of
2440  *  the cache-line pairs. The compression state of the cache-line pair
2441  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2442  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2443  *  cache-line-pairs. CCS is always Y tiled."
2444  *
2445  * Since cache line pairs refers to horizontally adjacent cache lines,
2446  * each cache line in the CCS corresponds to an area of 32x16 cache
2447  * lines on the main surface. Since each pixel is 4 bytes, this gives
2448  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2449  * main surface.
2450  */
2451 static const struct drm_format_info ccs_formats[] = {
2452         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2453           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2454         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2455           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2456         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2457           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2458         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2459           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2460 };
2461
2462 static const struct drm_format_info *
2463 lookup_format_info(const struct drm_format_info formats[],
2464                    int num_formats, u32 format)
2465 {
2466         int i;
2467
2468         for (i = 0; i < num_formats; i++) {
2469                 if (formats[i].format == format)
2470                         return &formats[i];
2471         }
2472
2473         return NULL;
2474 }
2475
2476 static const struct drm_format_info *
2477 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2478 {
2479         switch (cmd->modifier[0]) {
2480         case I915_FORMAT_MOD_Y_TILED_CCS:
2481         case I915_FORMAT_MOD_Yf_TILED_CCS:
2482                 return lookup_format_info(ccs_formats,
2483                                           ARRAY_SIZE(ccs_formats),
2484                                           cmd->pixel_format);
2485         default:
2486                 return NULL;
2487         }
2488 }
2489
2490 bool is_ccs_modifier(u64 modifier)
2491 {
2492         return modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2493                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2494 }
2495
2496 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
2497                               u32 pixel_format, u64 modifier)
2498 {
2499         struct intel_crtc *crtc;
2500         struct intel_plane *plane;
2501
2502         /*
2503          * We assume the primary plane for pipe A has
2504          * the highest stride limits of them all.
2505          */
2506         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
2507         plane = to_intel_plane(crtc->base.primary);
2508
2509         return plane->max_stride(plane, pixel_format, modifier,
2510                                  DRM_MODE_ROTATE_0);
2511 }
2512
2513 static
2514 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
2515                         u32 pixel_format, u64 modifier)
2516 {
2517         /*
2518          * Arbitrary limit for gen4+ chosen to match the
2519          * render engine max stride.
2520          *
2521          * The new CCS hash mode makes remapping impossible
2522          */
2523         if (!is_ccs_modifier(modifier)) {
2524                 if (INTEL_GEN(dev_priv) >= 7)
2525                         return 256*1024;
2526                 else if (INTEL_GEN(dev_priv) >= 4)
2527                         return 128*1024;
2528         }
2529
2530         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
2531 }
2532
2533 static u32
2534 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
2535 {
2536         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2537
2538         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2539                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
2540                                                            fb->format->format,
2541                                                            fb->modifier);
2542
2543                 /*
2544                  * To make remapping with linear generally feasible
2545                  * we need the stride to be page aligned.
2546                  */
2547                 if (fb->pitches[color_plane] > max_stride)
2548                         return intel_tile_size(dev_priv);
2549                 else
2550                         return 64;
2551         } else {
2552                 return intel_tile_width_bytes(fb, color_plane);
2553         }
2554 }
2555
2556 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
2557 {
2558         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2559         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2560         const struct drm_framebuffer *fb = plane_state->base.fb;
2561         int i;
2562
2563         /* We don't want to deal with remapping with cursors */
2564         if (plane->id == PLANE_CURSOR)
2565                 return false;
2566
2567         /*
2568          * The display engine limits already match/exceed the
2569          * render engine limits, so not much point in remapping.
2570          * Would also need to deal with the fence POT alignment
2571          * and gen2 2KiB GTT tile size.
2572          */
2573         if (INTEL_GEN(dev_priv) < 4)
2574                 return false;
2575
2576         /*
2577          * The new CCS hash mode isn't compatible with remapping as
2578          * the virtual address of the pages affects the compressed data.
2579          */
2580         if (is_ccs_modifier(fb->modifier))
2581                 return false;
2582
2583         /* Linear needs a page aligned stride for remapping */
2584         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2585                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2586
2587                 for (i = 0; i < fb->format->num_planes; i++) {
2588                         if (fb->pitches[i] & alignment)
2589                                 return false;
2590                 }
2591         }
2592
2593         return true;
2594 }
2595
2596 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2597 {
2598         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2599         const struct drm_framebuffer *fb = plane_state->base.fb;
2600         unsigned int rotation = plane_state->base.rotation;
2601         u32 stride, max_stride;
2602
2603         /*
2604          * No remapping for invisible planes since we don't have
2605          * an actual source viewport to remap.
2606          */
2607         if (!plane_state->base.visible)
2608                 return false;
2609
2610         if (!intel_plane_can_remap(plane_state))
2611                 return false;
2612
2613         /*
2614          * FIXME: aux plane limits on gen9+ are
2615          * unclear in Bspec, for now no checking.
2616          */
2617         stride = intel_fb_pitch(fb, 0, rotation);
2618         max_stride = plane->max_stride(plane, fb->format->format,
2619                                        fb->modifier, rotation);
2620
2621         return stride > max_stride;
2622 }
2623
2624 static int
2625 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2626                    struct drm_framebuffer *fb)
2627 {
2628         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2629         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2630         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2631         u32 gtt_offset_rotated = 0;
2632         unsigned int max_size = 0;
2633         int i, num_planes = fb->format->num_planes;
2634         unsigned int tile_size = intel_tile_size(dev_priv);
2635
2636         for (i = 0; i < num_planes; i++) {
2637                 unsigned int width, height;
2638                 unsigned int cpp, size;
2639                 u32 offset;
2640                 int x, y;
2641                 int ret;
2642
2643                 cpp = fb->format->cpp[i];
2644                 width = drm_framebuffer_plane_width(fb->width, fb, i);
2645                 height = drm_framebuffer_plane_height(fb->height, fb, i);
2646
2647                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2648                 if (ret) {
2649                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2650                                       i, fb->offsets[i]);
2651                         return ret;
2652                 }
2653
2654                 if (is_ccs_modifier(fb->modifier) && i == 1) {
2655                         int hsub = fb->format->hsub;
2656                         int vsub = fb->format->vsub;
2657                         int tile_width, tile_height;
2658                         int main_x, main_y;
2659                         int ccs_x, ccs_y;
2660
2661                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2662                         tile_width *= hsub;
2663                         tile_height *= vsub;
2664
2665                         ccs_x = (x * hsub) % tile_width;
2666                         ccs_y = (y * vsub) % tile_height;
2667                         main_x = intel_fb->normal[0].x % tile_width;
2668                         main_y = intel_fb->normal[0].y % tile_height;
2669
2670                         /*
2671                          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2672                          * x/y offsets must match between CCS and the main surface.
2673                          */
2674                         if (main_x != ccs_x || main_y != ccs_y) {
2675                                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2676                                               main_x, main_y,
2677                                               ccs_x, ccs_y,
2678                                               intel_fb->normal[0].x,
2679                                               intel_fb->normal[0].y,
2680                                               x, y);
2681                                 return -EINVAL;
2682                         }
2683                 }
2684
2685                 /*
2686                  * The fence (if used) is aligned to the start of the object
2687                  * so having the framebuffer wrap around across the edge of the
2688                  * fenced region doesn't really work. We have no API to configure
2689                  * the fence start offset within the object (nor could we probably
2690                  * on gen2/3). So it's just easier if we just require that the
2691                  * fb layout agrees with the fence layout. We already check that the
2692                  * fb stride matches the fence stride elsewhere.
2693                  */
2694                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2695                     (x + width) * cpp > fb->pitches[i]) {
2696                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2697                                       i, fb->offsets[i]);
2698                         return -EINVAL;
2699                 }
2700
2701                 /*
2702                  * First pixel of the framebuffer from
2703                  * the start of the normal gtt mapping.
2704                  */
2705                 intel_fb->normal[i].x = x;
2706                 intel_fb->normal[i].y = y;
2707
2708                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
2709                                                       fb->pitches[i],
2710                                                       DRM_MODE_ROTATE_0,
2711                                                       tile_size);
2712                 offset /= tile_size;
2713
2714                 if (!is_surface_linear(fb->modifier, i)) {
2715                         unsigned int tile_width, tile_height;
2716                         unsigned int pitch_tiles;
2717                         struct drm_rect r;
2718
2719                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2720
2721                         rot_info->plane[i].offset = offset;
2722                         rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
2723                         rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2724                         rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2725
2726                         intel_fb->rotated[i].pitch =
2727                                 rot_info->plane[i].height * tile_height;
2728
2729                         /* how many tiles does this plane need */
2730                         size = rot_info->plane[i].stride * rot_info->plane[i].height;
2731                         /*
2732                          * If the plane isn't horizontally tile aligned,
2733                          * we need one more tile.
2734                          */
2735                         if (x != 0)
2736                                 size++;
2737
2738                         /* rotate the x/y offsets to match the GTT view */
2739                         r.x1 = x;
2740                         r.y1 = y;
2741                         r.x2 = x + width;
2742                         r.y2 = y + height;
2743                         drm_rect_rotate(&r,
2744                                         rot_info->plane[i].width * tile_width,
2745                                         rot_info->plane[i].height * tile_height,
2746                                         DRM_MODE_ROTATE_270);
2747                         x = r.x1;
2748                         y = r.y1;
2749
2750                         /* rotate the tile dimensions to match the GTT view */
2751                         pitch_tiles = intel_fb->rotated[i].pitch / tile_height;
2752                         swap(tile_width, tile_height);
2753
2754                         /*
2755                          * We only keep the x/y offsets, so push all of the
2756                          * gtt offset into the x/y offsets.
2757                          */
2758                         intel_adjust_tile_offset(&x, &y,
2759                                                  tile_width, tile_height,
2760                                                  tile_size, pitch_tiles,
2761                                                  gtt_offset_rotated * tile_size, 0);
2762
2763                         gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
2764
2765                         /*
2766                          * First pixel of the framebuffer from
2767                          * the start of the rotated gtt mapping.
2768                          */
2769                         intel_fb->rotated[i].x = x;
2770                         intel_fb->rotated[i].y = y;
2771                 } else {
2772                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
2773                                             x * cpp, tile_size);
2774                 }
2775
2776                 /* how many tiles in total needed in the bo */
2777                 max_size = max(max_size, offset + size);
2778         }
2779
2780         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
2781                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
2782                               mul_u32_u32(max_size, tile_size), obj->base.size);
2783                 return -EINVAL;
2784         }
2785
2786         return 0;
2787 }
2788
2789 static void
2790 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
2791 {
2792         struct drm_i915_private *dev_priv =
2793                 to_i915(plane_state->base.plane->dev);
2794         struct drm_framebuffer *fb = plane_state->base.fb;
2795         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2796         struct intel_rotation_info *info = &plane_state->view.rotated;
2797         unsigned int rotation = plane_state->base.rotation;
2798         int i, num_planes = fb->format->num_planes;
2799         unsigned int tile_size = intel_tile_size(dev_priv);
2800         unsigned int src_x, src_y;
2801         unsigned int src_w, src_h;
2802         u32 gtt_offset = 0;
2803
2804         memset(&plane_state->view, 0, sizeof(plane_state->view));
2805         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
2806                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
2807
2808         src_x = plane_state->base.src.x1 >> 16;
2809         src_y = plane_state->base.src.y1 >> 16;
2810         src_w = drm_rect_width(&plane_state->base.src) >> 16;
2811         src_h = drm_rect_height(&plane_state->base.src) >> 16;
2812
2813         WARN_ON(is_ccs_modifier(fb->modifier));
2814
2815         /* Make src coordinates relative to the viewport */
2816         drm_rect_translate(&plane_state->base.src,
2817                            -(src_x << 16), -(src_y << 16));
2818
2819         /* Rotate src coordinates to match rotated GTT view */
2820         if (drm_rotation_90_or_270(rotation))
2821                 drm_rect_rotate(&plane_state->base.src,
2822                                 src_w << 16, src_h << 16,
2823                                 DRM_MODE_ROTATE_270);
2824
2825         for (i = 0; i < num_planes; i++) {
2826                 unsigned int hsub = i ? fb->format->hsub : 1;
2827                 unsigned int vsub = i ? fb->format->vsub : 1;
2828                 unsigned int cpp = fb->format->cpp[i];
2829                 unsigned int tile_width, tile_height;
2830                 unsigned int width, height;
2831                 unsigned int pitch_tiles;
2832                 unsigned int x, y;
2833                 u32 offset;
2834
2835                 intel_tile_dims(fb, i, &tile_width, &tile_height);
2836
2837                 x = src_x / hsub;
2838                 y = src_y / vsub;
2839                 width = src_w / hsub;
2840                 height = src_h / vsub;
2841
2842                 /*
2843                  * First pixel of the src viewport from the
2844                  * start of the normal gtt mapping.
2845                  */
2846                 x += intel_fb->normal[i].x;
2847                 y += intel_fb->normal[i].y;
2848
2849                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
2850                                                       fb, i, fb->pitches[i],
2851                                                       DRM_MODE_ROTATE_0, tile_size);
2852                 offset /= tile_size;
2853
2854                 info->plane[i].offset = offset;
2855                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
2856                                                      tile_width * cpp);
2857                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2858                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2859
2860                 if (drm_rotation_90_or_270(rotation)) {
2861                         struct drm_rect r;
2862
2863                         /* rotate the x/y offsets to match the GTT view */
2864                         r.x1 = x;
2865                         r.y1 = y;
2866                         r.x2 = x + width;
2867                         r.y2 = y + height;
2868                         drm_rect_rotate(&r,
2869                                         info->plane[i].width * tile_width,
2870                                         info->plane[i].height * tile_height,
2871                                         DRM_MODE_ROTATE_270);
2872                         x = r.x1;
2873                         y = r.y1;
2874
2875                         pitch_tiles = info->plane[i].height;
2876                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
2877
2878                         /* rotate the tile dimensions to match the GTT view */
2879                         swap(tile_width, tile_height);
2880                 } else {
2881                         pitch_tiles = info->plane[i].width;
2882                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
2883                 }
2884
2885                 /*
2886                  * We only keep the x/y offsets, so push all of the
2887                  * gtt offset into the x/y offsets.
2888                  */
2889                 intel_adjust_tile_offset(&x, &y,
2890                                          tile_width, tile_height,
2891                                          tile_size, pitch_tiles,
2892                                          gtt_offset * tile_size, 0);
2893
2894                 gtt_offset += info->plane[i].width * info->plane[i].height;
2895
2896                 plane_state->color_plane[i].offset = 0;
2897                 plane_state->color_plane[i].x = x;
2898                 plane_state->color_plane[i].y = y;
2899         }
2900 }
2901
2902 static int
2903 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
2904 {
2905         const struct intel_framebuffer *fb =
2906                 to_intel_framebuffer(plane_state->base.fb);
2907         unsigned int rotation = plane_state->base.rotation;
2908         int i, num_planes;
2909
2910         if (!fb)
2911                 return 0;
2912
2913         num_planes = fb->base.format->num_planes;
2914
2915         if (intel_plane_needs_remap(plane_state)) {
2916                 intel_plane_remap_gtt(plane_state);
2917
2918                 /*
2919                  * Sometimes even remapping can't overcome
2920                  * the stride limitations :( Can happen with
2921                  * big plane sizes and suitably misaligned
2922                  * offsets.
2923                  */
2924                 return intel_plane_check_stride(plane_state);
2925         }
2926
2927         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
2928
2929         for (i = 0; i < num_planes; i++) {
2930                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
2931                 plane_state->color_plane[i].offset = 0;
2932
2933                 if (drm_rotation_90_or_270(rotation)) {
2934                         plane_state->color_plane[i].x = fb->rotated[i].x;
2935                         plane_state->color_plane[i].y = fb->rotated[i].y;
2936                 } else {
2937                         plane_state->color_plane[i].x = fb->normal[i].x;
2938                         plane_state->color_plane[i].y = fb->normal[i].y;
2939                 }
2940         }
2941
2942         /* Rotate src coordinates to match rotated GTT view */
2943         if (drm_rotation_90_or_270(rotation))
2944                 drm_rect_rotate(&plane_state->base.src,
2945                                 fb->base.width << 16, fb->base.height << 16,
2946                                 DRM_MODE_ROTATE_270);
2947
2948         return intel_plane_check_stride(plane_state);
2949 }
2950
2951 static int i9xx_format_to_fourcc(int format)
2952 {
2953         switch (format) {
2954         case DISPPLANE_8BPP:
2955                 return DRM_FORMAT_C8;
2956         case DISPPLANE_BGRX555:
2957                 return DRM_FORMAT_XRGB1555;
2958         case DISPPLANE_BGRX565:
2959                 return DRM_FORMAT_RGB565;
2960         default:
2961         case DISPPLANE_BGRX888:
2962                 return DRM_FORMAT_XRGB8888;
2963         case DISPPLANE_RGBX888:
2964                 return DRM_FORMAT_XBGR8888;
2965         case DISPPLANE_BGRX101010:
2966                 return DRM_FORMAT_XRGB2101010;
2967         case DISPPLANE_RGBX101010:
2968                 return DRM_FORMAT_XBGR2101010;
2969         }
2970 }
2971
2972 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2973 {
2974         switch (format) {
2975         case PLANE_CTL_FORMAT_RGB_565:
2976                 return DRM_FORMAT_RGB565;
2977         case PLANE_CTL_FORMAT_NV12:
2978                 return DRM_FORMAT_NV12;
2979         case PLANE_CTL_FORMAT_P010:
2980                 return DRM_FORMAT_P010;
2981         case PLANE_CTL_FORMAT_P012:
2982                 return DRM_FORMAT_P012;
2983         case PLANE_CTL_FORMAT_P016:
2984                 return DRM_FORMAT_P016;
2985         case PLANE_CTL_FORMAT_Y210:
2986                 return DRM_FORMAT_Y210;
2987         case PLANE_CTL_FORMAT_Y212:
2988                 return DRM_FORMAT_Y212;
2989         case PLANE_CTL_FORMAT_Y216:
2990                 return DRM_FORMAT_Y216;
2991         case PLANE_CTL_FORMAT_Y410:
2992                 return DRM_FORMAT_XVYU2101010;
2993         case PLANE_CTL_FORMAT_Y412:
2994                 return DRM_FORMAT_XVYU12_16161616;
2995         case PLANE_CTL_FORMAT_Y416:
2996                 return DRM_FORMAT_XVYU16161616;
2997         default:
2998         case PLANE_CTL_FORMAT_XRGB_8888:
2999                 if (rgb_order) {
3000                         if (alpha)
3001                                 return DRM_FORMAT_ABGR8888;
3002                         else
3003                                 return DRM_FORMAT_XBGR8888;
3004                 } else {
3005                         if (alpha)
3006                                 return DRM_FORMAT_ARGB8888;
3007                         else
3008                                 return DRM_FORMAT_XRGB8888;
3009                 }
3010         case PLANE_CTL_FORMAT_XRGB_2101010:
3011                 if (rgb_order)
3012                         return DRM_FORMAT_XBGR2101010;
3013                 else
3014                         return DRM_FORMAT_XRGB2101010;
3015         case PLANE_CTL_FORMAT_XRGB_16161616F:
3016                 if (rgb_order) {
3017                         if (alpha)
3018                                 return DRM_FORMAT_ABGR16161616F;
3019                         else
3020                                 return DRM_FORMAT_XBGR16161616F;
3021                 } else {
3022                         if (alpha)
3023                                 return DRM_FORMAT_ARGB16161616F;
3024                         else
3025                                 return DRM_FORMAT_XRGB16161616F;
3026                 }
3027         }
3028 }
3029
3030 static bool
3031 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
3032                               struct intel_initial_plane_config *plane_config)
3033 {
3034         struct drm_device *dev = crtc->base.dev;
3035         struct drm_i915_private *dev_priv = to_i915(dev);
3036         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
3037         struct drm_framebuffer *fb = &plane_config->fb->base;
3038         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
3039         u32 size_aligned = round_up(plane_config->base + plane_config->size,
3040                                     PAGE_SIZE);
3041         struct drm_i915_gem_object *obj;
3042         bool ret = false;
3043
3044         size_aligned -= base_aligned;
3045
3046         if (plane_config->size == 0)
3047                 return false;
3048
3049         /* If the FB is too big, just don't use it since fbdev is not very
3050          * important and we should probably use that space with FBC or other
3051          * features. */
3052         if (size_aligned * 2 > dev_priv->stolen_usable_size)
3053                 return false;
3054
3055         switch (fb->modifier) {
3056         case DRM_FORMAT_MOD_LINEAR:
3057         case I915_FORMAT_MOD_X_TILED:
3058         case I915_FORMAT_MOD_Y_TILED:
3059                 break;
3060         default:
3061                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
3062                                  fb->modifier);
3063                 return false;
3064         }
3065
3066         mutex_lock(&dev->struct_mutex);
3067         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
3068                                                              base_aligned,
3069                                                              base_aligned,
3070                                                              size_aligned);
3071         mutex_unlock(&dev->struct_mutex);
3072         if (!obj)
3073                 return false;
3074
3075         switch (plane_config->tiling) {
3076         case I915_TILING_NONE:
3077                 break;
3078         case I915_TILING_X:
3079         case I915_TILING_Y:
3080                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
3081                 break;
3082         default:
3083                 MISSING_CASE(plane_config->tiling);
3084                 goto out;
3085         }
3086
3087         mode_cmd.pixel_format = fb->format->format;
3088         mode_cmd.width = fb->width;
3089         mode_cmd.height = fb->height;
3090         mode_cmd.pitches[0] = fb->pitches[0];
3091         mode_cmd.modifier[0] = fb->modifier;
3092         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
3093
3094         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
3095                 DRM_DEBUG_KMS("intel fb init failed\n");
3096                 goto out;
3097         }
3098
3099
3100         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
3101         ret = true;
3102 out:
3103         i915_gem_object_put(obj);
3104         return ret;
3105 }
3106
3107 static void
3108 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
3109                         struct intel_plane_state *plane_state,
3110                         bool visible)
3111 {
3112         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
3113
3114         plane_state->base.visible = visible;
3115
3116         if (visible)
3117                 crtc_state->base.plane_mask |= drm_plane_mask(&plane->base);
3118         else
3119                 crtc_state->base.plane_mask &= ~drm_plane_mask(&plane->base);
3120 }
3121
3122 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
3123 {
3124         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
3125         struct drm_plane *plane;
3126
3127         /*
3128          * Active_planes aliases if multiple "primary" or cursor planes
3129          * have been used on the same (or wrong) pipe. plane_mask uses
3130          * unique ids, hence we can use that to reconstruct active_planes.
3131          */
3132         crtc_state->active_planes = 0;
3133
3134         drm_for_each_plane_mask(plane, &dev_priv->drm,
3135                                 crtc_state->base.plane_mask)
3136                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
3137 }
3138
3139 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
3140                                          struct intel_plane *plane)
3141 {
3142         struct intel_crtc_state *crtc_state =
3143                 to_intel_crtc_state(crtc->base.state);
3144         struct intel_plane_state *plane_state =
3145                 to_intel_plane_state(plane->base.state);
3146
3147         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
3148                       plane->base.base.id, plane->base.name,
3149                       crtc->base.base.id, crtc->base.name);
3150
3151         intel_set_plane_visible(crtc_state, plane_state, false);
3152         fixup_active_planes(crtc_state);
3153         crtc_state->data_rate[plane->id] = 0;
3154
3155         if (plane->id == PLANE_PRIMARY)
3156                 intel_pre_disable_primary_noatomic(&crtc->base);
3157
3158         intel_disable_plane(plane, crtc_state);
3159 }
3160
3161 static struct intel_frontbuffer *
3162 to_intel_frontbuffer(struct drm_framebuffer *fb)
3163 {
3164         return fb ? to_intel_framebuffer(fb)->frontbuffer : NULL;
3165 }
3166
3167 static void
3168 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
3169                              struct intel_initial_plane_config *plane_config)
3170 {
3171         struct drm_device *dev = intel_crtc->base.dev;
3172         struct drm_i915_private *dev_priv = to_i915(dev);
3173         struct drm_crtc *c;
3174         struct drm_plane *primary = intel_crtc->base.primary;
3175         struct drm_plane_state *plane_state = primary->state;
3176         struct intel_plane *intel_plane = to_intel_plane(primary);
3177         struct intel_plane_state *intel_state =
3178                 to_intel_plane_state(plane_state);
3179         struct drm_framebuffer *fb;
3180
3181         if (!plane_config->fb)
3182                 return;
3183
3184         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
3185                 fb = &plane_config->fb->base;
3186                 goto valid_fb;
3187         }
3188
3189         kfree(plane_config->fb);
3190
3191         /*
3192          * Failed to alloc the obj, check to see if we should share
3193          * an fb with another CRTC instead
3194          */
3195         for_each_crtc(dev, c) {
3196                 struct intel_plane_state *state;
3197
3198                 if (c == &intel_crtc->base)
3199                         continue;
3200
3201                 if (!to_intel_crtc(c)->active)
3202                         continue;
3203
3204                 state = to_intel_plane_state(c->primary->state);
3205                 if (!state->vma)
3206                         continue;
3207
3208                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
3209                         fb = state->base.fb;
3210                         drm_framebuffer_get(fb);
3211                         goto valid_fb;
3212                 }
3213         }
3214
3215         /*
3216          * We've failed to reconstruct the BIOS FB.  Current display state
3217          * indicates that the primary plane is visible, but has a NULL FB,
3218          * which will lead to problems later if we don't fix it up.  The
3219          * simplest solution is to just disable the primary plane now and
3220          * pretend the BIOS never had it enabled.
3221          */
3222         intel_plane_disable_noatomic(intel_crtc, intel_plane);
3223
3224         return;
3225
3226 valid_fb:
3227         intel_state->base.rotation = plane_config->rotation;
3228         intel_fill_fb_ggtt_view(&intel_state->view, fb,
3229                                 intel_state->base.rotation);
3230         intel_state->color_plane[0].stride =
3231                 intel_fb_pitch(fb, 0, intel_state->base.rotation);
3232
3233         mutex_lock(&dev->struct_mutex);
3234         intel_state->vma =
3235                 intel_pin_and_fence_fb_obj(fb,
3236                                            &intel_state->view,
3237                                            intel_plane_uses_fence(intel_state),
3238                                            &intel_state->flags);
3239         mutex_unlock(&dev->struct_mutex);
3240         if (IS_ERR(intel_state->vma)) {
3241                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
3242                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
3243
3244                 intel_state->vma = NULL;
3245                 drm_framebuffer_put(fb);
3246                 return;
3247         }
3248
3249         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
3250
3251         plane_state->src_x = 0;
3252         plane_state->src_y = 0;
3253         plane_state->src_w = fb->width << 16;
3254         plane_state->src_h = fb->height << 16;
3255
3256         plane_state->crtc_x = 0;
3257         plane_state->crtc_y = 0;
3258         plane_state->crtc_w = fb->width;
3259         plane_state->crtc_h = fb->height;
3260
3261         intel_state->base.src = drm_plane_state_src(plane_state);
3262         intel_state->base.dst = drm_plane_state_dest(plane_state);
3263
3264         if (plane_config->tiling)
3265                 dev_priv->preserve_bios_swizzle = true;
3266
3267         plane_state->fb = fb;
3268         plane_state->crtc = &intel_crtc->base;
3269
3270         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
3271                   &to_intel_frontbuffer(fb)->bits);
3272 }
3273
3274 static int skl_max_plane_width(const struct drm_framebuffer *fb,
3275                                int color_plane,
3276                                unsigned int rotation)
3277 {
3278         int cpp = fb->format->cpp[color_plane];
3279
3280         switch (fb->modifier) {
3281         case DRM_FORMAT_MOD_LINEAR:
3282         case I915_FORMAT_MOD_X_TILED:
3283                 /*
3284                  * Validated limit is 4k, but has 5k should
3285                  * work apart from the following features:
3286                  * - Ytile (already limited to 4k)
3287                  * - FP16 (already limited to 4k)
3288                  * - render compression (already limited to 4k)
3289                  * - KVMR sprite and cursor (don't care)
3290                  * - horizontal panning (TODO verify this)
3291                  * - pipe and plane scaling (TODO verify this)
3292                  */
3293                 if (cpp == 8)
3294                         return 4096;
3295                 else
3296                         return 5120;
3297         case I915_FORMAT_MOD_Y_TILED_CCS:
3298         case I915_FORMAT_MOD_Yf_TILED_CCS:
3299                 /* FIXME AUX plane? */
3300         case I915_FORMAT_MOD_Y_TILED:
3301         case I915_FORMAT_MOD_Yf_TILED:
3302                 if (cpp == 8)
3303                         return 2048;
3304                 else
3305                         return 4096;
3306         default:
3307                 MISSING_CASE(fb->modifier);
3308                 return 2048;
3309         }
3310 }
3311
3312 static int glk_max_plane_width(const struct drm_framebuffer *fb,
3313                                int color_plane,
3314                                unsigned int rotation)
3315 {
3316         int cpp = fb->format->cpp[color_plane];
3317
3318         switch (fb->modifier) {
3319         case DRM_FORMAT_MOD_LINEAR:
3320         case I915_FORMAT_MOD_X_TILED:
3321                 if (cpp == 8)
3322                         return 4096;
3323                 else
3324                         return 5120;
3325         case I915_FORMAT_MOD_Y_TILED_CCS:
3326         case I915_FORMAT_MOD_Yf_TILED_CCS:
3327                 /* FIXME AUX plane? */
3328         case I915_FORMAT_MOD_Y_TILED:
3329         case I915_FORMAT_MOD_Yf_TILED:
3330                 if (cpp == 8)
3331                         return 2048;
3332                 else
3333                         return 5120;
3334         default:
3335                 MISSING_CASE(fb->modifier);
3336                 return 2048;
3337         }
3338 }
3339
3340 static int icl_max_plane_width(const struct drm_framebuffer *fb,
3341                                int color_plane,
3342                                unsigned int rotation)
3343 {
3344         return 5120;
3345 }
3346
3347 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
3348                                            int main_x, int main_y, u32 main_offset)
3349 {
3350         const struct drm_framebuffer *fb = plane_state->base.fb;
3351         int hsub = fb->format->hsub;
3352         int vsub = fb->format->vsub;
3353         int aux_x = plane_state->color_plane[1].x;
3354         int aux_y = plane_state->color_plane[1].y;
3355         u32 aux_offset = plane_state->color_plane[1].offset;
3356         u32 alignment = intel_surf_alignment(fb, 1);
3357
3358         while (aux_offset >= main_offset && aux_y <= main_y) {
3359                 int x, y;
3360
3361                 if (aux_x == main_x && aux_y == main_y)
3362                         break;
3363
3364                 if (aux_offset == 0)
3365                         break;
3366
3367                 x = aux_x / hsub;
3368                 y = aux_y / vsub;
3369                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 1,
3370                                                                aux_offset, aux_offset - alignment);
3371                 aux_x = x * hsub + aux_x % hsub;
3372                 aux_y = y * vsub + aux_y % vsub;
3373         }
3374
3375         if (aux_x != main_x || aux_y != main_y)
3376                 return false;
3377
3378         plane_state->color_plane[1].offset = aux_offset;
3379         plane_state->color_plane[1].x = aux_x;
3380         plane_state->color_plane[1].y = aux_y;
3381
3382         return true;
3383 }
3384
3385 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3386 {
3387         struct drm_i915_private *dev_priv = to_i915(plane_state->base.plane->dev);
3388         const struct drm_framebuffer *fb = plane_state->base.fb;
3389         unsigned int rotation = plane_state->base.rotation;
3390         int x = plane_state->base.src.x1 >> 16;
3391         int y = plane_state->base.src.y1 >> 16;
3392         int w = drm_rect_width(&plane_state->base.src) >> 16;
3393         int h = drm_rect_height(&plane_state->base.src) >> 16;
3394         int max_width;
3395         int max_height = 4096;
3396         u32 alignment, offset, aux_offset = plane_state->color_plane[1].offset;
3397
3398         if (INTEL_GEN(dev_priv) >= 11)
3399                 max_width = icl_max_plane_width(fb, 0, rotation);
3400         else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
3401                 max_width = glk_max_plane_width(fb, 0, rotation);
3402         else
3403                 max_width = skl_max_plane_width(fb, 0, rotation);
3404
3405         if (w > max_width || h > max_height) {
3406                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
3407                               w, h, max_width, max_height);
3408                 return -EINVAL;
3409         }
3410
3411         intel_add_fb_offsets(&x, &y, plane_state, 0);
3412         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3413         alignment = intel_surf_alignment(fb, 0);
3414
3415         /*
3416          * AUX surface offset is specified as the distance from the
3417          * main surface offset, and it must be non-negative. Make
3418          * sure that is what we will get.
3419          */
3420         if (offset > aux_offset)
3421                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3422                                                            offset, aux_offset & ~(alignment - 1));
3423
3424         /*
3425          * When using an X-tiled surface, the plane blows up
3426          * if the x offset + width exceed the stride.
3427          *
3428          * TODO: linear and Y-tiled seem fine, Yf untested,
3429          */
3430         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3431                 int cpp = fb->format->cpp[0];
3432
3433                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3434                         if (offset == 0) {
3435                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3436                                 return -EINVAL;
3437                         }
3438
3439                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3440                                                                    offset, offset - alignment);
3441                 }
3442         }
3443
3444         /*
3445          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3446          * they match with the main surface x/y offsets.
3447          */
3448         if (is_ccs_modifier(fb->modifier)) {
3449                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3450                         if (offset == 0)
3451                                 break;
3452
3453                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3454                                                                    offset, offset - alignment);
3455                 }
3456
3457                 if (x != plane_state->color_plane[1].x || y != plane_state->color_plane[1].y) {
3458                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3459                         return -EINVAL;
3460                 }
3461         }
3462
3463         plane_state->color_plane[0].offset = offset;
3464         plane_state->color_plane[0].x = x;
3465         plane_state->color_plane[0].y = y;
3466
3467         /*
3468          * Put the final coordinates back so that the src
3469          * coordinate checks will see the right values.
3470          */
3471         drm_rect_translate(&plane_state->base.src,
3472                            (x << 16) - plane_state->base.src.x1,
3473                            (y << 16) - plane_state->base.src.y1);
3474
3475         return 0;
3476 }
3477
3478 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3479 {
3480         const struct drm_framebuffer *fb = plane_state->base.fb;
3481         unsigned int rotation = plane_state->base.rotation;
3482         int max_width = skl_max_plane_width(fb, 1, rotation);
3483         int max_height = 4096;
3484         int x = plane_state->base.src.x1 >> 17;
3485         int y = plane_state->base.src.y1 >> 17;
3486         int w = drm_rect_width(&plane_state->base.src) >> 17;
3487         int h = drm_rect_height(&plane_state->base.src) >> 17;
3488         u32 offset;
3489
3490         intel_add_fb_offsets(&x, &y, plane_state, 1);
3491         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3492
3493         /* FIXME not quite sure how/if these apply to the chroma plane */
3494         if (w > max_width || h > max_height) {
3495                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3496                               w, h, max_width, max_height);
3497                 return -EINVAL;
3498         }
3499
3500         plane_state->color_plane[1].offset = offset;
3501         plane_state->color_plane[1].x = x;
3502         plane_state->color_plane[1].y = y;
3503
3504         return 0;
3505 }
3506
3507 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3508 {
3509         const struct drm_framebuffer *fb = plane_state->base.fb;
3510         int src_x = plane_state->base.src.x1 >> 16;
3511         int src_y = plane_state->base.src.y1 >> 16;
3512         int hsub = fb->format->hsub;
3513         int vsub = fb->format->vsub;
3514         int x = src_x / hsub;
3515         int y = src_y / vsub;
3516         u32 offset;
3517
3518         intel_add_fb_offsets(&x, &y, plane_state, 1);
3519         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3520
3521         plane_state->color_plane[1].offset = offset;
3522         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3523         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3524
3525         return 0;
3526 }
3527
3528 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3529 {
3530         const struct drm_framebuffer *fb = plane_state->base.fb;
3531         int ret;
3532
3533         ret = intel_plane_compute_gtt(plane_state);
3534         if (ret)
3535                 return ret;
3536
3537         if (!plane_state->base.visible)
3538                 return 0;
3539
3540         /*
3541          * Handle the AUX surface first since
3542          * the main surface setup depends on it.
3543          */
3544         if (is_planar_yuv_format(fb->format->format)) {
3545                 ret = skl_check_nv12_aux_surface(plane_state);
3546                 if (ret)
3547                         return ret;
3548         } else if (is_ccs_modifier(fb->modifier)) {
3549                 ret = skl_check_ccs_aux_surface(plane_state);
3550                 if (ret)
3551                         return ret;
3552         } else {
3553                 plane_state->color_plane[1].offset = ~0xfff;
3554                 plane_state->color_plane[1].x = 0;
3555                 plane_state->color_plane[1].y = 0;
3556         }
3557
3558         ret = skl_check_main_surface(plane_state);
3559         if (ret)
3560                 return ret;
3561
3562         return 0;
3563 }
3564
3565 unsigned int
3566 i9xx_plane_max_stride(struct intel_plane *plane,
3567                       u32 pixel_format, u64 modifier,
3568                       unsigned int rotation)
3569 {
3570         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3571
3572         if (!HAS_GMCH(dev_priv)) {
3573                 return 32*1024;
3574         } else if (INTEL_GEN(dev_priv) >= 4) {
3575                 if (modifier == I915_FORMAT_MOD_X_TILED)
3576                         return 16*1024;
3577                 else
3578                         return 32*1024;
3579         } else if (INTEL_GEN(dev_priv) >= 3) {
3580                 if (modifier == I915_FORMAT_MOD_X_TILED)
3581                         return 8*1024;
3582                 else
3583                         return 16*1024;
3584         } else {
3585                 if (plane->i9xx_plane == PLANE_C)
3586                         return 4*1024;
3587                 else
3588                         return 8*1024;
3589         }
3590 }
3591
3592 static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
3593 {
3594         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
3595         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3596         u32 dspcntr = 0;
3597
3598         if (crtc_state->gamma_enable)
3599                 dspcntr |= DISPPLANE_GAMMA_ENABLE;
3600
3601         if (crtc_state->csc_enable)
3602                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3603
3604         if (INTEL_GEN(dev_priv) < 5)
3605                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3606
3607         return dspcntr;
3608 }
3609
3610 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3611                           const struct intel_plane_state *plane_state)
3612 {
3613         struct drm_i915_private *dev_priv =
3614                 to_i915(plane_state->base.plane->dev);
3615         const struct drm_framebuffer *fb = plane_state->base.fb;
3616         unsigned int rotation = plane_state->base.rotation;
3617         u32 dspcntr;
3618
3619         dspcntr = DISPLAY_PLANE_ENABLE;
3620
3621         if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) ||
3622             IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
3623                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
3624
3625         switch (fb->format->format) {
3626         case DRM_FORMAT_C8:
3627                 dspcntr |= DISPPLANE_8BPP;
3628                 break;
3629         case DRM_FORMAT_XRGB1555:
3630                 dspcntr |= DISPPLANE_BGRX555;
3631                 break;
3632         case DRM_FORMAT_RGB565:
3633                 dspcntr |= DISPPLANE_BGRX565;
3634                 break;
3635         case DRM_FORMAT_XRGB8888:
3636                 dspcntr |= DISPPLANE_BGRX888;
3637                 break;
3638         case DRM_FORMAT_XBGR8888:
3639                 dspcntr |= DISPPLANE_RGBX888;
3640                 break;
3641         case DRM_FORMAT_XRGB2101010:
3642                 dspcntr |= DISPPLANE_BGRX101010;
3643                 break;
3644         case DRM_FORMAT_XBGR2101010:
3645                 dspcntr |= DISPPLANE_RGBX101010;
3646                 break;
3647         default:
3648                 MISSING_CASE(fb->format->format);
3649                 return 0;
3650         }
3651
3652         if (INTEL_GEN(dev_priv) >= 4 &&
3653             fb->modifier == I915_FORMAT_MOD_X_TILED)
3654                 dspcntr |= DISPPLANE_TILED;
3655
3656         if (rotation & DRM_MODE_ROTATE_180)
3657                 dspcntr |= DISPPLANE_ROTATE_180;
3658
3659         if (rotation & DRM_MODE_REFLECT_X)
3660                 dspcntr |= DISPPLANE_MIRROR;
3661
3662         return dspcntr;
3663 }
3664
3665 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
3666 {
3667         struct drm_i915_private *dev_priv =
3668                 to_i915(plane_state->base.plane->dev);
3669         int src_x, src_y;
3670         u32 offset;
3671         int ret;
3672
3673         ret = intel_plane_compute_gtt(plane_state);
3674         if (ret)
3675                 return ret;
3676
3677         if (!plane_state->base.visible)
3678                 return 0;
3679
3680         src_x = plane_state->base.src.x1 >> 16;
3681         src_y = plane_state->base.src.y1 >> 16;
3682
3683         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
3684
3685         if (INTEL_GEN(dev_priv) >= 4)
3686                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
3687                                                             plane_state, 0);
3688         else
3689                 offset = 0;
3690
3691         /*
3692          * Put the final coordinates back so that the src
3693          * coordinate checks will see the right values.
3694          */
3695         drm_rect_translate(&plane_state->base.src,
3696                            (src_x << 16) - plane_state->base.src.x1,
3697                            (src_y << 16) - plane_state->base.src.y1);
3698
3699         /* HSW/BDW do this automagically in hardware */
3700         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
3701                 unsigned int rotation = plane_state->base.rotation;
3702                 int src_w = drm_rect_width(&plane_state->base.src) >> 16;
3703                 int src_h = drm_rect_height(&plane_state->base.src) >> 16;
3704
3705                 if (rotation & DRM_MODE_ROTATE_180) {
3706                         src_x += src_w - 1;
3707                         src_y += src_h - 1;
3708                 } else if (rotation & DRM_MODE_REFLECT_X) {
3709                         src_x += src_w - 1;
3710                 }
3711         }
3712
3713         plane_state->color_plane[0].offset = offset;
3714         plane_state->color_plane[0].x = src_x;
3715         plane_state->color_plane[0].y = src_y;
3716
3717         return 0;
3718 }
3719
3720 static bool i9xx_plane_has_windowing(struct intel_plane *plane)
3721 {
3722         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3723         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3724
3725         if (IS_CHERRYVIEW(dev_priv))
3726                 return i9xx_plane == PLANE_B;
3727         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
3728                 return false;
3729         else if (IS_GEN(dev_priv, 4))
3730                 return i9xx_plane == PLANE_C;
3731         else
3732                 return i9xx_plane == PLANE_B ||
3733                         i9xx_plane == PLANE_C;
3734 }
3735
3736 static int
3737 i9xx_plane_check(struct intel_crtc_state *crtc_state,
3738                  struct intel_plane_state *plane_state)
3739 {
3740         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
3741         int ret;
3742
3743         ret = chv_plane_check_rotation(plane_state);
3744         if (ret)
3745                 return ret;
3746
3747         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
3748                                                   &crtc_state->base,
3749                                                   DRM_PLANE_HELPER_NO_SCALING,
3750                                                   DRM_PLANE_HELPER_NO_SCALING,
3751                                                   i9xx_plane_has_windowing(plane),
3752                                                   true);
3753         if (ret)
3754                 return ret;
3755
3756         ret = i9xx_check_plane_surface(plane_state);
3757         if (ret)
3758                 return ret;
3759
3760         if (!plane_state->base.visible)
3761                 return 0;
3762
3763         ret = intel_plane_check_src_coordinates(plane_state);
3764         if (ret)
3765                 return ret;
3766
3767         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
3768
3769         return 0;
3770 }
3771
3772 static void i9xx_update_plane(struct intel_plane *plane,
3773                               const struct intel_crtc_state *crtc_state,
3774                               const struct intel_plane_state *plane_state)
3775 {
3776         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3777         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3778         u32 linear_offset;
3779         int x = plane_state->color_plane[0].x;
3780         int y = plane_state->color_plane[0].y;
3781         int crtc_x = plane_state->base.dst.x1;
3782         int crtc_y = plane_state->base.dst.y1;
3783         int crtc_w = drm_rect_width(&plane_state->base.dst);
3784         int crtc_h = drm_rect_height(&plane_state->base.dst);
3785         unsigned long irqflags;
3786         u32 dspaddr_offset;
3787         u32 dspcntr;
3788
3789         dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state);
3790
3791         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
3792
3793         if (INTEL_GEN(dev_priv) >= 4)
3794                 dspaddr_offset = plane_state->color_plane[0].offset;
3795         else
3796                 dspaddr_offset = linear_offset;
3797
3798         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3799
3800         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
3801
3802         if (INTEL_GEN(dev_priv) < 4) {
3803                 /*
3804                  * PLANE_A doesn't actually have a full window
3805                  * generator but let's assume we still need to
3806                  * program whatever is there.
3807                  */
3808                 I915_WRITE_FW(DSPPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
3809                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
3810                               ((crtc_h - 1) << 16) | (crtc_w - 1));
3811         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
3812                 I915_WRITE_FW(PRIMPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
3813                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
3814                               ((crtc_h - 1) << 16) | (crtc_w - 1));
3815                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
3816         }
3817
3818         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3819                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
3820         } else if (INTEL_GEN(dev_priv) >= 4) {
3821                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
3822                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
3823         }
3824
3825         /*
3826          * The control register self-arms if the plane was previously
3827          * disabled. Try to make the plane enable atomic by writing
3828          * the control register just before the surface register.
3829          */
3830         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
3831         if (INTEL_GEN(dev_priv) >= 4)
3832                 I915_WRITE_FW(DSPSURF(i9xx_plane),
3833                               intel_plane_ggtt_offset(plane_state) +
3834                               dspaddr_offset);
3835         else
3836                 I915_WRITE_FW(DSPADDR(i9xx_plane),
3837                               intel_plane_ggtt_offset(plane_state) +
3838                               dspaddr_offset);
3839
3840         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3841 }
3842
3843 static void i9xx_disable_plane(struct intel_plane *plane,
3844                                const struct intel_crtc_state *crtc_state)
3845 {
3846         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3847         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3848         unsigned long irqflags;
3849         u32 dspcntr;
3850
3851         /*
3852          * DSPCNTR pipe gamma enable on g4x+ and pipe csc
3853          * enable on ilk+ affect the pipe bottom color as
3854          * well, so we must configure them even if the plane
3855          * is disabled.
3856          *
3857          * On pre-g4x there is no way to gamma correct the
3858          * pipe bottom color but we'll keep on doing this
3859          * anyway so that the crtc state readout works correctly.
3860          */
3861         dspcntr = i9xx_plane_ctl_crtc(crtc_state);
3862
3863         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3864
3865         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
3866         if (INTEL_GEN(dev_priv) >= 4)
3867                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
3868         else
3869                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
3870
3871         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3872 }
3873
3874 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
3875                                     enum pipe *pipe)
3876 {
3877         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3878         enum intel_display_power_domain power_domain;
3879         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3880         intel_wakeref_t wakeref;
3881         bool ret;
3882         u32 val;
3883
3884         /*
3885          * Not 100% correct for planes that can move between pipes,
3886          * but that's only the case for gen2-4 which don't have any
3887          * display power wells.
3888          */
3889         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
3890         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
3891         if (!wakeref)
3892                 return false;
3893
3894         val = I915_READ(DSPCNTR(i9xx_plane));
3895
3896         ret = val & DISPLAY_PLANE_ENABLE;
3897
3898         if (INTEL_GEN(dev_priv) >= 5)
3899                 *pipe = plane->pipe;
3900         else
3901                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
3902                         DISPPLANE_SEL_PIPE_SHIFT;
3903
3904         intel_display_power_put(dev_priv, power_domain, wakeref);
3905
3906         return ret;
3907 }
3908
3909 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
3910 {
3911         struct drm_device *dev = intel_crtc->base.dev;
3912         struct drm_i915_private *dev_priv = to_i915(dev);
3913
3914         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
3915         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
3916         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
3917 }
3918
3919 /*
3920  * This function detaches (aka. unbinds) unused scalers in hardware
3921  */
3922 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
3923 {
3924         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3925         const struct intel_crtc_scaler_state *scaler_state =
3926                 &crtc_state->scaler_state;
3927         int i;
3928
3929         /* loop through and disable scalers that aren't in use */
3930         for (i = 0; i < intel_crtc->num_scalers; i++) {
3931                 if (!scaler_state->scalers[i].in_use)
3932                         skl_detach_scaler(intel_crtc, i);
3933         }
3934 }
3935
3936 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
3937                                           int color_plane, unsigned int rotation)
3938 {
3939         /*
3940          * The stride is either expressed as a multiple of 64 bytes chunks for
3941          * linear buffers or in number of tiles for tiled buffers.
3942          */
3943         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
3944                 return 64;
3945         else if (drm_rotation_90_or_270(rotation))
3946                 return intel_tile_height(fb, color_plane);
3947         else
3948                 return intel_tile_width_bytes(fb, color_plane);
3949 }
3950
3951 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
3952                      int color_plane)
3953 {
3954         const struct drm_framebuffer *fb = plane_state->base.fb;
3955         unsigned int rotation = plane_state->base.rotation;
3956         u32 stride = plane_state->color_plane[color_plane].stride;
3957
3958         if (color_plane >= fb->format->num_planes)
3959                 return 0;
3960
3961         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
3962 }
3963
3964 static u32 skl_plane_ctl_format(u32 pixel_format)
3965 {
3966         switch (pixel_format) {
3967         case DRM_FORMAT_C8:
3968                 return PLANE_CTL_FORMAT_INDEXED;
3969         case DRM_FORMAT_RGB565:
3970                 return PLANE_CTL_FORMAT_RGB_565;
3971         case DRM_FORMAT_XBGR8888:
3972         case DRM_FORMAT_ABGR8888:
3973                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3974         case DRM_FORMAT_XRGB8888:
3975         case DRM_FORMAT_ARGB8888:
3976                 return PLANE_CTL_FORMAT_XRGB_8888;
3977         case DRM_FORMAT_XBGR2101010:
3978                 return PLANE_CTL_FORMAT_XRGB_2101010 | PLANE_CTL_ORDER_RGBX;
3979         case DRM_FORMAT_XRGB2101010:
3980                 return PLANE_CTL_FORMAT_XRGB_2101010;
3981         case DRM_FORMAT_XBGR16161616F:
3982         case DRM_FORMAT_ABGR16161616F:
3983                 return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
3984         case DRM_FORMAT_XRGB16161616F:
3985         case DRM_FORMAT_ARGB16161616F:
3986                 return PLANE_CTL_FORMAT_XRGB_16161616F;
3987         case DRM_FORMAT_YUYV:
3988                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3989         case DRM_FORMAT_YVYU:
3990                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3991         case DRM_FORMAT_UYVY:
3992                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3993         case DRM_FORMAT_VYUY:
3994                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3995         case DRM_FORMAT_NV12:
3996                 return PLANE_CTL_FORMAT_NV12;
3997         case DRM_FORMAT_P010:
3998                 return PLANE_CTL_FORMAT_P010;
3999         case DRM_FORMAT_P012:
4000                 return PLANE_CTL_FORMAT_P012;
4001         case DRM_FORMAT_P016:
4002                 return PLANE_CTL_FORMAT_P016;
4003         case DRM_FORMAT_Y210:
4004                 return PLANE_CTL_FORMAT_Y210;
4005         case DRM_FORMAT_Y212:
4006                 return PLANE_CTL_FORMAT_Y212;
4007         case DRM_FORMAT_Y216:
4008                 return PLANE_CTL_FORMAT_Y216;
4009         case DRM_FORMAT_XVYU2101010:
4010                 return PLANE_CTL_FORMAT_Y410;
4011         case DRM_FORMAT_XVYU12_16161616:
4012                 return PLANE_CTL_FORMAT_Y412;
4013         case DRM_FORMAT_XVYU16161616:
4014                 return PLANE_CTL_FORMAT_Y416;
4015         default:
4016                 MISSING_CASE(pixel_format);
4017         }
4018
4019         return 0;
4020 }
4021
4022 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
4023 {
4024         if (!plane_state->base.fb->format->has_alpha)
4025                 return PLANE_CTL_ALPHA_DISABLE;
4026
4027         switch (plane_state->base.pixel_blend_mode) {
4028         case DRM_MODE_BLEND_PIXEL_NONE:
4029                 return PLANE_CTL_ALPHA_DISABLE;
4030         case DRM_MODE_BLEND_PREMULTI:
4031                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
4032         case DRM_MODE_BLEND_COVERAGE:
4033                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
4034         default:
4035                 MISSING_CASE(plane_state->base.pixel_blend_mode);
4036                 return PLANE_CTL_ALPHA_DISABLE;
4037         }
4038 }
4039
4040 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
4041 {
4042         if (!plane_state->base.fb->format->has_alpha)
4043                 return PLANE_COLOR_ALPHA_DISABLE;
4044
4045         switch (plane_state->base.pixel_blend_mode) {
4046         case DRM_MODE_BLEND_PIXEL_NONE:
4047                 return PLANE_COLOR_ALPHA_DISABLE;
4048         case DRM_MODE_BLEND_PREMULTI:
4049                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
4050         case DRM_MODE_BLEND_COVERAGE:
4051                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
4052         default:
4053                 MISSING_CASE(plane_state->base.pixel_blend_mode);
4054                 return PLANE_COLOR_ALPHA_DISABLE;
4055         }
4056 }
4057
4058 static u32 skl_plane_ctl_tiling(u64 fb_modifier)
4059 {
4060         switch (fb_modifier) {
4061         case DRM_FORMAT_MOD_LINEAR:
4062                 break;
4063         case I915_FORMAT_MOD_X_TILED:
4064                 return PLANE_CTL_TILED_X;
4065         case I915_FORMAT_MOD_Y_TILED:
4066                 return PLANE_CTL_TILED_Y;
4067         case I915_FORMAT_MOD_Y_TILED_CCS:
4068                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4069         case I915_FORMAT_MOD_Yf_TILED:
4070                 return PLANE_CTL_TILED_YF;
4071         case I915_FORMAT_MOD_Yf_TILED_CCS:
4072                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4073         default:
4074                 MISSING_CASE(fb_modifier);
4075         }
4076
4077         return 0;
4078 }
4079
4080 static u32 skl_plane_ctl_rotate(unsigned int rotate)
4081 {
4082         switch (rotate) {
4083         case DRM_MODE_ROTATE_0:
4084                 break;
4085         /*
4086          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
4087          * while i915 HW rotation is clockwise, thats why this swapping.
4088          */
4089         case DRM_MODE_ROTATE_90:
4090                 return PLANE_CTL_ROTATE_270;
4091         case DRM_MODE_ROTATE_180:
4092                 return PLANE_CTL_ROTATE_180;
4093         case DRM_MODE_ROTATE_270:
4094                 return PLANE_CTL_ROTATE_90;
4095         default:
4096                 MISSING_CASE(rotate);
4097         }
4098
4099         return 0;
4100 }
4101
4102 static u32 cnl_plane_ctl_flip(unsigned int reflect)
4103 {
4104         switch (reflect) {
4105         case 0:
4106                 break;
4107         case DRM_MODE_REFLECT_X:
4108                 return PLANE_CTL_FLIP_HORIZONTAL;
4109         case DRM_MODE_REFLECT_Y:
4110         default:
4111                 MISSING_CASE(reflect);
4112         }
4113
4114         return 0;
4115 }
4116
4117 u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
4118 {
4119         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4120         u32 plane_ctl = 0;
4121
4122         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4123                 return plane_ctl;
4124
4125         if (crtc_state->gamma_enable)
4126                 plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
4127
4128         if (crtc_state->csc_enable)
4129                 plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
4130
4131         return plane_ctl;
4132 }
4133
4134 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
4135                   const struct intel_plane_state *plane_state)
4136 {
4137         struct drm_i915_private *dev_priv =
4138                 to_i915(plane_state->base.plane->dev);
4139         const struct drm_framebuffer *fb = plane_state->base.fb;
4140         unsigned int rotation = plane_state->base.rotation;
4141         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
4142         u32 plane_ctl;
4143
4144         plane_ctl = PLANE_CTL_ENABLE;
4145
4146         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
4147                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
4148                 plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
4149
4150                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
4151                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
4152
4153                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4154                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
4155         }
4156
4157         plane_ctl |= skl_plane_ctl_format(fb->format->format);
4158         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
4159         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
4160
4161         if (INTEL_GEN(dev_priv) >= 10)
4162                 plane_ctl |= cnl_plane_ctl_flip(rotation &
4163                                                 DRM_MODE_REFLECT_MASK);
4164
4165         if (key->flags & I915_SET_COLORKEY_DESTINATION)
4166                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
4167         else if (key->flags & I915_SET_COLORKEY_SOURCE)
4168                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
4169
4170         return plane_ctl;
4171 }
4172
4173 u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
4174 {
4175         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4176         u32 plane_color_ctl = 0;
4177
4178         if (INTEL_GEN(dev_priv) >= 11)
4179                 return plane_color_ctl;
4180
4181         if (crtc_state->gamma_enable)
4182                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
4183
4184         if (crtc_state->csc_enable)
4185                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
4186
4187         return plane_color_ctl;
4188 }
4189
4190 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
4191                         const struct intel_plane_state *plane_state)
4192 {
4193         struct drm_i915_private *dev_priv =
4194                 to_i915(plane_state->base.plane->dev);
4195         const struct drm_framebuffer *fb = plane_state->base.fb;
4196         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
4197         u32 plane_color_ctl = 0;
4198
4199         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
4200         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
4201
4202         if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
4203                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
4204                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
4205                 else
4206                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
4207
4208                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4209                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
4210         } else if (fb->format->is_yuv) {
4211                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
4212         }
4213
4214         return plane_color_ctl;
4215 }
4216
4217 static int
4218 __intel_display_resume(struct drm_device *dev,
4219                        struct drm_atomic_state *state,
4220                        struct drm_modeset_acquire_ctx *ctx)
4221 {
4222         struct drm_crtc_state *crtc_state;
4223         struct drm_crtc *crtc;
4224         int i, ret;
4225
4226         intel_modeset_setup_hw_state(dev, ctx);
4227         i915_redisable_vga(to_i915(dev));
4228
4229         if (!state)
4230                 return 0;
4231
4232         /*
4233          * We've duplicated the state, pointers to the old state are invalid.
4234          *
4235          * Don't attempt to use the old state until we commit the duplicated state.
4236          */
4237         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
4238                 /*
4239                  * Force recalculation even if we restore
4240                  * current state. With fast modeset this may not result
4241                  * in a modeset when the state is compatible.
4242                  */
4243                 crtc_state->mode_changed = true;
4244         }
4245
4246         /* ignore any reset values/BIOS leftovers in the WM registers */
4247         if (!HAS_GMCH(to_i915(dev)))
4248                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
4249
4250         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
4251
4252         WARN_ON(ret == -EDEADLK);
4253         return ret;
4254 }
4255
4256 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
4257 {
4258         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
4259                 intel_has_gpu_reset(dev_priv));
4260 }
4261
4262 void intel_prepare_reset(struct drm_i915_private *dev_priv)
4263 {
4264         struct drm_device *dev = &dev_priv->drm;
4265         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4266         struct drm_atomic_state *state;
4267         int ret;
4268
4269         /* reset doesn't touch the display */
4270         if (!i915_modparams.force_reset_modeset_test &&
4271             !gpu_reset_clobbers_display(dev_priv))
4272                 return;
4273
4274         /* We have a modeset vs reset deadlock, defensively unbreak it. */
4275         set_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4276         smp_mb__after_atomic();
4277         wake_up_bit(&dev_priv->gt.reset.flags, I915_RESET_MODESET);
4278
4279         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
4280                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
4281                 intel_gt_set_wedged(&dev_priv->gt);
4282         }
4283
4284         /*
4285          * Need mode_config.mutex so that we don't
4286          * trample ongoing ->detect() and whatnot.
4287          */
4288         mutex_lock(&dev->mode_config.mutex);
4289         drm_modeset_acquire_init(ctx, 0);
4290         while (1) {
4291                 ret = drm_modeset_lock_all_ctx(dev, ctx);
4292                 if (ret != -EDEADLK)
4293                         break;
4294
4295                 drm_modeset_backoff(ctx);
4296         }
4297         /*
4298          * Disabling the crtcs gracefully seems nicer. Also the
4299          * g33 docs say we should at least disable all the planes.
4300          */
4301         state = drm_atomic_helper_duplicate_state(dev, ctx);
4302         if (IS_ERR(state)) {
4303                 ret = PTR_ERR(state);
4304                 DRM_ERROR("Duplicating state failed with %i\n", ret);
4305                 return;
4306         }
4307
4308         ret = drm_atomic_helper_disable_all(dev, ctx);
4309         if (ret) {
4310                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
4311                 drm_atomic_state_put(state);
4312                 return;
4313         }
4314
4315         dev_priv->modeset_restore_state = state;
4316         state->acquire_ctx = ctx;
4317 }
4318
4319 void intel_finish_reset(struct drm_i915_private *dev_priv)
4320 {
4321         struct drm_device *dev = &dev_priv->drm;
4322         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4323         struct drm_atomic_state *state;
4324         int ret;
4325
4326         /* reset doesn't touch the display */
4327         if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
4328                 return;
4329
4330         state = fetch_and_zero(&dev_priv->modeset_restore_state);
4331         if (!state)
4332                 goto unlock;
4333
4334         /* reset doesn't touch the display */
4335         if (!gpu_reset_clobbers_display(dev_priv)) {
4336                 /* for testing only restore the display */
4337                 ret = __intel_display_resume(dev, state, ctx);
4338                 if (ret)
4339                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4340         } else {
4341                 /*
4342                  * The display has been reset as well,
4343                  * so need a full re-initialization.
4344                  */
4345                 intel_pps_unlock_regs_wa(dev_priv);
4346                 intel_modeset_init_hw(dev);
4347                 intel_init_clock_gating(dev_priv);
4348
4349                 spin_lock_irq(&dev_priv->irq_lock);
4350                 if (dev_priv->display.hpd_irq_setup)
4351                         dev_priv->display.hpd_irq_setup(dev_priv);
4352                 spin_unlock_irq(&dev_priv->irq_lock);
4353
4354                 ret = __intel_display_resume(dev, state, ctx);
4355                 if (ret)
4356                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4357
4358                 intel_hpd_init(dev_priv);
4359         }
4360
4361         drm_atomic_state_put(state);
4362 unlock:
4363         drm_modeset_drop_locks(ctx);
4364         drm_modeset_acquire_fini(ctx);
4365         mutex_unlock(&dev->mode_config.mutex);
4366
4367         clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4368 }
4369
4370 static void icl_set_pipe_chicken(struct intel_crtc *crtc)
4371 {
4372         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4373         enum pipe pipe = crtc->pipe;
4374         u32 tmp;
4375
4376         tmp = I915_READ(PIPE_CHICKEN(pipe));
4377
4378         /*
4379          * Display WA #1153: icl
4380          * enable hardware to bypass the alpha math
4381          * and rounding for per-pixel values 00 and 0xff
4382          */
4383         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
4384         /*
4385          * Display WA # 1605353570: icl
4386          * Set the pixel rounding bit to 1 for allowing
4387          * passthrough of Frame buffer pixels unmodified
4388          * across pipe
4389          */
4390         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
4391         I915_WRITE(PIPE_CHICKEN(pipe), tmp);
4392 }
4393
4394 static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_state,
4395                                      const struct intel_crtc_state *new_crtc_state)
4396 {
4397         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
4398         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4399
4400         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
4401         crtc->base.mode = new_crtc_state->base.mode;
4402
4403         /*
4404          * Update pipe size and adjust fitter if needed: the reason for this is
4405          * that in compute_mode_changes we check the native mode (not the pfit
4406          * mode) to see if we can flip rather than do a full mode set. In the
4407          * fastboot case, we'll flip, but if we don't update the pipesrc and
4408          * pfit state, we'll end up with a big fb scanned out into the wrong
4409          * sized surface.
4410          */
4411
4412         I915_WRITE(PIPESRC(crtc->pipe),
4413                    ((new_crtc_state->pipe_src_w - 1) << 16) |
4414                    (new_crtc_state->pipe_src_h - 1));
4415
4416         /* on skylake this is done by detaching scalers */
4417         if (INTEL_GEN(dev_priv) >= 9) {
4418                 skl_detach_scalers(new_crtc_state);
4419
4420                 if (new_crtc_state->pch_pfit.enabled)
4421                         skylake_pfit_enable(new_crtc_state);
4422         } else if (HAS_PCH_SPLIT(dev_priv)) {
4423                 if (new_crtc_state->pch_pfit.enabled)
4424                         ironlake_pfit_enable(new_crtc_state);
4425                 else if (old_crtc_state->pch_pfit.enabled)
4426                         ironlake_pfit_disable(old_crtc_state);
4427         }
4428
4429         if (INTEL_GEN(dev_priv) >= 11)
4430                 icl_set_pipe_chicken(crtc);
4431 }
4432
4433 static void intel_fdi_normal_train(struct intel_crtc *crtc)
4434 {
4435         struct drm_device *dev = crtc->base.dev;
4436         struct drm_i915_private *dev_priv = to_i915(dev);
4437         int pipe = crtc->pipe;
4438         i915_reg_t reg;
4439         u32 temp;
4440
4441         /* enable normal train */
4442         reg = FDI_TX_CTL(pipe);
4443         temp = I915_READ(reg);
4444         if (IS_IVYBRIDGE(dev_priv)) {
4445                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4446                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
4447         } else {
4448                 temp &= ~FDI_LINK_TRAIN_NONE;
4449                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
4450         }
4451         I915_WRITE(reg, temp);
4452
4453         reg = FDI_RX_CTL(pipe);
4454         temp = I915_READ(reg);
4455         if (HAS_PCH_CPT(dev_priv)) {
4456                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4457                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
4458         } else {
4459                 temp &= ~FDI_LINK_TRAIN_NONE;
4460                 temp |= FDI_LINK_TRAIN_NONE;
4461         }
4462         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
4463
4464         /* wait one idle pattern time */
4465         POSTING_READ(reg);
4466         udelay(1000);
4467
4468         /* IVB wants error correction enabled */
4469         if (IS_IVYBRIDGE(dev_priv))
4470                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
4471                            FDI_FE_ERRC_ENABLE);
4472 }
4473
4474 /* The FDI link training functions for ILK/Ibexpeak. */
4475 static void ironlake_fdi_link_train(struct intel_crtc *crtc,
4476                                     const struct intel_crtc_state *crtc_state)
4477 {
4478         struct drm_device *dev = crtc->base.dev;
4479         struct drm_i915_private *dev_priv = to_i915(dev);
4480         int pipe = crtc->pipe;
4481         i915_reg_t reg;
4482         u32 temp, tries;
4483
4484         /* FDI needs bits from pipe first */
4485         assert_pipe_enabled(dev_priv, pipe);
4486
4487         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4488            for train result */
4489         reg = FDI_RX_IMR(pipe);
4490         temp = I915_READ(reg);
4491         temp &= ~FDI_RX_SYMBOL_LOCK;
4492         temp &= ~FDI_RX_BIT_LOCK;
4493         I915_WRITE(reg, temp);
4494         I915_READ(reg);
4495         udelay(150);
4496
4497         /* enable CPU FDI TX and PCH FDI RX */
4498         reg = FDI_TX_CTL(pipe);
4499         temp = I915_READ(reg);
4500         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4501         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4502         temp &= ~FDI_LINK_TRAIN_NONE;
4503         temp |= FDI_LINK_TRAIN_PATTERN_1;
4504         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4505
4506         reg = FDI_RX_CTL(pipe);
4507         temp = I915_READ(reg);
4508         temp &= ~FDI_LINK_TRAIN_NONE;
4509         temp |= FDI_LINK_TRAIN_PATTERN_1;
4510         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4511
4512         POSTING_READ(reg);
4513         udelay(150);
4514
4515         /* Ironlake workaround, enable clock pointer after FDI enable*/
4516         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4517         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
4518                    FDI_RX_PHASE_SYNC_POINTER_EN);
4519
4520         reg = FDI_RX_IIR(pipe);
4521         for (tries = 0; tries < 5; tries++) {
4522                 temp = I915_READ(reg);
4523                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4524
4525                 if ((temp & FDI_RX_BIT_LOCK)) {
4526                         DRM_DEBUG_KMS("FDI train 1 done.\n");
4527                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4528                         break;
4529                 }
4530         }
4531         if (tries == 5)
4532                 DRM_ERROR("FDI train 1 fail!\n");
4533
4534         /* Train 2 */
4535         reg = FDI_TX_CTL(pipe);
4536         temp = I915_READ(reg);
4537         temp &= ~FDI_LINK_TRAIN_NONE;
4538         temp |= FDI_LINK_TRAIN_PATTERN_2;
4539         I915_WRITE(reg, temp);
4540
4541         reg = FDI_RX_CTL(pipe);
4542         temp = I915_READ(reg);
4543         temp &= ~FDI_LINK_TRAIN_NONE;
4544         temp |= FDI_LINK_TRAIN_PATTERN_2;
4545         I915_WRITE(reg, temp);
4546
4547         POSTING_READ(reg);
4548         udelay(150);
4549
4550         reg = FDI_RX_IIR(pipe);
4551         for (tries = 0; tries < 5; tries++) {
4552                 temp = I915_READ(reg);
4553                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4554
4555                 if (temp & FDI_RX_SYMBOL_LOCK) {
4556                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4557                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4558                         break;
4559                 }
4560         }
4561         if (tries == 5)
4562                 DRM_ERROR("FDI train 2 fail!\n");
4563
4564         DRM_DEBUG_KMS("FDI train done\n");
4565
4566 }
4567
4568 static const int snb_b_fdi_train_param[] = {
4569         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4570         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4571         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4572         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4573 };
4574
4575 /* The FDI link training functions for SNB/Cougarpoint. */
4576 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4577                                 const struct intel_crtc_state *crtc_state)
4578 {
4579         struct drm_device *dev = crtc->base.dev;
4580         struct drm_i915_private *dev_priv = to_i915(dev);
4581         int pipe = crtc->pipe;
4582         i915_reg_t reg;
4583         u32 temp, i, retry;
4584
4585         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4586            for train result */
4587         reg = FDI_RX_IMR(pipe);
4588         temp = I915_READ(reg);
4589         temp &= ~FDI_RX_SYMBOL_LOCK;
4590         temp &= ~FDI_RX_BIT_LOCK;
4591         I915_WRITE(reg, temp);
4592
4593         POSTING_READ(reg);
4594         udelay(150);
4595
4596         /* enable CPU FDI TX and PCH FDI RX */
4597         reg = FDI_TX_CTL(pipe);
4598         temp = I915_READ(reg);
4599         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4600         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4601         temp &= ~FDI_LINK_TRAIN_NONE;
4602         temp |= FDI_LINK_TRAIN_PATTERN_1;
4603         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4604         /* SNB-B */
4605         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4606         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4607
4608         I915_WRITE(FDI_RX_MISC(pipe),
4609                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4610
4611         reg = FDI_RX_CTL(pipe);
4612         temp = I915_READ(reg);
4613         if (HAS_PCH_CPT(dev_priv)) {
4614                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4615                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4616         } else {
4617                 temp &= ~FDI_LINK_TRAIN_NONE;
4618                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4619         }
4620         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4621
4622         POSTING_READ(reg);
4623         udelay(150);
4624
4625         for (i = 0; i < 4; i++) {
4626                 reg = FDI_TX_CTL(pipe);
4627                 temp = I915_READ(reg);
4628                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4629                 temp |= snb_b_fdi_train_param[i];
4630                 I915_WRITE(reg, temp);
4631
4632                 POSTING_READ(reg);
4633                 udelay(500);
4634
4635                 for (retry = 0; retry < 5; retry++) {
4636                         reg = FDI_RX_IIR(pipe);
4637                         temp = I915_READ(reg);
4638                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4639                         if (temp & FDI_RX_BIT_LOCK) {
4640                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4641                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
4642                                 break;
4643                         }
4644                         udelay(50);
4645                 }
4646                 if (retry < 5)
4647                         break;
4648         }
4649         if (i == 4)
4650                 DRM_ERROR("FDI train 1 fail!\n");
4651
4652         /* Train 2 */
4653         reg = FDI_TX_CTL(pipe);
4654         temp = I915_READ(reg);
4655         temp &= ~FDI_LINK_TRAIN_NONE;
4656         temp |= FDI_LINK_TRAIN_PATTERN_2;
4657         if (IS_GEN(dev_priv, 6)) {
4658                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4659                 /* SNB-B */
4660                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4661         }
4662         I915_WRITE(reg, temp);
4663
4664         reg = FDI_RX_CTL(pipe);
4665         temp = I915_READ(reg);
4666         if (HAS_PCH_CPT(dev_priv)) {
4667                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4668                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4669         } else {
4670                 temp &= ~FDI_LINK_TRAIN_NONE;
4671                 temp |= FDI_LINK_TRAIN_PATTERN_2;
4672         }
4673         I915_WRITE(reg, temp);
4674
4675         POSTING_READ(reg);
4676         udelay(150);
4677
4678         for (i = 0; i < 4; i++) {
4679                 reg = FDI_TX_CTL(pipe);
4680                 temp = I915_READ(reg);
4681                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4682                 temp |= snb_b_fdi_train_param[i];
4683                 I915_WRITE(reg, temp);
4684
4685                 POSTING_READ(reg);
4686                 udelay(500);
4687
4688                 for (retry = 0; retry < 5; retry++) {
4689                         reg = FDI_RX_IIR(pipe);
4690                         temp = I915_READ(reg);
4691                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4692                         if (temp & FDI_RX_SYMBOL_LOCK) {
4693                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4694                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
4695                                 break;
4696                         }
4697                         udelay(50);
4698                 }
4699                 if (retry < 5)
4700                         break;
4701         }
4702         if (i == 4)
4703                 DRM_ERROR("FDI train 2 fail!\n");
4704
4705         DRM_DEBUG_KMS("FDI train done.\n");
4706 }
4707
4708 /* Manual link training for Ivy Bridge A0 parts */
4709 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
4710                                       const struct intel_crtc_state *crtc_state)
4711 {
4712         struct drm_device *dev = crtc->base.dev;
4713         struct drm_i915_private *dev_priv = to_i915(dev);
4714         int pipe = crtc->pipe;
4715         i915_reg_t reg;
4716         u32 temp, i, j;
4717
4718         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4719            for train result */
4720         reg = FDI_RX_IMR(pipe);
4721         temp = I915_READ(reg);
4722         temp &= ~FDI_RX_SYMBOL_LOCK;
4723         temp &= ~FDI_RX_BIT_LOCK;
4724         I915_WRITE(reg, temp);
4725
4726         POSTING_READ(reg);
4727         udelay(150);
4728
4729         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
4730                       I915_READ(FDI_RX_IIR(pipe)));
4731
4732         /* Try each vswing and preemphasis setting twice before moving on */
4733         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
4734                 /* disable first in case we need to retry */
4735                 reg = FDI_TX_CTL(pipe);
4736                 temp = I915_READ(reg);
4737                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
4738                 temp &= ~FDI_TX_ENABLE;
4739                 I915_WRITE(reg, temp);
4740
4741                 reg = FDI_RX_CTL(pipe);
4742                 temp = I915_READ(reg);
4743                 temp &= ~FDI_LINK_TRAIN_AUTO;
4744                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4745                 temp &= ~FDI_RX_ENABLE;
4746                 I915_WRITE(reg, temp);
4747
4748                 /* enable CPU FDI TX and PCH FDI RX */
4749                 reg = FDI_TX_CTL(pipe);
4750                 temp = I915_READ(reg);
4751                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
4752                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4753                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
4754                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4755                 temp |= snb_b_fdi_train_param[j/2];
4756                 temp |= FDI_COMPOSITE_SYNC;
4757                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
4758
4759                 I915_WRITE(FDI_RX_MISC(pipe),
4760                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4761
4762                 reg = FDI_RX_CTL(pipe);
4763                 temp = I915_READ(reg);
4764                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4765                 temp |= FDI_COMPOSITE_SYNC;
4766                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
4767
4768                 POSTING_READ(reg);
4769                 udelay(1); /* should be 0.5us */
4770
4771                 for (i = 0; i < 4; i++) {
4772                         reg = FDI_RX_IIR(pipe);
4773                         temp = I915_READ(reg);
4774                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4775
4776                         if (temp & FDI_RX_BIT_LOCK ||
4777                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
4778                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4779                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
4780                                               i);
4781                                 break;
4782                         }
4783                         udelay(1); /* should be 0.5us */
4784                 }
4785                 if (i == 4) {
4786                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
4787                         continue;
4788                 }
4789
4790                 /* Train 2 */
4791                 reg = FDI_TX_CTL(pipe);
4792                 temp = I915_READ(reg);
4793                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4794                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
4795                 I915_WRITE(reg, temp);
4796
4797                 reg = FDI_RX_CTL(pipe);
4798                 temp = I915_READ(reg);
4799                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4800                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4801                 I915_WRITE(reg, temp);
4802
4803                 POSTING_READ(reg);
4804                 udelay(2); /* should be 1.5us */
4805
4806                 for (i = 0; i < 4; i++) {
4807                         reg = FDI_RX_IIR(pipe);
4808                         temp = I915_READ(reg);
4809                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4810
4811                         if (temp & FDI_RX_SYMBOL_LOCK ||
4812                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
4813                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4814                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
4815                                               i);
4816                                 goto train_done;
4817                         }
4818                         udelay(2); /* should be 1.5us */
4819                 }
4820                 if (i == 4)
4821                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
4822         }
4823
4824 train_done:
4825         DRM_DEBUG_KMS("FDI train done.\n");
4826 }
4827
4828 static void ironlake_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
4829 {
4830         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4831         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4832         int pipe = intel_crtc->pipe;
4833         i915_reg_t reg;
4834         u32 temp;
4835
4836         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
4837         reg = FDI_RX_CTL(pipe);
4838         temp = I915_READ(reg);
4839         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
4840         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4841         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4842         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
4843
4844         POSTING_READ(reg);
4845         udelay(200);
4846
4847         /* Switch from Rawclk to PCDclk */
4848         temp = I915_READ(reg);
4849         I915_WRITE(reg, temp | FDI_PCDCLK);
4850
4851         POSTING_READ(reg);
4852         udelay(200);
4853
4854         /* Enable CPU FDI TX PLL, always on for Ironlake */
4855         reg = FDI_TX_CTL(pipe);
4856         temp = I915_READ(reg);
4857         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
4858                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
4859
4860                 POSTING_READ(reg);
4861                 udelay(100);
4862         }
4863 }
4864
4865 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
4866 {
4867         struct drm_device *dev = intel_crtc->base.dev;
4868         struct drm_i915_private *dev_priv = to_i915(dev);
4869         int pipe = intel_crtc->pipe;
4870         i915_reg_t reg;
4871         u32 temp;
4872
4873         /* Switch from PCDclk to Rawclk */
4874         reg = FDI_RX_CTL(pipe);
4875         temp = I915_READ(reg);
4876         I915_WRITE(reg, temp & ~FDI_PCDCLK);
4877
4878         /* Disable CPU FDI TX PLL */
4879         reg = FDI_TX_CTL(pipe);
4880         temp = I915_READ(reg);
4881         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
4882
4883         POSTING_READ(reg);
4884         udelay(100);
4885
4886         reg = FDI_RX_CTL(pipe);
4887         temp = I915_READ(reg);
4888         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
4889
4890         /* Wait for the clocks to turn off. */
4891         POSTING_READ(reg);
4892         udelay(100);
4893 }
4894
4895 static void ironlake_fdi_disable(struct drm_crtc *crtc)
4896 {
4897         struct drm_device *dev = crtc->dev;
4898         struct drm_i915_private *dev_priv = to_i915(dev);
4899         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4900         int pipe = intel_crtc->pipe;
4901         i915_reg_t reg;
4902         u32 temp;
4903
4904         /* disable CPU FDI tx and PCH FDI rx */
4905         reg = FDI_TX_CTL(pipe);
4906         temp = I915_READ(reg);
4907         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
4908         POSTING_READ(reg);
4909
4910         reg = FDI_RX_CTL(pipe);
4911         temp = I915_READ(reg);
4912         temp &= ~(0x7 << 16);
4913         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4914         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
4915
4916         POSTING_READ(reg);
4917         udelay(100);
4918
4919         /* Ironlake workaround, disable clock pointer after downing FDI */
4920         if (HAS_PCH_IBX(dev_priv))
4921                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4922
4923         /* still set train pattern 1 */
4924         reg = FDI_TX_CTL(pipe);
4925         temp = I915_READ(reg);
4926         temp &= ~FDI_LINK_TRAIN_NONE;
4927         temp |= FDI_LINK_TRAIN_PATTERN_1;
4928         I915_WRITE(reg, temp);
4929
4930         reg = FDI_RX_CTL(pipe);
4931         temp = I915_READ(reg);
4932         if (HAS_PCH_CPT(dev_priv)) {
4933                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4934                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4935         } else {
4936                 temp &= ~FDI_LINK_TRAIN_NONE;
4937                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4938         }
4939         /* BPC in FDI rx is consistent with that in PIPECONF */
4940         temp &= ~(0x07 << 16);
4941         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4942         I915_WRITE(reg, temp);
4943
4944         POSTING_READ(reg);
4945         udelay(100);
4946 }
4947
4948 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
4949 {
4950         struct drm_crtc *crtc;
4951         bool cleanup_done;
4952
4953         drm_for_each_crtc(crtc, &dev_priv->drm) {
4954                 struct drm_crtc_commit *commit;
4955                 spin_lock(&crtc->commit_lock);
4956                 commit = list_first_entry_or_null(&crtc->commit_list,
4957                                                   struct drm_crtc_commit, commit_entry);
4958                 cleanup_done = commit ?
4959                         try_wait_for_completion(&commit->cleanup_done) : true;
4960                 spin_unlock(&crtc->commit_lock);
4961
4962                 if (cleanup_done)
4963                         continue;
4964
4965                 drm_crtc_wait_one_vblank(crtc);
4966
4967                 return true;
4968         }
4969
4970         return false;
4971 }
4972
4973 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
4974 {
4975         u32 temp;
4976
4977         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
4978
4979         mutex_lock(&dev_priv->sb_lock);
4980
4981         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4982         temp |= SBI_SSCCTL_DISABLE;
4983         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4984
4985         mutex_unlock(&dev_priv->sb_lock);
4986 }
4987
4988 /* Program iCLKIP clock to the desired frequency */
4989 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
4990 {
4991         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4992         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4993         int clock = crtc_state->base.adjusted_mode.crtc_clock;
4994         u32 divsel, phaseinc, auxdiv, phasedir = 0;
4995         u32 temp;
4996
4997         lpt_disable_iclkip(dev_priv);
4998
4999         /* The iCLK virtual clock root frequency is in MHz,
5000          * but the adjusted_mode->crtc_clock in in KHz. To get the
5001          * divisors, it is necessary to divide one by another, so we
5002          * convert the virtual clock precision to KHz here for higher
5003          * precision.
5004          */
5005         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
5006                 u32 iclk_virtual_root_freq = 172800 * 1000;
5007                 u32 iclk_pi_range = 64;
5008                 u32 desired_divisor;
5009
5010                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5011                                                     clock << auxdiv);
5012                 divsel = (desired_divisor / iclk_pi_range) - 2;
5013                 phaseinc = desired_divisor % iclk_pi_range;
5014
5015                 /*
5016                  * Near 20MHz is a corner case which is
5017                  * out of range for the 7-bit divisor
5018                  */
5019                 if (divsel <= 0x7f)
5020                         break;
5021         }
5022
5023         /* This should not happen with any sane values */
5024         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
5025                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
5026         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
5027                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
5028
5029         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
5030                         clock,
5031                         auxdiv,
5032                         divsel,
5033                         phasedir,
5034                         phaseinc);
5035
5036         mutex_lock(&dev_priv->sb_lock);
5037
5038         /* Program SSCDIVINTPHASE6 */
5039         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5040         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
5041         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
5042         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
5043         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
5044         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
5045         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
5046         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
5047
5048         /* Program SSCAUXDIV */
5049         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5050         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
5051         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
5052         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
5053
5054         /* Enable modulator and associated divider */
5055         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5056         temp &= ~SBI_SSCCTL_DISABLE;
5057         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5058
5059         mutex_unlock(&dev_priv->sb_lock);
5060
5061         /* Wait for initialization time */
5062         udelay(24);
5063
5064         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
5065 }
5066
5067 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
5068 {
5069         u32 divsel, phaseinc, auxdiv;
5070         u32 iclk_virtual_root_freq = 172800 * 1000;
5071         u32 iclk_pi_range = 64;
5072         u32 desired_divisor;
5073         u32 temp;
5074
5075         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
5076                 return 0;
5077
5078         mutex_lock(&dev_priv->sb_lock);
5079
5080         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5081         if (temp & SBI_SSCCTL_DISABLE) {
5082                 mutex_unlock(&dev_priv->sb_lock);
5083                 return 0;
5084         }
5085
5086         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5087         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
5088                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
5089         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
5090                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
5091
5092         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5093         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
5094                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
5095
5096         mutex_unlock(&dev_priv->sb_lock);
5097
5098         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
5099
5100         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5101                                  desired_divisor << auxdiv);
5102 }
5103
5104 static void ironlake_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
5105                                                 enum pipe pch_transcoder)
5106 {
5107         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5108         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5109         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5110
5111         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
5112                    I915_READ(HTOTAL(cpu_transcoder)));
5113         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
5114                    I915_READ(HBLANK(cpu_transcoder)));
5115         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
5116                    I915_READ(HSYNC(cpu_transcoder)));
5117
5118         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
5119                    I915_READ(VTOTAL(cpu_transcoder)));
5120         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
5121                    I915_READ(VBLANK(cpu_transcoder)));
5122         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
5123                    I915_READ(VSYNC(cpu_transcoder)));
5124         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
5125                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
5126 }
5127
5128 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
5129 {
5130         u32 temp;
5131
5132         temp = I915_READ(SOUTH_CHICKEN1);
5133         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
5134                 return;
5135
5136         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
5137         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
5138
5139         temp &= ~FDI_BC_BIFURCATION_SELECT;
5140         if (enable)
5141                 temp |= FDI_BC_BIFURCATION_SELECT;
5142
5143         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
5144         I915_WRITE(SOUTH_CHICKEN1, temp);
5145         POSTING_READ(SOUTH_CHICKEN1);
5146 }
5147
5148 static void ivybridge_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
5149 {
5150         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5151         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5152
5153         switch (crtc->pipe) {
5154         case PIPE_A:
5155                 break;
5156         case PIPE_B:
5157                 if (crtc_state->fdi_lanes > 2)
5158                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
5159                 else
5160                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
5161
5162                 break;
5163         case PIPE_C:
5164                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
5165
5166                 break;
5167         default:
5168                 BUG();
5169         }
5170 }
5171
5172 /*
5173  * Finds the encoder associated with the given CRTC. This can only be
5174  * used when we know that the CRTC isn't feeding multiple encoders!
5175  */
5176 static struct intel_encoder *
5177 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
5178                            const struct intel_crtc_state *crtc_state)
5179 {
5180         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5181         const struct drm_connector_state *connector_state;
5182         const struct drm_connector *connector;
5183         struct intel_encoder *encoder = NULL;
5184         int num_encoders = 0;
5185         int i;
5186
5187         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5188                 if (connector_state->crtc != &crtc->base)
5189                         continue;
5190
5191                 encoder = to_intel_encoder(connector_state->best_encoder);
5192                 num_encoders++;
5193         }
5194
5195         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
5196              num_encoders, pipe_name(crtc->pipe));
5197
5198         return encoder;
5199 }
5200
5201 /*
5202  * Enable PCH resources required for PCH ports:
5203  *   - PCH PLLs
5204  *   - FDI training & RX/TX
5205  *   - update transcoder timings
5206  *   - DP transcoding bits
5207  *   - transcoder
5208  */
5209 static void ironlake_pch_enable(const struct intel_atomic_state *state,
5210                                 const struct intel_crtc_state *crtc_state)
5211 {
5212         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5213         struct drm_device *dev = crtc->base.dev;
5214         struct drm_i915_private *dev_priv = to_i915(dev);
5215         int pipe = crtc->pipe;
5216         u32 temp;
5217
5218         assert_pch_transcoder_disabled(dev_priv, pipe);
5219
5220         if (IS_IVYBRIDGE(dev_priv))
5221                 ivybridge_update_fdi_bc_bifurcation(crtc_state);
5222
5223         /* Write the TU size bits before fdi link training, so that error
5224          * detection works. */
5225         I915_WRITE(FDI_RX_TUSIZE1(pipe),
5226                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
5227
5228         /* For PCH output, training FDI link */
5229         dev_priv->display.fdi_link_train(crtc, crtc_state);
5230
5231         /* We need to program the right clock selection before writing the pixel
5232          * mutliplier into the DPLL. */
5233         if (HAS_PCH_CPT(dev_priv)) {
5234                 u32 sel;
5235
5236                 temp = I915_READ(PCH_DPLL_SEL);
5237                 temp |= TRANS_DPLL_ENABLE(pipe);
5238                 sel = TRANS_DPLLB_SEL(pipe);
5239                 if (crtc_state->shared_dpll ==
5240                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
5241                         temp |= sel;
5242                 else
5243                         temp &= ~sel;
5244                 I915_WRITE(PCH_DPLL_SEL, temp);
5245         }
5246
5247         /* XXX: pch pll's can be enabled any time before we enable the PCH
5248          * transcoder, and we actually should do this to not upset any PCH
5249          * transcoder that already use the clock when we share it.
5250          *
5251          * Note that enable_shared_dpll tries to do the right thing, but
5252          * get_shared_dpll unconditionally resets the pll - we need that to have
5253          * the right LVDS enable sequence. */
5254         intel_enable_shared_dpll(crtc_state);
5255
5256         /* set transcoder timing, panel must allow it */
5257         assert_panel_unlocked(dev_priv, pipe);
5258         ironlake_pch_transcoder_set_timings(crtc_state, pipe);
5259
5260         intel_fdi_normal_train(crtc);
5261
5262         /* For PCH DP, enable TRANS_DP_CTL */
5263         if (HAS_PCH_CPT(dev_priv) &&
5264             intel_crtc_has_dp_encoder(crtc_state)) {
5265                 const struct drm_display_mode *adjusted_mode =
5266                         &crtc_state->base.adjusted_mode;
5267                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
5268                 i915_reg_t reg = TRANS_DP_CTL(pipe);
5269                 enum port port;
5270
5271                 temp = I915_READ(reg);
5272                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
5273                           TRANS_DP_SYNC_MASK |
5274                           TRANS_DP_BPC_MASK);
5275                 temp |= TRANS_DP_OUTPUT_ENABLE;
5276                 temp |= bpc << 9; /* same format but at 11:9 */
5277
5278                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
5279                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
5280                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
5281                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
5282
5283                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
5284                 WARN_ON(port < PORT_B || port > PORT_D);
5285                 temp |= TRANS_DP_PORT_SEL(port);
5286
5287                 I915_WRITE(reg, temp);
5288         }
5289
5290         ironlake_enable_pch_transcoder(crtc_state);
5291 }
5292
5293 static void lpt_pch_enable(const struct intel_atomic_state *state,
5294                            const struct intel_crtc_state *crtc_state)
5295 {
5296         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5297         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5298         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5299
5300         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
5301
5302         lpt_program_iclkip(crtc_state);
5303
5304         /* Set transcoder timing. */
5305         ironlake_pch_transcoder_set_timings(crtc_state, PIPE_A);
5306
5307         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
5308 }
5309
5310 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
5311 {
5312         struct drm_i915_private *dev_priv = to_i915(dev);
5313         i915_reg_t dslreg = PIPEDSL(pipe);
5314         u32 temp;
5315
5316         temp = I915_READ(dslreg);
5317         udelay(500);
5318         if (wait_for(I915_READ(dslreg) != temp, 5)) {
5319                 if (wait_for(I915_READ(dslreg) != temp, 5))
5320                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
5321         }
5322 }
5323
5324 /*
5325  * The hardware phase 0.0 refers to the center of the pixel.
5326  * We want to start from the top/left edge which is phase
5327  * -0.5. That matches how the hardware calculates the scaling
5328  * factors (from top-left of the first pixel to bottom-right
5329  * of the last pixel, as opposed to the pixel centers).
5330  *
5331  * For 4:2:0 subsampled chroma planes we obviously have to
5332  * adjust that so that the chroma sample position lands in
5333  * the right spot.
5334  *
5335  * Note that for packed YCbCr 4:2:2 formats there is no way to
5336  * control chroma siting. The hardware simply replicates the
5337  * chroma samples for both of the luma samples, and thus we don't
5338  * actually get the expected MPEG2 chroma siting convention :(
5339  * The same behaviour is observed on pre-SKL platforms as well.
5340  *
5341  * Theory behind the formula (note that we ignore sub-pixel
5342  * source coordinates):
5343  * s = source sample position
5344  * d = destination sample position
5345  *
5346  * Downscaling 4:1:
5347  * -0.5
5348  * | 0.0
5349  * | |     1.5 (initial phase)
5350  * | |     |
5351  * v v     v
5352  * | s | s | s | s |
5353  * |       d       |
5354  *
5355  * Upscaling 1:4:
5356  * -0.5
5357  * | -0.375 (initial phase)
5358  * | |     0.0
5359  * | |     |
5360  * v v     v
5361  * |       s       |
5362  * | d | d | d | d |
5363  */
5364 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
5365 {
5366         int phase = -0x8000;
5367         u16 trip = 0;
5368
5369         if (chroma_cosited)
5370                 phase += (sub - 1) * 0x8000 / sub;
5371
5372         phase += scale / (2 * sub);
5373
5374         /*
5375          * Hardware initial phase limited to [-0.5:1.5].
5376          * Since the max hardware scale factor is 3.0, we
5377          * should never actually excdeed 1.0 here.
5378          */
5379         WARN_ON(phase < -0x8000 || phase > 0x18000);
5380
5381         if (phase < 0)
5382                 phase = 0x10000 + phase;
5383         else
5384                 trip = PS_PHASE_TRIP;
5385
5386         return ((phase >> 2) & PS_PHASE_MASK) | trip;
5387 }
5388
5389 #define SKL_MIN_SRC_W 8
5390 #define SKL_MAX_SRC_W 4096
5391 #define SKL_MIN_SRC_H 8
5392 #define SKL_MAX_SRC_H 4096
5393 #define SKL_MIN_DST_W 8
5394 #define SKL_MAX_DST_W 4096
5395 #define SKL_MIN_DST_H 8
5396 #define SKL_MAX_DST_H 4096
5397 #define ICL_MAX_SRC_W 5120
5398 #define ICL_MAX_SRC_H 4096
5399 #define ICL_MAX_DST_W 5120
5400 #define ICL_MAX_DST_H 4096
5401 #define SKL_MIN_YUV_420_SRC_W 16
5402 #define SKL_MIN_YUV_420_SRC_H 16
5403
5404 static int
5405 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
5406                   unsigned int scaler_user, int *scaler_id,
5407                   int src_w, int src_h, int dst_w, int dst_h,
5408                   const struct drm_format_info *format, bool need_scaler)
5409 {
5410         struct intel_crtc_scaler_state *scaler_state =
5411                 &crtc_state->scaler_state;
5412         struct intel_crtc *intel_crtc =
5413                 to_intel_crtc(crtc_state->base.crtc);
5414         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5415         const struct drm_display_mode *adjusted_mode =
5416                 &crtc_state->base.adjusted_mode;
5417
5418         /*
5419          * Src coordinates are already rotated by 270 degrees for
5420          * the 90/270 degree plane rotation cases (to match the
5421          * GTT mapping), hence no need to account for rotation here.
5422          */
5423         if (src_w != dst_w || src_h != dst_h)
5424                 need_scaler = true;
5425
5426         /*
5427          * Scaling/fitting not supported in IF-ID mode in GEN9+
5428          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
5429          * Once NV12 is enabled, handle it here while allocating scaler
5430          * for NV12.
5431          */
5432         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable &&
5433             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5434                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
5435                 return -EINVAL;
5436         }
5437
5438         /*
5439          * if plane is being disabled or scaler is no more required or force detach
5440          *  - free scaler binded to this plane/crtc
5441          *  - in order to do this, update crtc->scaler_usage
5442          *
5443          * Here scaler state in crtc_state is set free so that
5444          * scaler can be assigned to other user. Actual register
5445          * update to free the scaler is done in plane/panel-fit programming.
5446          * For this purpose crtc/plane_state->scaler_id isn't reset here.
5447          */
5448         if (force_detach || !need_scaler) {
5449                 if (*scaler_id >= 0) {
5450                         scaler_state->scaler_users &= ~(1 << scaler_user);
5451                         scaler_state->scalers[*scaler_id].in_use = 0;
5452
5453                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5454                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
5455                                 intel_crtc->pipe, scaler_user, *scaler_id,
5456                                 scaler_state->scaler_users);
5457                         *scaler_id = -1;
5458                 }
5459                 return 0;
5460         }
5461
5462         if (format && is_planar_yuv_format(format->format) &&
5463             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
5464                 DRM_DEBUG_KMS("Planar YUV: src dimensions not met\n");
5465                 return -EINVAL;
5466         }
5467
5468         /* range checks */
5469         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
5470             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
5471             (INTEL_GEN(dev_priv) >= 11 &&
5472              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
5473               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
5474             (INTEL_GEN(dev_priv) < 11 &&
5475              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
5476               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
5477                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
5478                         "size is out of scaler range\n",
5479                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
5480                 return -EINVAL;
5481         }
5482
5483         /* mark this plane as a scaler user in crtc_state */
5484         scaler_state->scaler_users |= (1 << scaler_user);
5485         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5486                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
5487                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
5488                 scaler_state->scaler_users);
5489
5490         return 0;
5491 }
5492
5493 /**
5494  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
5495  *
5496  * @state: crtc's scaler state
5497  *
5498  * Return
5499  *     0 - scaler_usage updated successfully
5500  *    error - requested scaling cannot be supported or other error condition
5501  */
5502 int skl_update_scaler_crtc(struct intel_crtc_state *state)
5503 {
5504         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
5505         bool need_scaler = false;
5506
5507         if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
5508                 need_scaler = true;
5509
5510         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
5511                                  &state->scaler_state.scaler_id,
5512                                  state->pipe_src_w, state->pipe_src_h,
5513                                  adjusted_mode->crtc_hdisplay,
5514                                  adjusted_mode->crtc_vdisplay, NULL, need_scaler);
5515 }
5516
5517 /**
5518  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
5519  * @crtc_state: crtc's scaler state
5520  * @plane_state: atomic plane state to update
5521  *
5522  * Return
5523  *     0 - scaler_usage updated successfully
5524  *    error - requested scaling cannot be supported or other error condition
5525  */
5526 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
5527                                    struct intel_plane_state *plane_state)
5528 {
5529         struct intel_plane *intel_plane =
5530                 to_intel_plane(plane_state->base.plane);
5531         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
5532         struct drm_framebuffer *fb = plane_state->base.fb;
5533         int ret;
5534         bool force_detach = !fb || !plane_state->base.visible;
5535         bool need_scaler = false;
5536
5537         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
5538         if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
5539             fb && is_planar_yuv_format(fb->format->format))
5540                 need_scaler = true;
5541
5542         ret = skl_update_scaler(crtc_state, force_detach,
5543                                 drm_plane_index(&intel_plane->base),
5544                                 &plane_state->scaler_id,
5545                                 drm_rect_width(&plane_state->base.src) >> 16,
5546                                 drm_rect_height(&plane_state->base.src) >> 16,
5547                                 drm_rect_width(&plane_state->base.dst),
5548                                 drm_rect_height(&plane_state->base.dst),
5549                                 fb ? fb->format : NULL, need_scaler);
5550
5551         if (ret || plane_state->scaler_id < 0)
5552                 return ret;
5553
5554         /* check colorkey */
5555         if (plane_state->ckey.flags) {
5556                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5557                               intel_plane->base.base.id,
5558                               intel_plane->base.name);
5559                 return -EINVAL;
5560         }
5561
5562         /* Check src format */
5563         switch (fb->format->format) {
5564         case DRM_FORMAT_RGB565:
5565         case DRM_FORMAT_XBGR8888:
5566         case DRM_FORMAT_XRGB8888:
5567         case DRM_FORMAT_ABGR8888:
5568         case DRM_FORMAT_ARGB8888:
5569         case DRM_FORMAT_XRGB2101010:
5570         case DRM_FORMAT_XBGR2101010:
5571         case DRM_FORMAT_XBGR16161616F:
5572         case DRM_FORMAT_ABGR16161616F:
5573         case DRM_FORMAT_XRGB16161616F:
5574         case DRM_FORMAT_ARGB16161616F:
5575         case DRM_FORMAT_YUYV:
5576         case DRM_FORMAT_YVYU:
5577         case DRM_FORMAT_UYVY:
5578         case DRM_FORMAT_VYUY:
5579         case DRM_FORMAT_NV12:
5580         case DRM_FORMAT_P010:
5581         case DRM_FORMAT_P012:
5582         case DRM_FORMAT_P016:
5583         case DRM_FORMAT_Y210:
5584         case DRM_FORMAT_Y212:
5585         case DRM_FORMAT_Y216:
5586         case DRM_FORMAT_XVYU2101010:
5587         case DRM_FORMAT_XVYU12_16161616:
5588         case DRM_FORMAT_XVYU16161616:
5589                 break;
5590         default:
5591                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5592                               intel_plane->base.base.id, intel_plane->base.name,
5593                               fb->base.id, fb->format->format);
5594                 return -EINVAL;
5595         }
5596
5597         return 0;
5598 }
5599
5600 static void skylake_scaler_disable(struct intel_crtc *crtc)
5601 {
5602         int i;
5603
5604         for (i = 0; i < crtc->num_scalers; i++)
5605                 skl_detach_scaler(crtc, i);
5606 }
5607
5608 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state)
5609 {
5610         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5611         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5612         enum pipe pipe = crtc->pipe;
5613         const struct intel_crtc_scaler_state *scaler_state =
5614                 &crtc_state->scaler_state;
5615
5616         if (crtc_state->pch_pfit.enabled) {
5617                 u16 uv_rgb_hphase, uv_rgb_vphase;
5618                 int pfit_w, pfit_h, hscale, vscale;
5619                 int id;
5620
5621                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
5622                         return;
5623
5624                 pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF;
5625                 pfit_h = crtc_state->pch_pfit.size & 0xFFFF;
5626
5627                 hscale = (crtc_state->pipe_src_w << 16) / pfit_w;
5628                 vscale = (crtc_state->pipe_src_h << 16) / pfit_h;
5629
5630                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
5631                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
5632
5633                 id = scaler_state->scaler_id;
5634                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
5635                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
5636                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
5637                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
5638                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
5639                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
5640                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
5641                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
5642         }
5643 }
5644
5645 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state)
5646 {
5647         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5648         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5649         int pipe = crtc->pipe;
5650
5651         if (crtc_state->pch_pfit.enabled) {
5652                 /* Force use of hard-coded filter coefficients
5653                  * as some pre-programmed values are broken,
5654                  * e.g. x201.
5655                  */
5656                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
5657                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
5658                                                  PF_PIPE_SEL_IVB(pipe));
5659                 else
5660                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
5661                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
5662                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
5663         }
5664 }
5665
5666 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
5667 {
5668         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5669         struct drm_device *dev = crtc->base.dev;
5670         struct drm_i915_private *dev_priv = to_i915(dev);
5671
5672         if (!crtc_state->ips_enabled)
5673                 return;
5674
5675         /*
5676          * We can only enable IPS after we enable a plane and wait for a vblank
5677          * This function is called from post_plane_update, which is run after
5678          * a vblank wait.
5679          */
5680         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
5681
5682         if (IS_BROADWELL(dev_priv)) {
5683                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
5684                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
5685                 /* Quoting Art Runyan: "its not safe to expect any particular
5686                  * value in IPS_CTL bit 31 after enabling IPS through the
5687                  * mailbox." Moreover, the mailbox may return a bogus state,
5688                  * so we need to just enable it and continue on.
5689                  */
5690         } else {
5691                 I915_WRITE(IPS_CTL, IPS_ENABLE);
5692                 /* The bit only becomes 1 in the next vblank, so this wait here
5693                  * is essentially intel_wait_for_vblank. If we don't have this
5694                  * and don't wait for vblanks until the end of crtc_enable, then
5695                  * the HW state readout code will complain that the expected
5696                  * IPS_CTL value is not the one we read. */
5697                 if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50))
5698                         DRM_ERROR("Timed out waiting for IPS enable\n");
5699         }
5700 }
5701
5702 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
5703 {
5704         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5705         struct drm_device *dev = crtc->base.dev;
5706         struct drm_i915_private *dev_priv = to_i915(dev);
5707
5708         if (!crtc_state->ips_enabled)
5709                 return;
5710
5711         if (IS_BROADWELL(dev_priv)) {
5712                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
5713                 /*
5714                  * Wait for PCODE to finish disabling IPS. The BSpec specified
5715                  * 42ms timeout value leads to occasional timeouts so use 100ms
5716                  * instead.
5717                  */
5718                 if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100))
5719                         DRM_ERROR("Timed out waiting for IPS disable\n");
5720         } else {
5721                 I915_WRITE(IPS_CTL, 0);
5722                 POSTING_READ(IPS_CTL);
5723         }
5724
5725         /* We need to wait for a vblank before we can disable the plane. */
5726         intel_wait_for_vblank(dev_priv, crtc->pipe);
5727 }
5728
5729 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
5730 {
5731         if (intel_crtc->overlay) {
5732                 struct drm_device *dev = intel_crtc->base.dev;
5733
5734                 mutex_lock(&dev->struct_mutex);
5735                 (void) intel_overlay_switch_off(intel_crtc->overlay);
5736                 mutex_unlock(&dev->struct_mutex);
5737         }
5738
5739         /* Let userspace switch the overlay on again. In most cases userspace
5740          * has to recompute where to put it anyway.
5741          */
5742 }
5743
5744 /**
5745  * intel_post_enable_primary - Perform operations after enabling primary plane
5746  * @crtc: the CRTC whose primary plane was just enabled
5747  * @new_crtc_state: the enabling state
5748  *
5749  * Performs potentially sleeping operations that must be done after the primary
5750  * plane is enabled, such as updating FBC and IPS.  Note that this may be
5751  * called due to an explicit primary plane update, or due to an implicit
5752  * re-enable that is caused when a sprite plane is updated to no longer
5753  * completely hide the primary plane.
5754  */
5755 static void
5756 intel_post_enable_primary(struct drm_crtc *crtc,
5757                           const struct intel_crtc_state *new_crtc_state)
5758 {
5759         struct drm_device *dev = crtc->dev;
5760         struct drm_i915_private *dev_priv = to_i915(dev);
5761         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5762         int pipe = intel_crtc->pipe;
5763
5764         /*
5765          * Gen2 reports pipe underruns whenever all planes are disabled.
5766          * So don't enable underrun reporting before at least some planes
5767          * are enabled.
5768          * FIXME: Need to fix the logic to work when we turn off all planes
5769          * but leave the pipe running.
5770          */
5771         if (IS_GEN(dev_priv, 2))
5772                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5773
5774         /* Underruns don't always raise interrupts, so check manually. */
5775         intel_check_cpu_fifo_underruns(dev_priv);
5776         intel_check_pch_fifo_underruns(dev_priv);
5777 }
5778
5779 /* FIXME get rid of this and use pre_plane_update */
5780 static void
5781 intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
5782 {
5783         struct drm_device *dev = crtc->dev;
5784         struct drm_i915_private *dev_priv = to_i915(dev);
5785         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5786         int pipe = intel_crtc->pipe;
5787
5788         /*
5789          * Gen2 reports pipe underruns whenever all planes are disabled.
5790          * So disable underrun reporting before all the planes get disabled.
5791          */
5792         if (IS_GEN(dev_priv, 2))
5793                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5794
5795         hsw_disable_ips(to_intel_crtc_state(crtc->state));
5796
5797         /*
5798          * Vblank time updates from the shadow to live plane control register
5799          * are blocked if the memory self-refresh mode is active at that
5800          * moment. So to make sure the plane gets truly disabled, disable
5801          * first the self-refresh mode. The self-refresh enable bit in turn
5802          * will be checked/applied by the HW only at the next frame start
5803          * event which is after the vblank start event, so we need to have a
5804          * wait-for-vblank between disabling the plane and the pipe.
5805          */
5806         if (HAS_GMCH(dev_priv) &&
5807             intel_set_memory_cxsr(dev_priv, false))
5808                 intel_wait_for_vblank(dev_priv, pipe);
5809 }
5810
5811 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
5812                                        const struct intel_crtc_state *new_crtc_state)
5813 {
5814         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5815         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5816
5817         if (!old_crtc_state->ips_enabled)
5818                 return false;
5819
5820         if (needs_modeset(new_crtc_state))
5821                 return true;
5822
5823         /*
5824          * Workaround : Do not read or write the pipe palette/gamma data while
5825          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
5826          *
5827          * Disable IPS before we program the LUT.
5828          */
5829         if (IS_HASWELL(dev_priv) &&
5830             (new_crtc_state->base.color_mgmt_changed ||
5831              new_crtc_state->update_pipe) &&
5832             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
5833                 return true;
5834
5835         return !new_crtc_state->ips_enabled;
5836 }
5837
5838 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
5839                                        const struct intel_crtc_state *new_crtc_state)
5840 {
5841         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5842         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5843
5844         if (!new_crtc_state->ips_enabled)
5845                 return false;
5846
5847         if (needs_modeset(new_crtc_state))
5848                 return true;
5849
5850         /*
5851          * Workaround : Do not read or write the pipe palette/gamma data while
5852          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
5853          *
5854          * Re-enable IPS after the LUT has been programmed.
5855          */
5856         if (IS_HASWELL(dev_priv) &&
5857             (new_crtc_state->base.color_mgmt_changed ||
5858              new_crtc_state->update_pipe) &&
5859             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
5860                 return true;
5861
5862         /*
5863          * We can't read out IPS on broadwell, assume the worst and
5864          * forcibly enable IPS on the first fastset.
5865          */
5866         if (new_crtc_state->update_pipe &&
5867             old_crtc_state->base.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
5868                 return true;
5869
5870         return !old_crtc_state->ips_enabled;
5871 }
5872
5873 static bool needs_nv12_wa(struct drm_i915_private *dev_priv,
5874                           const struct intel_crtc_state *crtc_state)
5875 {
5876         if (!crtc_state->nv12_planes)
5877                 return false;
5878
5879         /* WA Display #0827: Gen9:all */
5880         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
5881                 return true;
5882
5883         return false;
5884 }
5885
5886 static bool needs_scalerclk_wa(struct drm_i915_private *dev_priv,
5887                                const struct intel_crtc_state *crtc_state)
5888 {
5889         /* Wa_2006604312:icl */
5890         if (crtc_state->scaler_state.scaler_users > 0 && IS_ICELAKE(dev_priv))
5891                 return true;
5892
5893         return false;
5894 }
5895
5896 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
5897 {
5898         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5899         struct drm_device *dev = crtc->base.dev;
5900         struct drm_i915_private *dev_priv = to_i915(dev);
5901         struct drm_atomic_state *state = old_crtc_state->base.state;
5902         struct intel_crtc_state *pipe_config =
5903                 intel_atomic_get_new_crtc_state(to_intel_atomic_state(state),
5904                                                 crtc);
5905         struct drm_plane *primary = crtc->base.primary;
5906         struct drm_plane_state *old_primary_state =
5907                 drm_atomic_get_old_plane_state(state, primary);
5908
5909         intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits);
5910
5911         if (pipe_config->update_wm_post && pipe_config->base.active)
5912                 intel_update_watermarks(crtc);
5913
5914         if (hsw_post_update_enable_ips(old_crtc_state, pipe_config))
5915                 hsw_enable_ips(pipe_config);
5916
5917         if (old_primary_state) {
5918                 struct drm_plane_state *new_primary_state =
5919                         drm_atomic_get_new_plane_state(state, primary);
5920
5921                 intel_fbc_post_update(crtc);
5922
5923                 if (new_primary_state->visible &&
5924                     (needs_modeset(pipe_config) ||
5925                      !old_primary_state->visible))
5926                         intel_post_enable_primary(&crtc->base, pipe_config);
5927         }
5928
5929         if (needs_nv12_wa(dev_priv, old_crtc_state) &&
5930             !needs_nv12_wa(dev_priv, pipe_config))
5931                 skl_wa_827(dev_priv, crtc->pipe, false);
5932
5933         if (needs_scalerclk_wa(dev_priv, old_crtc_state) &&
5934             !needs_scalerclk_wa(dev_priv, pipe_config))
5935                 icl_wa_scalerclkgating(dev_priv, crtc->pipe, false);
5936 }
5937
5938 static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
5939                                    struct intel_crtc_state *pipe_config)
5940 {
5941         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5942         struct drm_device *dev = crtc->base.dev;
5943         struct drm_i915_private *dev_priv = to_i915(dev);
5944         struct drm_atomic_state *state = old_crtc_state->base.state;
5945         struct drm_plane *primary = crtc->base.primary;
5946         struct drm_plane_state *old_primary_state =
5947                 drm_atomic_get_old_plane_state(state, primary);
5948         bool modeset = needs_modeset(pipe_config);
5949         struct intel_atomic_state *intel_state =
5950                 to_intel_atomic_state(state);
5951
5952         if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config))
5953                 hsw_disable_ips(old_crtc_state);
5954
5955         if (old_primary_state) {
5956                 struct intel_plane_state *new_primary_state =
5957                         intel_atomic_get_new_plane_state(intel_state,
5958                                                          to_intel_plane(primary));
5959
5960                 intel_fbc_pre_update(crtc, pipe_config, new_primary_state);
5961                 /*
5962                  * Gen2 reports pipe underruns whenever all planes are disabled.
5963                  * So disable underrun reporting before all the planes get disabled.
5964                  */
5965                 if (IS_GEN(dev_priv, 2) && old_primary_state->visible &&
5966                     (modeset || !new_primary_state->base.visible))
5967                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
5968         }
5969
5970         /* Display WA 827 */
5971         if (!needs_nv12_wa(dev_priv, old_crtc_state) &&
5972             needs_nv12_wa(dev_priv, pipe_config))
5973                 skl_wa_827(dev_priv, crtc->pipe, true);
5974
5975         /* Wa_2006604312:icl */
5976         if (!needs_scalerclk_wa(dev_priv, old_crtc_state) &&
5977             needs_scalerclk_wa(dev_priv, pipe_config))
5978                 icl_wa_scalerclkgating(dev_priv, crtc->pipe, true);
5979
5980         /*
5981          * Vblank time updates from the shadow to live plane control register
5982          * are blocked if the memory self-refresh mode is active at that
5983          * moment. So to make sure the plane gets truly disabled, disable
5984          * first the self-refresh mode. The self-refresh enable bit in turn
5985          * will be checked/applied by the HW only at the next frame start
5986          * event which is after the vblank start event, so we need to have a
5987          * wait-for-vblank between disabling the plane and the pipe.
5988          */
5989         if (HAS_GMCH(dev_priv) && old_crtc_state->base.active &&
5990             pipe_config->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
5991                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5992
5993         /*
5994          * IVB workaround: must disable low power watermarks for at least
5995          * one frame before enabling scaling.  LP watermarks can be re-enabled
5996          * when scaling is disabled.
5997          *
5998          * WaCxSRDisabledForSpriteScaling:ivb
5999          */
6000         if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev) &&
6001             old_crtc_state->base.active)
6002                 intel_wait_for_vblank(dev_priv, crtc->pipe);
6003
6004         /*
6005          * If we're doing a modeset, we're done.  No need to do any pre-vblank
6006          * watermark programming here.
6007          */
6008         if (needs_modeset(pipe_config))
6009                 return;
6010
6011         /*
6012          * For platforms that support atomic watermarks, program the
6013          * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
6014          * will be the intermediate values that are safe for both pre- and
6015          * post- vblank; when vblank happens, the 'active' values will be set
6016          * to the final 'target' values and we'll do this again to get the
6017          * optimal watermarks.  For gen9+ platforms, the values we program here
6018          * will be the final target values which will get automatically latched
6019          * at vblank time; no further programming will be necessary.
6020          *
6021          * If a platform hasn't been transitioned to atomic watermarks yet,
6022          * we'll continue to update watermarks the old way, if flags tell
6023          * us to.
6024          */
6025         if (dev_priv->display.initial_watermarks != NULL)
6026                 dev_priv->display.initial_watermarks(intel_state,
6027                                                      pipe_config);
6028         else if (pipe_config->update_wm_pre)
6029                 intel_update_watermarks(crtc);
6030 }
6031
6032 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
6033                                       struct intel_crtc *crtc)
6034 {
6035         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6036         const struct intel_crtc_state *new_crtc_state =
6037                 intel_atomic_get_new_crtc_state(state, crtc);
6038         unsigned int update_mask = new_crtc_state->update_planes;
6039         const struct intel_plane_state *old_plane_state;
6040         struct intel_plane *plane;
6041         unsigned fb_bits = 0;
6042         int i;
6043
6044         intel_crtc_dpms_overlay_disable(crtc);
6045
6046         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
6047                 if (crtc->pipe != plane->pipe ||
6048                     !(update_mask & BIT(plane->id)))
6049                         continue;
6050
6051                 intel_disable_plane(plane, new_crtc_state);
6052
6053                 if (old_plane_state->base.visible)
6054                         fb_bits |= plane->frontbuffer_bit;
6055         }
6056
6057         intel_frontbuffer_flip(dev_priv, fb_bits);
6058 }
6059
6060 /*
6061  * intel_connector_primary_encoder - get the primary encoder for a connector
6062  * @connector: connector for which to return the encoder
6063  *
6064  * Returns the primary encoder for a connector. There is a 1:1 mapping from
6065  * all connectors to their encoder, except for DP-MST connectors which have
6066  * both a virtual and a primary encoder. These DP-MST primary encoders can be
6067  * pointed to by as many DP-MST connectors as there are pipes.
6068  */
6069 static struct intel_encoder *
6070 intel_connector_primary_encoder(struct intel_connector *connector)
6071 {
6072         struct intel_encoder *encoder;
6073
6074         if (connector->mst_port)
6075                 return &dp_to_dig_port(connector->mst_port)->base;
6076
6077         encoder = intel_attached_encoder(&connector->base);
6078         WARN_ON(!encoder);
6079
6080         return encoder;
6081 }
6082
6083 static bool
6084 intel_connector_needs_modeset(struct intel_atomic_state *state,
6085                               const struct drm_connector_state *old_conn_state,
6086                               const struct drm_connector_state *new_conn_state)
6087 {
6088         struct intel_crtc *old_crtc = old_conn_state->crtc ?
6089                                       to_intel_crtc(old_conn_state->crtc) : NULL;
6090         struct intel_crtc *new_crtc = new_conn_state->crtc ?
6091                                       to_intel_crtc(new_conn_state->crtc) : NULL;
6092
6093         return new_crtc != old_crtc ||
6094                (new_crtc &&
6095                 needs_modeset(intel_atomic_get_new_crtc_state(state, new_crtc)));
6096 }
6097
6098 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
6099 {
6100         struct drm_connector_state *old_conn_state;
6101         struct drm_connector_state *new_conn_state;
6102         struct drm_connector *conn;
6103         int i;
6104
6105         for_each_oldnew_connector_in_state(&state->base, conn,
6106                                            old_conn_state, new_conn_state, i) {
6107                 struct intel_encoder *encoder;
6108                 struct intel_crtc *crtc;
6109
6110                 if (!intel_connector_needs_modeset(state,
6111                                                    old_conn_state,
6112                                                    new_conn_state))
6113                         continue;
6114
6115                 encoder = intel_connector_primary_encoder(to_intel_connector(conn));
6116                 if (!encoder->update_prepare)
6117                         continue;
6118
6119                 crtc = new_conn_state->crtc ?
6120                         to_intel_crtc(new_conn_state->crtc) : NULL;
6121                 encoder->update_prepare(state, encoder, crtc);
6122         }
6123 }
6124
6125 static void intel_encoders_update_complete(struct intel_atomic_state *state)
6126 {
6127         struct drm_connector_state *old_conn_state;
6128         struct drm_connector_state *new_conn_state;
6129         struct drm_connector *conn;
6130         int i;
6131
6132         for_each_oldnew_connector_in_state(&state->base, conn,
6133                                            old_conn_state, new_conn_state, i) {
6134                 struct intel_encoder *encoder;
6135                 struct intel_crtc *crtc;
6136
6137                 if (!intel_connector_needs_modeset(state,
6138                                                    old_conn_state,
6139                                                    new_conn_state))
6140                         continue;
6141
6142                 encoder = intel_connector_primary_encoder(to_intel_connector(conn));
6143                 if (!encoder->update_complete)
6144                         continue;
6145
6146                 crtc = new_conn_state->crtc ?
6147                         to_intel_crtc(new_conn_state->crtc) : NULL;
6148                 encoder->update_complete(state, encoder, crtc);
6149         }
6150 }
6151
6152 static void intel_encoders_pre_pll_enable(struct intel_crtc *crtc,
6153                                           struct intel_crtc_state *crtc_state,
6154                                           struct intel_atomic_state *state)
6155 {
6156         struct drm_connector_state *conn_state;
6157         struct drm_connector *conn;
6158         int i;
6159
6160         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6161                 struct intel_encoder *encoder =
6162                         to_intel_encoder(conn_state->best_encoder);
6163
6164                 if (conn_state->crtc != &crtc->base)
6165                         continue;
6166
6167                 if (encoder->pre_pll_enable)
6168                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
6169         }
6170 }
6171
6172 static void intel_encoders_pre_enable(struct intel_crtc *crtc,
6173                                       struct intel_crtc_state *crtc_state,
6174                                       struct intel_atomic_state *state)
6175 {
6176         struct drm_connector_state *conn_state;
6177         struct drm_connector *conn;
6178         int i;
6179
6180         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6181                 struct intel_encoder *encoder =
6182                         to_intel_encoder(conn_state->best_encoder);
6183
6184                 if (conn_state->crtc != &crtc->base)
6185                         continue;
6186
6187                 if (encoder->pre_enable)
6188                         encoder->pre_enable(encoder, crtc_state, conn_state);
6189         }
6190 }
6191
6192 static void intel_encoders_enable(struct intel_crtc *crtc,
6193                                   struct intel_crtc_state *crtc_state,
6194                                   struct intel_atomic_state *state)
6195 {
6196         struct drm_connector_state *conn_state;
6197         struct drm_connector *conn;
6198         int i;
6199
6200         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6201                 struct intel_encoder *encoder =
6202                         to_intel_encoder(conn_state->best_encoder);
6203
6204                 if (conn_state->crtc != &crtc->base)
6205                         continue;
6206
6207                 if (encoder->enable)
6208                         encoder->enable(encoder, crtc_state, conn_state);
6209                 intel_opregion_notify_encoder(encoder, true);
6210         }
6211 }
6212
6213 static void intel_encoders_disable(struct intel_crtc *crtc,
6214                                    struct intel_crtc_state *old_crtc_state,
6215                                    struct intel_atomic_state *state)
6216 {
6217         struct drm_connector_state *old_conn_state;
6218         struct drm_connector *conn;
6219         int i;
6220
6221         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6222                 struct intel_encoder *encoder =
6223                         to_intel_encoder(old_conn_state->best_encoder);
6224
6225                 if (old_conn_state->crtc != &crtc->base)
6226                         continue;
6227
6228                 intel_opregion_notify_encoder(encoder, false);
6229                 if (encoder->disable)
6230                         encoder->disable(encoder, old_crtc_state, old_conn_state);
6231         }
6232 }
6233
6234 static void intel_encoders_post_disable(struct intel_crtc *crtc,
6235                                         struct intel_crtc_state *old_crtc_state,
6236                                         struct intel_atomic_state *state)
6237 {
6238         struct drm_connector_state *old_conn_state;
6239         struct drm_connector *conn;
6240         int i;
6241
6242         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6243                 struct intel_encoder *encoder =
6244                         to_intel_encoder(old_conn_state->best_encoder);
6245
6246                 if (old_conn_state->crtc != &crtc->base)
6247                         continue;
6248
6249                 if (encoder->post_disable)
6250                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
6251         }
6252 }
6253
6254 static void intel_encoders_post_pll_disable(struct intel_crtc *crtc,
6255                                             struct intel_crtc_state *old_crtc_state,
6256                                             struct intel_atomic_state *state)
6257 {
6258         struct drm_connector_state *old_conn_state;
6259         struct drm_connector *conn;
6260         int i;
6261
6262         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6263                 struct intel_encoder *encoder =
6264                         to_intel_encoder(old_conn_state->best_encoder);
6265
6266                 if (old_conn_state->crtc != &crtc->base)
6267                         continue;
6268
6269                 if (encoder->post_pll_disable)
6270                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
6271         }
6272 }
6273
6274 static void intel_encoders_update_pipe(struct intel_crtc *crtc,
6275                                        struct intel_crtc_state *crtc_state,
6276                                        struct intel_atomic_state *state)
6277 {
6278         struct drm_connector_state *conn_state;
6279         struct drm_connector *conn;
6280         int i;
6281
6282         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6283                 struct intel_encoder *encoder =
6284                         to_intel_encoder(conn_state->best_encoder);
6285
6286                 if (conn_state->crtc != &crtc->base)
6287                         continue;
6288
6289                 if (encoder->update_pipe)
6290                         encoder->update_pipe(encoder, crtc_state, conn_state);
6291         }
6292 }
6293
6294 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
6295 {
6296         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6297         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6298
6299         plane->disable_plane(plane, crtc_state);
6300 }
6301
6302 static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
6303                                  struct intel_atomic_state *state)
6304 {
6305         struct drm_crtc *crtc = pipe_config->base.crtc;
6306         struct drm_device *dev = crtc->dev;
6307         struct drm_i915_private *dev_priv = to_i915(dev);
6308         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6309         int pipe = intel_crtc->pipe;
6310
6311         if (WARN_ON(intel_crtc->active))
6312                 return;
6313
6314         /*
6315          * Sometimes spurious CPU pipe underruns happen during FDI
6316          * training, at least with VGA+HDMI cloning. Suppress them.
6317          *
6318          * On ILK we get an occasional spurious CPU pipe underruns
6319          * between eDP port A enable and vdd enable. Also PCH port
6320          * enable seems to result in the occasional CPU pipe underrun.
6321          *
6322          * Spurious PCH underruns also occur during PCH enabling.
6323          */
6324         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6325         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6326
6327         if (pipe_config->has_pch_encoder)
6328                 intel_prepare_shared_dpll(pipe_config);
6329
6330         if (intel_crtc_has_dp_encoder(pipe_config))
6331                 intel_dp_set_m_n(pipe_config, M1_N1);
6332
6333         intel_set_pipe_timings(pipe_config);
6334         intel_set_pipe_src_size(pipe_config);
6335
6336         if (pipe_config->has_pch_encoder) {
6337                 intel_cpu_transcoder_set_m_n(pipe_config,
6338                                              &pipe_config->fdi_m_n, NULL);
6339         }
6340
6341         ironlake_set_pipeconf(pipe_config);
6342
6343         intel_crtc->active = true;
6344
6345         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6346
6347         if (pipe_config->has_pch_encoder) {
6348                 /* Note: FDI PLL enabling _must_ be done before we enable the
6349                  * cpu pipes, hence this is separate from all the other fdi/pch
6350                  * enabling. */
6351                 ironlake_fdi_pll_enable(pipe_config);
6352         } else {
6353                 assert_fdi_tx_disabled(dev_priv, pipe);
6354                 assert_fdi_rx_disabled(dev_priv, pipe);
6355         }
6356
6357         ironlake_pfit_enable(pipe_config);
6358
6359         /*
6360          * On ILK+ LUT must be loaded before the pipe is running but with
6361          * clocks enabled
6362          */
6363         intel_color_load_luts(pipe_config);
6364         intel_color_commit(pipe_config);
6365         /* update DSPCNTR to configure gamma for pipe bottom color */
6366         intel_disable_primary_plane(pipe_config);
6367
6368         if (dev_priv->display.initial_watermarks != NULL)
6369                 dev_priv->display.initial_watermarks(state, pipe_config);
6370         intel_enable_pipe(pipe_config);
6371
6372         if (pipe_config->has_pch_encoder)
6373                 ironlake_pch_enable(state, pipe_config);
6374
6375         assert_vblank_disabled(crtc);
6376         intel_crtc_vblank_on(pipe_config);
6377
6378         intel_encoders_enable(intel_crtc, pipe_config, state);
6379
6380         if (HAS_PCH_CPT(dev_priv))
6381                 cpt_verify_modeset(dev, intel_crtc->pipe);
6382
6383         /*
6384          * Must wait for vblank to avoid spurious PCH FIFO underruns.
6385          * And a second vblank wait is needed at least on ILK with
6386          * some interlaced HDMI modes. Let's do the double wait always
6387          * in case there are more corner cases we don't know about.
6388          */
6389         if (pipe_config->has_pch_encoder) {
6390                 intel_wait_for_vblank(dev_priv, pipe);
6391                 intel_wait_for_vblank(dev_priv, pipe);
6392         }
6393         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6394         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6395 }
6396
6397 /* IPS only exists on ULT machines and is tied to pipe A. */
6398 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
6399 {
6400         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
6401 }
6402
6403 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
6404                                             enum pipe pipe, bool apply)
6405 {
6406         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
6407         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
6408
6409         if (apply)
6410                 val |= mask;
6411         else
6412                 val &= ~mask;
6413
6414         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
6415 }
6416
6417 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
6418 {
6419         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6420         enum pipe pipe = crtc->pipe;
6421         u32 val;
6422
6423         val = MBUS_DBOX_A_CREDIT(2);
6424
6425         if (INTEL_GEN(dev_priv) >= 12) {
6426                 val |= MBUS_DBOX_BW_CREDIT(2);
6427                 val |= MBUS_DBOX_B_CREDIT(12);
6428         } else {
6429                 val |= MBUS_DBOX_BW_CREDIT(1);
6430                 val |= MBUS_DBOX_B_CREDIT(8);
6431         }
6432
6433         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
6434 }
6435
6436 static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
6437                                 struct intel_atomic_state *state)
6438 {
6439         struct drm_crtc *crtc = pipe_config->base.crtc;
6440         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6441         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6442         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
6443         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
6444         bool psl_clkgate_wa;
6445
6446         if (WARN_ON(intel_crtc->active))
6447                 return;
6448
6449         intel_encoders_pre_pll_enable(intel_crtc, pipe_config, state);
6450
6451         if (pipe_config->shared_dpll)
6452                 intel_enable_shared_dpll(pipe_config);
6453
6454         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6455
6456         if (intel_crtc_has_dp_encoder(pipe_config))
6457                 intel_dp_set_m_n(pipe_config, M1_N1);
6458
6459         if (!transcoder_is_dsi(cpu_transcoder))
6460                 intel_set_pipe_timings(pipe_config);
6461
6462         intel_set_pipe_src_size(pipe_config);
6463
6464         if (cpu_transcoder != TRANSCODER_EDP &&
6465             !transcoder_is_dsi(cpu_transcoder)) {
6466                 I915_WRITE(PIPE_MULT(cpu_transcoder),
6467                            pipe_config->pixel_multiplier - 1);
6468         }
6469
6470         if (pipe_config->has_pch_encoder) {
6471                 intel_cpu_transcoder_set_m_n(pipe_config,
6472                                              &pipe_config->fdi_m_n, NULL);
6473         }
6474
6475         if (!transcoder_is_dsi(cpu_transcoder))
6476                 haswell_set_pipeconf(pipe_config);
6477
6478         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
6479                 bdw_set_pipemisc(pipe_config);
6480
6481         intel_crtc->active = true;
6482
6483         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
6484         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
6485                          pipe_config->pch_pfit.enabled;
6486         if (psl_clkgate_wa)
6487                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
6488
6489         if (INTEL_GEN(dev_priv) >= 9)
6490                 skylake_pfit_enable(pipe_config);
6491         else
6492                 ironlake_pfit_enable(pipe_config);
6493
6494         /*
6495          * On ILK+ LUT must be loaded before the pipe is running but with
6496          * clocks enabled
6497          */
6498         intel_color_load_luts(pipe_config);
6499         intel_color_commit(pipe_config);
6500         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
6501         if (INTEL_GEN(dev_priv) < 9)
6502                 intel_disable_primary_plane(pipe_config);
6503
6504         if (INTEL_GEN(dev_priv) >= 11)
6505                 icl_set_pipe_chicken(intel_crtc);
6506
6507         intel_ddi_set_pipe_settings(pipe_config);
6508         if (!transcoder_is_dsi(cpu_transcoder))
6509                 intel_ddi_enable_transcoder_func(pipe_config);
6510
6511         if (dev_priv->display.initial_watermarks != NULL)
6512                 dev_priv->display.initial_watermarks(state, pipe_config);
6513
6514         if (INTEL_GEN(dev_priv) >= 11)
6515                 icl_pipe_mbus_enable(intel_crtc);
6516
6517         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6518         if (!transcoder_is_dsi(cpu_transcoder))
6519                 intel_enable_pipe(pipe_config);
6520
6521         if (pipe_config->has_pch_encoder)
6522                 lpt_pch_enable(state, pipe_config);
6523
6524         if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DP_MST))
6525                 intel_ddi_set_vc_payload_alloc(pipe_config, true);
6526
6527         assert_vblank_disabled(crtc);
6528         intel_crtc_vblank_on(pipe_config);
6529
6530         intel_encoders_enable(intel_crtc, pipe_config, state);
6531
6532         if (psl_clkgate_wa) {
6533                 intel_wait_for_vblank(dev_priv, pipe);
6534                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
6535         }
6536
6537         /* If we change the relative order between pipe/planes enabling, we need
6538          * to change the workaround. */
6539         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
6540         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
6541                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6542                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6543         }
6544 }
6545
6546 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6547 {
6548         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6549         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6550         enum pipe pipe = crtc->pipe;
6551
6552         /* To avoid upsetting the power well on haswell only disable the pfit if
6553          * it's in use. The hw state code will make sure we get this right. */
6554         if (old_crtc_state->pch_pfit.enabled) {
6555                 I915_WRITE(PF_CTL(pipe), 0);
6556                 I915_WRITE(PF_WIN_POS(pipe), 0);
6557                 I915_WRITE(PF_WIN_SZ(pipe), 0);
6558         }
6559 }
6560
6561 static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
6562                                   struct intel_atomic_state *state)
6563 {
6564         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6565         struct drm_device *dev = crtc->dev;
6566         struct drm_i915_private *dev_priv = to_i915(dev);
6567         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6568         int pipe = intel_crtc->pipe;
6569
6570         /*
6571          * Sometimes spurious CPU pipe underruns happen when the
6572          * pipe is already disabled, but FDI RX/TX is still enabled.
6573          * Happens at least with VGA+HDMI cloning. Suppress them.
6574          */
6575         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6576         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6577
6578         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6579
6580         drm_crtc_vblank_off(crtc);
6581         assert_vblank_disabled(crtc);
6582
6583         intel_disable_pipe(old_crtc_state);
6584
6585         ironlake_pfit_disable(old_crtc_state);
6586
6587         if (old_crtc_state->has_pch_encoder)
6588                 ironlake_fdi_disable(crtc);
6589
6590         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
6591
6592         if (old_crtc_state->has_pch_encoder) {
6593                 ironlake_disable_pch_transcoder(dev_priv, pipe);
6594
6595                 if (HAS_PCH_CPT(dev_priv)) {
6596                         i915_reg_t reg;
6597                         u32 temp;
6598
6599                         /* disable TRANS_DP_CTL */
6600                         reg = TRANS_DP_CTL(pipe);
6601                         temp = I915_READ(reg);
6602                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
6603                                   TRANS_DP_PORT_SEL_MASK);
6604                         temp |= TRANS_DP_PORT_SEL_NONE;
6605                         I915_WRITE(reg, temp);
6606
6607                         /* disable DPLL_SEL */
6608                         temp = I915_READ(PCH_DPLL_SEL);
6609                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
6610                         I915_WRITE(PCH_DPLL_SEL, temp);
6611                 }
6612
6613                 ironlake_fdi_pll_disable(intel_crtc);
6614         }
6615
6616         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6617         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6618 }
6619
6620 static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
6621                                  struct intel_atomic_state *state)
6622 {
6623         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6624         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6625         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6626         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
6627
6628         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6629
6630         drm_crtc_vblank_off(crtc);
6631         assert_vblank_disabled(crtc);
6632
6633         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6634         if (!transcoder_is_dsi(cpu_transcoder))
6635                 intel_disable_pipe(old_crtc_state);
6636
6637         if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
6638                 intel_ddi_set_vc_payload_alloc(old_crtc_state, false);
6639
6640         if (!transcoder_is_dsi(cpu_transcoder))
6641                 intel_ddi_disable_transcoder_func(old_crtc_state);
6642
6643         intel_dsc_disable(old_crtc_state);
6644
6645         if (INTEL_GEN(dev_priv) >= 9)
6646                 skylake_scaler_disable(intel_crtc);
6647         else
6648                 ironlake_pfit_disable(old_crtc_state);
6649
6650         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
6651
6652         intel_encoders_post_pll_disable(intel_crtc, old_crtc_state, state);
6653 }
6654
6655 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
6656 {
6657         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6658         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6659
6660         if (!crtc_state->gmch_pfit.control)
6661                 return;
6662
6663         /*
6664          * The panel fitter should only be adjusted whilst the pipe is disabled,
6665          * according to register description and PRM.
6666          */
6667         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
6668         assert_pipe_disabled(dev_priv, crtc->pipe);
6669
6670         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
6671         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
6672
6673         /* Border color in case we don't scale up to the full screen. Black by
6674          * default, change to something else for debugging. */
6675         I915_WRITE(BCLRPAT(crtc->pipe), 0);
6676 }
6677
6678 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
6679 {
6680         if (phy == PHY_NONE)
6681                 return false;
6682
6683         if (IS_ELKHARTLAKE(dev_priv))
6684                 return phy <= PHY_C;
6685
6686         if (INTEL_GEN(dev_priv) >= 11)
6687                 return phy <= PHY_B;
6688
6689         return false;
6690 }
6691
6692 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
6693 {
6694         if (INTEL_GEN(dev_priv) >= 12)
6695                 return phy >= PHY_D && phy <= PHY_I;
6696
6697         if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
6698                 return phy >= PHY_C && phy <= PHY_F;
6699
6700         return false;
6701 }
6702
6703 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
6704 {
6705         if (IS_ELKHARTLAKE(i915) && port == PORT_D)
6706                 return PHY_A;
6707
6708         return (enum phy)port;
6709 }
6710
6711 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
6712 {
6713         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
6714                 return PORT_TC_NONE;
6715
6716         if (INTEL_GEN(dev_priv) >= 12)
6717                 return port - PORT_D;
6718
6719         return port - PORT_C;
6720 }
6721
6722 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
6723 {
6724         switch (port) {
6725         case PORT_A:
6726                 return POWER_DOMAIN_PORT_DDI_A_LANES;
6727         case PORT_B:
6728                 return POWER_DOMAIN_PORT_DDI_B_LANES;
6729         case PORT_C:
6730                 return POWER_DOMAIN_PORT_DDI_C_LANES;
6731         case PORT_D:
6732                 return POWER_DOMAIN_PORT_DDI_D_LANES;
6733         case PORT_E:
6734                 return POWER_DOMAIN_PORT_DDI_E_LANES;
6735         case PORT_F:
6736                 return POWER_DOMAIN_PORT_DDI_F_LANES;
6737         default:
6738                 MISSING_CASE(port);
6739                 return POWER_DOMAIN_PORT_OTHER;
6740         }
6741 }
6742
6743 enum intel_display_power_domain
6744 intel_aux_power_domain(struct intel_digital_port *dig_port)
6745 {
6746         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
6747         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
6748
6749         if (intel_phy_is_tc(dev_priv, phy) &&
6750             dig_port->tc_mode == TC_PORT_TBT_ALT) {
6751                 switch (dig_port->aux_ch) {
6752                 case AUX_CH_C:
6753                         return POWER_DOMAIN_AUX_TBT1;
6754                 case AUX_CH_D:
6755                         return POWER_DOMAIN_AUX_TBT2;
6756                 case AUX_CH_E:
6757                         return POWER_DOMAIN_AUX_TBT3;
6758                 case AUX_CH_F:
6759                         return POWER_DOMAIN_AUX_TBT4;
6760                 default:
6761                         MISSING_CASE(dig_port->aux_ch);
6762                         return POWER_DOMAIN_AUX_TBT1;
6763                 }
6764         }
6765
6766         switch (dig_port->aux_ch) {
6767         case AUX_CH_A:
6768                 return POWER_DOMAIN_AUX_A;
6769         case AUX_CH_B:
6770                 return POWER_DOMAIN_AUX_B;
6771         case AUX_CH_C:
6772                 return POWER_DOMAIN_AUX_C;
6773         case AUX_CH_D:
6774                 return POWER_DOMAIN_AUX_D;
6775         case AUX_CH_E:
6776                 return POWER_DOMAIN_AUX_E;
6777         case AUX_CH_F:
6778                 return POWER_DOMAIN_AUX_F;
6779         default:
6780                 MISSING_CASE(dig_port->aux_ch);
6781                 return POWER_DOMAIN_AUX_A;
6782         }
6783 }
6784
6785 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
6786 {
6787         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6788         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6789         struct drm_encoder *encoder;
6790         enum pipe pipe = crtc->pipe;
6791         u64 mask;
6792         enum transcoder transcoder = crtc_state->cpu_transcoder;
6793
6794         if (!crtc_state->base.active)
6795                 return 0;
6796
6797         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
6798         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
6799         if (crtc_state->pch_pfit.enabled ||
6800             crtc_state->pch_pfit.force_thru)
6801                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
6802
6803         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
6804                                   crtc_state->base.encoder_mask) {
6805                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6806
6807                 mask |= BIT_ULL(intel_encoder->power_domain);
6808         }
6809
6810         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
6811                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
6812
6813         if (crtc_state->shared_dpll)
6814                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
6815
6816         return mask;
6817 }
6818
6819 static u64
6820 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
6821 {
6822         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6823         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6824         enum intel_display_power_domain domain;
6825         u64 domains, new_domains, old_domains;
6826
6827         old_domains = crtc->enabled_power_domains;
6828         crtc->enabled_power_domains = new_domains =
6829                 get_crtc_power_domains(crtc_state);
6830
6831         domains = new_domains & ~old_domains;
6832
6833         for_each_power_domain(domain, domains)
6834                 intel_display_power_get(dev_priv, domain);
6835
6836         return old_domains & ~new_domains;
6837 }
6838
6839 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
6840                                       u64 domains)
6841 {
6842         enum intel_display_power_domain domain;
6843
6844         for_each_power_domain(domain, domains)
6845                 intel_display_power_put_unchecked(dev_priv, domain);
6846 }
6847
6848 static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
6849                                    struct intel_atomic_state *state)
6850 {
6851         struct drm_crtc *crtc = pipe_config->base.crtc;
6852         struct drm_device *dev = crtc->dev;
6853         struct drm_i915_private *dev_priv = to_i915(dev);
6854         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6855         int pipe = intel_crtc->pipe;
6856
6857         if (WARN_ON(intel_crtc->active))
6858                 return;
6859
6860         if (intel_crtc_has_dp_encoder(pipe_config))
6861                 intel_dp_set_m_n(pipe_config, M1_N1);
6862
6863         intel_set_pipe_timings(pipe_config);
6864         intel_set_pipe_src_size(pipe_config);
6865
6866         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
6867                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6868                 I915_WRITE(CHV_CANVAS(pipe), 0);
6869         }
6870
6871         i9xx_set_pipeconf(pipe_config);
6872
6873         intel_crtc->active = true;
6874
6875         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6876
6877         intel_encoders_pre_pll_enable(intel_crtc, pipe_config, state);
6878
6879         if (IS_CHERRYVIEW(dev_priv)) {
6880                 chv_prepare_pll(intel_crtc, pipe_config);
6881                 chv_enable_pll(intel_crtc, pipe_config);
6882         } else {
6883                 vlv_prepare_pll(intel_crtc, pipe_config);
6884                 vlv_enable_pll(intel_crtc, pipe_config);
6885         }
6886
6887         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6888
6889         i9xx_pfit_enable(pipe_config);
6890
6891         intel_color_load_luts(pipe_config);
6892         intel_color_commit(pipe_config);
6893         /* update DSPCNTR to configure gamma for pipe bottom color */
6894         intel_disable_primary_plane(pipe_config);
6895
6896         dev_priv->display.initial_watermarks(state, pipe_config);
6897         intel_enable_pipe(pipe_config);
6898
6899         assert_vblank_disabled(crtc);
6900         intel_crtc_vblank_on(pipe_config);
6901
6902         intel_encoders_enable(intel_crtc, pipe_config, state);
6903 }
6904
6905 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
6906 {
6907         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6908         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6909
6910         I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0);
6911         I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1);
6912 }
6913
6914 static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
6915                              struct intel_atomic_state *state)
6916 {
6917         struct drm_crtc *crtc = pipe_config->base.crtc;
6918         struct drm_device *dev = crtc->dev;
6919         struct drm_i915_private *dev_priv = to_i915(dev);
6920         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6921         enum pipe pipe = intel_crtc->pipe;
6922
6923         if (WARN_ON(intel_crtc->active))
6924                 return;
6925
6926         i9xx_set_pll_dividers(pipe_config);
6927
6928         if (intel_crtc_has_dp_encoder(pipe_config))
6929                 intel_dp_set_m_n(pipe_config, M1_N1);
6930
6931         intel_set_pipe_timings(pipe_config);
6932         intel_set_pipe_src_size(pipe_config);
6933
6934         i9xx_set_pipeconf(pipe_config);
6935
6936         intel_crtc->active = true;
6937
6938         if (!IS_GEN(dev_priv, 2))
6939                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6940
6941         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6942
6943         i9xx_enable_pll(intel_crtc, pipe_config);
6944
6945         i9xx_pfit_enable(pipe_config);
6946
6947         intel_color_load_luts(pipe_config);
6948         intel_color_commit(pipe_config);
6949         /* update DSPCNTR to configure gamma for pipe bottom color */
6950         intel_disable_primary_plane(pipe_config);
6951
6952         if (dev_priv->display.initial_watermarks != NULL)
6953                 dev_priv->display.initial_watermarks(state,
6954                                                      pipe_config);
6955         else
6956                 intel_update_watermarks(intel_crtc);
6957         intel_enable_pipe(pipe_config);
6958
6959         assert_vblank_disabled(crtc);
6960         intel_crtc_vblank_on(pipe_config);
6961
6962         intel_encoders_enable(intel_crtc, pipe_config, state);
6963 }
6964
6965 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6966 {
6967         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6968         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6969
6970         if (!old_crtc_state->gmch_pfit.control)
6971                 return;
6972
6973         assert_pipe_disabled(dev_priv, crtc->pipe);
6974
6975         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
6976                       I915_READ(PFIT_CONTROL));
6977         I915_WRITE(PFIT_CONTROL, 0);
6978 }
6979
6980 static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
6981                               struct intel_atomic_state *state)
6982 {
6983         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6984         struct drm_device *dev = crtc->dev;
6985         struct drm_i915_private *dev_priv = to_i915(dev);
6986         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6987         int pipe = intel_crtc->pipe;
6988
6989         /*
6990          * On gen2 planes are double buffered but the pipe isn't, so we must
6991          * wait for planes to fully turn off before disabling the pipe.
6992          */
6993         if (IS_GEN(dev_priv, 2))
6994                 intel_wait_for_vblank(dev_priv, pipe);
6995
6996         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6997
6998         drm_crtc_vblank_off(crtc);
6999         assert_vblank_disabled(crtc);
7000
7001         intel_disable_pipe(old_crtc_state);
7002
7003         i9xx_pfit_disable(old_crtc_state);
7004
7005         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
7006
7007         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
7008                 if (IS_CHERRYVIEW(dev_priv))
7009                         chv_disable_pll(dev_priv, pipe);
7010                 else if (IS_VALLEYVIEW(dev_priv))
7011                         vlv_disable_pll(dev_priv, pipe);
7012                 else
7013                         i9xx_disable_pll(old_crtc_state);
7014         }
7015
7016         intel_encoders_post_pll_disable(intel_crtc, old_crtc_state, state);
7017
7018         if (!IS_GEN(dev_priv, 2))
7019                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
7020
7021         if (!dev_priv->display.initial_watermarks)
7022                 intel_update_watermarks(intel_crtc);
7023
7024         /* clock the pipe down to 640x480@60 to potentially save power */
7025         if (IS_I830(dev_priv))
7026                 i830_enable_pipe(dev_priv, pipe);
7027 }
7028
7029 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
7030                                         struct drm_modeset_acquire_ctx *ctx)
7031 {
7032         struct intel_encoder *encoder;
7033         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7034         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
7035         struct intel_bw_state *bw_state =
7036                 to_intel_bw_state(dev_priv->bw_obj.state);
7037         enum intel_display_power_domain domain;
7038         struct intel_plane *plane;
7039         u64 domains;
7040         struct drm_atomic_state *state;
7041         struct intel_crtc_state *crtc_state;
7042         int ret;
7043
7044         if (!intel_crtc->active)
7045                 return;
7046
7047         for_each_intel_plane_on_crtc(&dev_priv->drm, intel_crtc, plane) {
7048                 const struct intel_plane_state *plane_state =
7049                         to_intel_plane_state(plane->base.state);
7050
7051                 if (plane_state->base.visible)
7052                         intel_plane_disable_noatomic(intel_crtc, plane);
7053         }
7054
7055         state = drm_atomic_state_alloc(crtc->dev);
7056         if (!state) {
7057                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
7058                               crtc->base.id, crtc->name);
7059                 return;
7060         }
7061
7062         state->acquire_ctx = ctx;
7063
7064         /* Everything's already locked, -EDEADLK can't happen. */
7065         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
7066         ret = drm_atomic_add_affected_connectors(state, crtc);
7067
7068         WARN_ON(IS_ERR(crtc_state) || ret);
7069
7070         dev_priv->display.crtc_disable(crtc_state, to_intel_atomic_state(state));
7071
7072         drm_atomic_state_put(state);
7073
7074         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
7075                       crtc->base.id, crtc->name);
7076
7077         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0);
7078         crtc->state->active = false;
7079         intel_crtc->active = false;
7080         crtc->enabled = false;
7081         crtc->state->connector_mask = 0;
7082         crtc->state->encoder_mask = 0;
7083
7084         for_each_encoder_on_crtc(crtc->dev, crtc, encoder)
7085                 encoder->base.crtc = NULL;
7086
7087         intel_fbc_disable(intel_crtc);
7088         intel_update_watermarks(intel_crtc);
7089         intel_disable_shared_dpll(to_intel_crtc_state(crtc->state));
7090
7091         domains = intel_crtc->enabled_power_domains;
7092         for_each_power_domain(domain, domains)
7093                 intel_display_power_put_unchecked(dev_priv, domain);
7094         intel_crtc->enabled_power_domains = 0;
7095
7096         dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
7097         dev_priv->min_cdclk[intel_crtc->pipe] = 0;
7098         dev_priv->min_voltage_level[intel_crtc->pipe] = 0;
7099
7100         bw_state->data_rate[intel_crtc->pipe] = 0;
7101         bw_state->num_active_planes[intel_crtc->pipe] = 0;
7102 }
7103
7104 /*
7105  * turn all crtc's off, but do not adjust state
7106  * This has to be paired with a call to intel_modeset_setup_hw_state.
7107  */
7108 int intel_display_suspend(struct drm_device *dev)
7109 {
7110         struct drm_i915_private *dev_priv = to_i915(dev);
7111         struct drm_atomic_state *state;
7112         int ret;
7113
7114         state = drm_atomic_helper_suspend(dev);
7115         ret = PTR_ERR_OR_ZERO(state);
7116         if (ret)
7117                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
7118         else
7119                 dev_priv->modeset_restore_state = state;
7120         return ret;
7121 }
7122
7123 void intel_encoder_destroy(struct drm_encoder *encoder)
7124 {
7125         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7126
7127         drm_encoder_cleanup(encoder);
7128         kfree(intel_encoder);
7129 }
7130
7131 /* Cross check the actual hw state with our own modeset state tracking (and it's
7132  * internal consistency). */
7133 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
7134                                          struct drm_connector_state *conn_state)
7135 {
7136         struct intel_connector *connector = to_intel_connector(conn_state->connector);
7137
7138         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
7139                       connector->base.base.id,
7140                       connector->base.name);
7141
7142         if (connector->get_hw_state(connector)) {
7143                 struct intel_encoder *encoder = connector->encoder;
7144
7145                 I915_STATE_WARN(!crtc_state,
7146                          "connector enabled without attached crtc\n");
7147
7148                 if (!crtc_state)
7149                         return;
7150
7151                 I915_STATE_WARN(!crtc_state->base.active,
7152                       "connector is active, but attached crtc isn't\n");
7153
7154                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
7155                         return;
7156
7157                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
7158                         "atomic encoder doesn't match attached encoder\n");
7159
7160                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
7161                         "attached encoder crtc differs from connector crtc\n");
7162         } else {
7163                 I915_STATE_WARN(crtc_state && crtc_state->base.active,
7164                         "attached crtc is active, but connector isn't\n");
7165                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
7166                         "best encoder set without crtc!\n");
7167         }
7168 }
7169
7170 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
7171 {
7172         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
7173                 return crtc_state->fdi_lanes;
7174
7175         return 0;
7176 }
7177
7178 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
7179                                      struct intel_crtc_state *pipe_config)
7180 {
7181         struct drm_i915_private *dev_priv = to_i915(dev);
7182         struct drm_atomic_state *state = pipe_config->base.state;
7183         struct intel_crtc *other_crtc;
7184         struct intel_crtc_state *other_crtc_state;
7185
7186         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
7187                       pipe_name(pipe), pipe_config->fdi_lanes);
7188         if (pipe_config->fdi_lanes > 4) {
7189                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
7190                               pipe_name(pipe), pipe_config->fdi_lanes);
7191                 return -EINVAL;
7192         }
7193
7194         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7195                 if (pipe_config->fdi_lanes > 2) {
7196                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
7197                                       pipe_config->fdi_lanes);
7198                         return -EINVAL;
7199                 } else {
7200                         return 0;
7201                 }
7202         }
7203
7204         if (INTEL_INFO(dev_priv)->num_pipes == 2)
7205                 return 0;
7206
7207         /* Ivybridge 3 pipe is really complicated */
7208         switch (pipe) {
7209         case PIPE_A:
7210                 return 0;
7211         case PIPE_B:
7212                 if (pipe_config->fdi_lanes <= 2)
7213                         return 0;
7214
7215                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
7216                 other_crtc_state =
7217                         intel_atomic_get_crtc_state(state, other_crtc);
7218                 if (IS_ERR(other_crtc_state))
7219                         return PTR_ERR(other_crtc_state);
7220
7221                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
7222                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
7223                                       pipe_name(pipe), pipe_config->fdi_lanes);
7224                         return -EINVAL;
7225                 }
7226                 return 0;
7227         case PIPE_C:
7228                 if (pipe_config->fdi_lanes > 2) {
7229                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
7230                                       pipe_name(pipe), pipe_config->fdi_lanes);
7231                         return -EINVAL;
7232                 }
7233
7234                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
7235                 other_crtc_state =
7236                         intel_atomic_get_crtc_state(state, other_crtc);
7237                 if (IS_ERR(other_crtc_state))
7238                         return PTR_ERR(other_crtc_state);
7239
7240                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
7241                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
7242                         return -EINVAL;
7243                 }
7244                 return 0;
7245         default:
7246                 BUG();
7247         }
7248 }
7249
7250 #define RETRY 1
7251 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
7252                                        struct intel_crtc_state *pipe_config)
7253 {
7254         struct drm_device *dev = intel_crtc->base.dev;
7255         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
7256         int lane, link_bw, fdi_dotclock, ret;
7257         bool needs_recompute = false;
7258
7259 retry:
7260         /* FDI is a binary signal running at ~2.7GHz, encoding
7261          * each output octet as 10 bits. The actual frequency
7262          * is stored as a divider into a 100MHz clock, and the
7263          * mode pixel clock is stored in units of 1KHz.
7264          * Hence the bw of each lane in terms of the mode signal
7265          * is:
7266          */
7267         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
7268
7269         fdi_dotclock = adjusted_mode->crtc_clock;
7270
7271         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
7272                                            pipe_config->pipe_bpp);
7273
7274         pipe_config->fdi_lanes = lane;
7275
7276         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
7277                                link_bw, &pipe_config->fdi_m_n, false, false);
7278
7279         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
7280         if (ret == -EDEADLK)
7281                 return ret;
7282
7283         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
7284                 pipe_config->pipe_bpp -= 2*3;
7285                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
7286                               pipe_config->pipe_bpp);
7287                 needs_recompute = true;
7288                 pipe_config->bw_constrained = true;
7289
7290                 goto retry;
7291         }
7292
7293         if (needs_recompute)
7294                 return RETRY;
7295
7296         return ret;
7297 }
7298
7299 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
7300 {
7301         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7302         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7303
7304         /* IPS only exists on ULT machines and is tied to pipe A. */
7305         if (!hsw_crtc_supports_ips(crtc))
7306                 return false;
7307
7308         if (!i915_modparams.enable_ips)
7309                 return false;
7310
7311         if (crtc_state->pipe_bpp > 24)
7312                 return false;
7313
7314         /*
7315          * We compare against max which means we must take
7316          * the increased cdclk requirement into account when
7317          * calculating the new cdclk.
7318          *
7319          * Should measure whether using a lower cdclk w/o IPS
7320          */
7321         if (IS_BROADWELL(dev_priv) &&
7322             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
7323                 return false;
7324
7325         return true;
7326 }
7327
7328 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
7329 {
7330         struct drm_i915_private *dev_priv =
7331                 to_i915(crtc_state->base.crtc->dev);
7332         struct intel_atomic_state *intel_state =
7333                 to_intel_atomic_state(crtc_state->base.state);
7334
7335         if (!hsw_crtc_state_ips_capable(crtc_state))
7336                 return false;
7337
7338         /*
7339          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
7340          * enabled and disabled dynamically based on package C states,
7341          * user space can't make reliable use of the CRCs, so let's just
7342          * completely disable it.
7343          */
7344         if (crtc_state->crc_enabled)
7345                 return false;
7346
7347         /* IPS should be fine as long as at least one plane is enabled. */
7348         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
7349                 return false;
7350
7351         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
7352         if (IS_BROADWELL(dev_priv) &&
7353             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
7354                 return false;
7355
7356         return true;
7357 }
7358
7359 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
7360 {
7361         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7362
7363         /* GDG double wide on either pipe, otherwise pipe A only */
7364         return INTEL_GEN(dev_priv) < 4 &&
7365                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
7366 }
7367
7368 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
7369 {
7370         u32 pixel_rate;
7371
7372         pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
7373
7374         /*
7375          * We only use IF-ID interlacing. If we ever use
7376          * PF-ID we'll need to adjust the pixel_rate here.
7377          */
7378
7379         if (pipe_config->pch_pfit.enabled) {
7380                 u64 pipe_w, pipe_h, pfit_w, pfit_h;
7381                 u32 pfit_size = pipe_config->pch_pfit.size;
7382
7383                 pipe_w = pipe_config->pipe_src_w;
7384                 pipe_h = pipe_config->pipe_src_h;
7385
7386                 pfit_w = (pfit_size >> 16) & 0xFFFF;
7387                 pfit_h = pfit_size & 0xFFFF;
7388                 if (pipe_w < pfit_w)
7389                         pipe_w = pfit_w;
7390                 if (pipe_h < pfit_h)
7391                         pipe_h = pfit_h;
7392
7393                 if (WARN_ON(!pfit_w || !pfit_h))
7394                         return pixel_rate;
7395
7396                 pixel_rate = div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h),
7397                                      pfit_w * pfit_h);
7398         }
7399
7400         return pixel_rate;
7401 }
7402
7403 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
7404 {
7405         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
7406
7407         if (HAS_GMCH(dev_priv))
7408                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
7409                 crtc_state->pixel_rate =
7410                         crtc_state->base.adjusted_mode.crtc_clock;
7411         else
7412                 crtc_state->pixel_rate =
7413                         ilk_pipe_pixel_rate(crtc_state);
7414 }
7415
7416 static int intel_crtc_compute_config(struct intel_crtc *crtc,
7417                                      struct intel_crtc_state *pipe_config)
7418 {
7419         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7420         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
7421         int clock_limit = dev_priv->max_dotclk_freq;
7422
7423         if (INTEL_GEN(dev_priv) < 4) {
7424                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
7425
7426                 /*
7427                  * Enable double wide mode when the dot clock
7428                  * is > 90% of the (display) core speed.
7429                  */
7430                 if (intel_crtc_supports_double_wide(crtc) &&
7431                     adjusted_mode->crtc_clock > clock_limit) {
7432                         clock_limit = dev_priv->max_dotclk_freq;
7433                         pipe_config->double_wide = true;
7434                 }
7435         }
7436
7437         if (adjusted_mode->crtc_clock > clock_limit) {
7438                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
7439                               adjusted_mode->crtc_clock, clock_limit,
7440                               yesno(pipe_config->double_wide));
7441                 return -EINVAL;
7442         }
7443
7444         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
7445              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
7446              pipe_config->base.ctm) {
7447                 /*
7448                  * There is only one pipe CSC unit per pipe, and we need that
7449                  * for output conversion from RGB->YCBCR. So if CTM is already
7450                  * applied we can't support YCBCR420 output.
7451                  */
7452                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
7453                 return -EINVAL;
7454         }
7455
7456         /*
7457          * Pipe horizontal size must be even in:
7458          * - DVO ganged mode
7459          * - LVDS dual channel mode
7460          * - Double wide pipe
7461          */
7462         if (pipe_config->pipe_src_w & 1) {
7463                 if (pipe_config->double_wide) {
7464                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
7465                         return -EINVAL;
7466                 }
7467
7468                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
7469                     intel_is_dual_link_lvds(dev_priv)) {
7470                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
7471                         return -EINVAL;
7472                 }
7473         }
7474
7475         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
7476          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
7477          */
7478         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
7479                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
7480                 return -EINVAL;
7481
7482         intel_crtc_compute_pixel_rate(pipe_config);
7483
7484         if (pipe_config->has_pch_encoder)
7485                 return ironlake_fdi_compute_config(crtc, pipe_config);
7486
7487         return 0;
7488 }
7489
7490 static void
7491 intel_reduce_m_n_ratio(u32 *num, u32 *den)
7492 {
7493         while (*num > DATA_LINK_M_N_MASK ||
7494                *den > DATA_LINK_M_N_MASK) {
7495                 *num >>= 1;
7496                 *den >>= 1;
7497         }
7498 }
7499
7500 static void compute_m_n(unsigned int m, unsigned int n,
7501                         u32 *ret_m, u32 *ret_n,
7502                         bool constant_n)
7503 {
7504         /*
7505          * Several DP dongles in particular seem to be fussy about
7506          * too large link M/N values. Give N value as 0x8000 that
7507          * should be acceptable by specific devices. 0x8000 is the
7508          * specified fixed N value for asynchronous clock mode,
7509          * which the devices expect also in synchronous clock mode.
7510          */
7511         if (constant_n)
7512                 *ret_n = 0x8000;
7513         else
7514                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7515
7516         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
7517         intel_reduce_m_n_ratio(ret_m, ret_n);
7518 }
7519
7520 void
7521 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
7522                        int pixel_clock, int link_clock,
7523                        struct intel_link_m_n *m_n,
7524                        bool constant_n, bool fec_enable)
7525 {
7526         u32 data_clock = bits_per_pixel * pixel_clock;
7527
7528         if (fec_enable)
7529                 data_clock = intel_dp_mode_to_fec_clock(data_clock);
7530
7531         m_n->tu = 64;
7532         compute_m_n(data_clock,
7533                     link_clock * nlanes * 8,
7534                     &m_n->gmch_m, &m_n->gmch_n,
7535                     constant_n);
7536
7537         compute_m_n(pixel_clock, link_clock,
7538                     &m_n->link_m, &m_n->link_n,
7539                     constant_n);
7540 }
7541
7542 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7543 {
7544         if (i915_modparams.panel_use_ssc >= 0)
7545                 return i915_modparams.panel_use_ssc != 0;
7546         return dev_priv->vbt.lvds_use_ssc
7547                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7548 }
7549
7550 static u32 pnv_dpll_compute_fp(struct dpll *dpll)
7551 {
7552         return (1 << dpll->n) << 16 | dpll->m2;
7553 }
7554
7555 static u32 i9xx_dpll_compute_fp(struct dpll *dpll)
7556 {
7557         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7558 }
7559
7560 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7561                                      struct intel_crtc_state *crtc_state,
7562                                      struct dpll *reduced_clock)
7563 {
7564         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7565         u32 fp, fp2 = 0;
7566
7567         if (IS_PINEVIEW(dev_priv)) {
7568                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7569                 if (reduced_clock)
7570                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7571         } else {
7572                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7573                 if (reduced_clock)
7574                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7575         }
7576
7577         crtc_state->dpll_hw_state.fp0 = fp;
7578
7579         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7580             reduced_clock) {
7581                 crtc_state->dpll_hw_state.fp1 = fp2;
7582         } else {
7583                 crtc_state->dpll_hw_state.fp1 = fp;
7584         }
7585 }
7586
7587 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7588                 pipe)
7589 {
7590         u32 reg_val;
7591
7592         /*
7593          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7594          * and set it to a reasonable value instead.
7595          */
7596         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7597         reg_val &= 0xffffff00;
7598         reg_val |= 0x00000030;
7599         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7600
7601         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7602         reg_val &= 0x00ffffff;
7603         reg_val |= 0x8c000000;
7604         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7605
7606         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7607         reg_val &= 0xffffff00;
7608         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7609
7610         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7611         reg_val &= 0x00ffffff;
7612         reg_val |= 0xb0000000;
7613         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7614 }
7615
7616 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7617                                          const struct intel_link_m_n *m_n)
7618 {
7619         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7620         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7621         enum pipe pipe = crtc->pipe;
7622
7623         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7624         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7625         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7626         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7627 }
7628
7629 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
7630                                  enum transcoder transcoder)
7631 {
7632         if (IS_HASWELL(dev_priv))
7633                 return transcoder == TRANSCODER_EDP;
7634
7635         /*
7636          * Strictly speaking some registers are available before
7637          * gen7, but we only support DRRS on gen7+
7638          */
7639         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
7640 }
7641
7642 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7643                                          const struct intel_link_m_n *m_n,
7644                                          const struct intel_link_m_n *m2_n2)
7645 {
7646         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7647         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7648         enum pipe pipe = crtc->pipe;
7649         enum transcoder transcoder = crtc_state->cpu_transcoder;
7650
7651         if (INTEL_GEN(dev_priv) >= 5) {
7652                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7653                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7654                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7655                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7656                 /*
7657                  *  M2_N2 registers are set only if DRRS is supported
7658                  * (to make sure the registers are not unnecessarily accessed).
7659                  */
7660                 if (m2_n2 && crtc_state->has_drrs &&
7661                     transcoder_has_m2_n2(dev_priv, transcoder)) {
7662                         I915_WRITE(PIPE_DATA_M2(transcoder),
7663                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7664                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
7665                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
7666                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
7667                 }
7668         } else {
7669                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7670                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
7671                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
7672                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
7673         }
7674 }
7675
7676 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
7677 {
7678         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
7679
7680         if (m_n == M1_N1) {
7681                 dp_m_n = &crtc_state->dp_m_n;
7682                 dp_m2_n2 = &crtc_state->dp_m2_n2;
7683         } else if (m_n == M2_N2) {
7684
7685                 /*
7686                  * M2_N2 registers are not supported. Hence m2_n2 divider value
7687                  * needs to be programmed into M1_N1.
7688                  */
7689                 dp_m_n = &crtc_state->dp_m2_n2;
7690         } else {
7691                 DRM_ERROR("Unsupported divider value\n");
7692                 return;
7693         }
7694
7695         if (crtc_state->has_pch_encoder)
7696                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
7697         else
7698                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
7699 }
7700
7701 static void vlv_compute_dpll(struct intel_crtc *crtc,
7702                              struct intel_crtc_state *pipe_config)
7703 {
7704         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
7705                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7706         if (crtc->pipe != PIPE_A)
7707                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7708
7709         /* DPLL not used with DSI, but still need the rest set up */
7710         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
7711                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
7712                         DPLL_EXT_BUFFER_ENABLE_VLV;
7713
7714         pipe_config->dpll_hw_state.dpll_md =
7715                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7716 }
7717
7718 static void chv_compute_dpll(struct intel_crtc *crtc,
7719                              struct intel_crtc_state *pipe_config)
7720 {
7721         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
7722                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7723         if (crtc->pipe != PIPE_A)
7724                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7725
7726         /* DPLL not used with DSI, but still need the rest set up */
7727         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
7728                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
7729
7730         pipe_config->dpll_hw_state.dpll_md =
7731                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7732 }
7733
7734 static void vlv_prepare_pll(struct intel_crtc *crtc,
7735                             const struct intel_crtc_state *pipe_config)
7736 {
7737         struct drm_device *dev = crtc->base.dev;
7738         struct drm_i915_private *dev_priv = to_i915(dev);
7739         enum pipe pipe = crtc->pipe;
7740         u32 mdiv;
7741         u32 bestn, bestm1, bestm2, bestp1, bestp2;
7742         u32 coreclk, reg_val;
7743
7744         /* Enable Refclk */
7745         I915_WRITE(DPLL(pipe),
7746                    pipe_config->dpll_hw_state.dpll &
7747                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
7748
7749         /* No need to actually set up the DPLL with DSI */
7750         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7751                 return;
7752
7753         vlv_dpio_get(dev_priv);
7754
7755         bestn = pipe_config->dpll.n;
7756         bestm1 = pipe_config->dpll.m1;
7757         bestm2 = pipe_config->dpll.m2;
7758         bestp1 = pipe_config->dpll.p1;
7759         bestp2 = pipe_config->dpll.p2;
7760
7761         /* See eDP HDMI DPIO driver vbios notes doc */
7762
7763         /* PLL B needs special handling */
7764         if (pipe == PIPE_B)
7765                 vlv_pllb_recal_opamp(dev_priv, pipe);
7766
7767         /* Set up Tx target for periodic Rcomp update */
7768         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7769
7770         /* Disable target IRef on PLL */
7771         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7772         reg_val &= 0x00ffffff;
7773         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7774
7775         /* Disable fast lock */
7776         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7777
7778         /* Set idtafcrecal before PLL is enabled */
7779         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7780         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7781         mdiv |= ((bestn << DPIO_N_SHIFT));
7782         mdiv |= (1 << DPIO_K_SHIFT);
7783
7784         /*
7785          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7786          * but we don't support that).
7787          * Note: don't use the DAC post divider as it seems unstable.
7788          */
7789         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7790         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7791
7792         mdiv |= DPIO_ENABLE_CALIBRATION;
7793         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7794
7795         /* Set HBR and RBR LPF coefficients */
7796         if (pipe_config->port_clock == 162000 ||
7797             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
7798             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
7799                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7800                                  0x009f0003);
7801         else
7802                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7803                                  0x00d0000f);
7804
7805         if (intel_crtc_has_dp_encoder(pipe_config)) {
7806                 /* Use SSC source */
7807                 if (pipe == PIPE_A)
7808                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7809                                          0x0df40000);
7810                 else
7811                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7812                                          0x0df70000);
7813         } else { /* HDMI or VGA */
7814                 /* Use bend source */
7815                 if (pipe == PIPE_A)
7816                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7817                                          0x0df70000);
7818                 else
7819                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7820                                          0x0df40000);
7821         }
7822
7823         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7824         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7825         if (intel_crtc_has_dp_encoder(pipe_config))
7826                 coreclk |= 0x01000000;
7827         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7828
7829         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7830
7831         vlv_dpio_put(dev_priv);
7832 }
7833
7834 static void chv_prepare_pll(struct intel_crtc *crtc,
7835                             const struct intel_crtc_state *pipe_config)
7836 {
7837         struct drm_device *dev = crtc->base.dev;
7838         struct drm_i915_private *dev_priv = to_i915(dev);
7839         enum pipe pipe = crtc->pipe;
7840         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7841         u32 loopfilter, tribuf_calcntr;
7842         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7843         u32 dpio_val;
7844         int vco;
7845
7846         /* Enable Refclk and SSC */
7847         I915_WRITE(DPLL(pipe),
7848                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7849
7850         /* No need to actually set up the DPLL with DSI */
7851         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7852                 return;
7853
7854         bestn = pipe_config->dpll.n;
7855         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7856         bestm1 = pipe_config->dpll.m1;
7857         bestm2 = pipe_config->dpll.m2 >> 22;
7858         bestp1 = pipe_config->dpll.p1;
7859         bestp2 = pipe_config->dpll.p2;
7860         vco = pipe_config->dpll.vco;
7861         dpio_val = 0;
7862         loopfilter = 0;
7863
7864         vlv_dpio_get(dev_priv);
7865
7866         /* p1 and p2 divider */
7867         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7868                         5 << DPIO_CHV_S1_DIV_SHIFT |
7869                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7870                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7871                         1 << DPIO_CHV_K_DIV_SHIFT);
7872
7873         /* Feedback post-divider - m2 */
7874         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7875
7876         /* Feedback refclk divider - n and m1 */
7877         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7878                         DPIO_CHV_M1_DIV_BY_2 |
7879                         1 << DPIO_CHV_N_DIV_SHIFT);
7880
7881         /* M2 fraction division */
7882         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7883
7884         /* M2 fraction division enable */
7885         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7886         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7887         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7888         if (bestm2_frac)
7889                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7890         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7891
7892         /* Program digital lock detect threshold */
7893         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7894         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7895                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7896         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7897         if (!bestm2_frac)
7898                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7899         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7900
7901         /* Loop filter */
7902         if (vco == 5400000) {
7903                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7904                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7905                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7906                 tribuf_calcntr = 0x9;
7907         } else if (vco <= 6200000) {
7908                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7909                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7910                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7911                 tribuf_calcntr = 0x9;
7912         } else if (vco <= 6480000) {
7913                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7914                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7915                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7916                 tribuf_calcntr = 0x8;
7917         } else {
7918                 /* Not supported. Apply the same limits as in the max case */
7919                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7920                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7921                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7922                 tribuf_calcntr = 0;
7923         }
7924         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7925
7926         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7927         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7928         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7929         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7930
7931         /* AFC Recal */
7932         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7933                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7934                         DPIO_AFC_RECAL);
7935
7936         vlv_dpio_put(dev_priv);
7937 }
7938
7939 /**
7940  * vlv_force_pll_on - forcibly enable just the PLL
7941  * @dev_priv: i915 private structure
7942  * @pipe: pipe PLL to enable
7943  * @dpll: PLL configuration
7944  *
7945  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7946  * in cases where we need the PLL enabled even when @pipe is not going to
7947  * be enabled.
7948  */
7949 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
7950                      const struct dpll *dpll)
7951 {
7952         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
7953         struct intel_crtc_state *pipe_config;
7954
7955         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
7956         if (!pipe_config)
7957                 return -ENOMEM;
7958
7959         pipe_config->base.crtc = &crtc->base;
7960         pipe_config->pixel_multiplier = 1;
7961         pipe_config->dpll = *dpll;
7962
7963         if (IS_CHERRYVIEW(dev_priv)) {
7964                 chv_compute_dpll(crtc, pipe_config);
7965                 chv_prepare_pll(crtc, pipe_config);
7966                 chv_enable_pll(crtc, pipe_config);
7967         } else {
7968                 vlv_compute_dpll(crtc, pipe_config);
7969                 vlv_prepare_pll(crtc, pipe_config);
7970                 vlv_enable_pll(crtc, pipe_config);
7971         }
7972
7973         kfree(pipe_config);
7974
7975         return 0;
7976 }
7977
7978 /**
7979  * vlv_force_pll_off - forcibly disable just the PLL
7980  * @dev_priv: i915 private structure
7981  * @pipe: pipe PLL to disable
7982  *
7983  * Disable the PLL for @pipe. To be used in cases where we need
7984  * the PLL enabled even when @pipe is not going to be enabled.
7985  */
7986 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
7987 {
7988         if (IS_CHERRYVIEW(dev_priv))
7989                 chv_disable_pll(dev_priv, pipe);
7990         else
7991                 vlv_disable_pll(dev_priv, pipe);
7992 }
7993
7994 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7995                               struct intel_crtc_state *crtc_state,
7996                               struct dpll *reduced_clock)
7997 {
7998         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7999         u32 dpll;
8000         struct dpll *clock = &crtc_state->dpll;
8001
8002         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8003
8004         dpll = DPLL_VGA_MODE_DIS;
8005
8006         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8007                 dpll |= DPLLB_MODE_LVDS;
8008         else
8009                 dpll |= DPLLB_MODE_DAC_SERIAL;
8010
8011         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8012             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8013                 dpll |= (crtc_state->pixel_multiplier - 1)
8014                         << SDVO_MULTIPLIER_SHIFT_HIRES;
8015         }
8016
8017         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8018             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8019                 dpll |= DPLL_SDVO_HIGH_SPEED;
8020
8021         if (intel_crtc_has_dp_encoder(crtc_state))
8022                 dpll |= DPLL_SDVO_HIGH_SPEED;
8023
8024         /* compute bitmask from p1 value */
8025         if (IS_PINEVIEW(dev_priv))
8026                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
8027         else {
8028                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8029                 if (IS_G4X(dev_priv) && reduced_clock)
8030                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8031         }
8032         switch (clock->p2) {
8033         case 5:
8034                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8035                 break;
8036         case 7:
8037                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8038                 break;
8039         case 10:
8040                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8041                 break;
8042         case 14:
8043                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8044                 break;
8045         }
8046         if (INTEL_GEN(dev_priv) >= 4)
8047                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
8048
8049         if (crtc_state->sdvo_tv_clock)
8050                 dpll |= PLL_REF_INPUT_TVCLKINBC;
8051         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8052                  intel_panel_use_ssc(dev_priv))
8053                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8054         else
8055                 dpll |= PLL_REF_INPUT_DREFCLK;
8056
8057         dpll |= DPLL_VCO_ENABLE;
8058         crtc_state->dpll_hw_state.dpll = dpll;
8059
8060         if (INTEL_GEN(dev_priv) >= 4) {
8061                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
8062                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8063                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
8064         }
8065 }
8066
8067 static void i8xx_compute_dpll(struct intel_crtc *crtc,
8068                               struct intel_crtc_state *crtc_state,
8069                               struct dpll *reduced_clock)
8070 {
8071         struct drm_device *dev = crtc->base.dev;
8072         struct drm_i915_private *dev_priv = to_i915(dev);
8073         u32 dpll;
8074         struct dpll *clock = &crtc_state->dpll;
8075
8076         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8077
8078         dpll = DPLL_VGA_MODE_DIS;
8079
8080         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8081                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8082         } else {
8083                 if (clock->p1 == 2)
8084                         dpll |= PLL_P1_DIVIDE_BY_TWO;
8085                 else
8086                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8087                 if (clock->p2 == 4)
8088                         dpll |= PLL_P2_DIVIDE_BY_4;
8089         }
8090
8091         /*
8092          * Bspec:
8093          * "[Almador Errata}: For the correct operation of the muxed DVO pins
8094          *  (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data,
8095          *  GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock
8096          *  Enable) must be set to “1” in both the DPLL A Control Register
8097          *  (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)."
8098          *
8099          * For simplicity We simply keep both bits always enabled in
8100          * both DPLLS. The spec says we should disable the DVO 2X clock
8101          * when not needed, but this seems to work fine in practice.
8102          */
8103         if (IS_I830(dev_priv) ||
8104             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
8105                 dpll |= DPLL_DVO_2X_MODE;
8106
8107         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8108             intel_panel_use_ssc(dev_priv))
8109                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8110         else
8111                 dpll |= PLL_REF_INPUT_DREFCLK;
8112
8113         dpll |= DPLL_VCO_ENABLE;
8114         crtc_state->dpll_hw_state.dpll = dpll;
8115 }
8116
8117 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
8118 {
8119         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8120         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8121         enum pipe pipe = crtc->pipe;
8122         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8123         const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
8124         u32 crtc_vtotal, crtc_vblank_end;
8125         int vsyncshift = 0;
8126
8127         /* We need to be careful not to changed the adjusted mode, for otherwise
8128          * the hw state checker will get angry at the mismatch. */
8129         crtc_vtotal = adjusted_mode->crtc_vtotal;
8130         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
8131
8132         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
8133                 /* the chip adds 2 halflines automatically */
8134                 crtc_vtotal -= 1;
8135                 crtc_vblank_end -= 1;
8136
8137                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8138                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
8139                 else
8140                         vsyncshift = adjusted_mode->crtc_hsync_start -
8141                                 adjusted_mode->crtc_htotal / 2;
8142                 if (vsyncshift < 0)
8143                         vsyncshift += adjusted_mode->crtc_htotal;
8144         }
8145
8146         if (INTEL_GEN(dev_priv) > 3)
8147                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
8148
8149         I915_WRITE(HTOTAL(cpu_transcoder),
8150                    (adjusted_mode->crtc_hdisplay - 1) |
8151                    ((adjusted_mode->crtc_htotal - 1) << 16));
8152         I915_WRITE(HBLANK(cpu_transcoder),
8153                    (adjusted_mode->crtc_hblank_start - 1) |
8154                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
8155         I915_WRITE(HSYNC(cpu_transcoder),
8156                    (adjusted_mode->crtc_hsync_start - 1) |
8157                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
8158
8159         I915_WRITE(VTOTAL(cpu_transcoder),
8160                    (adjusted_mode->crtc_vdisplay - 1) |
8161                    ((crtc_vtotal - 1) << 16));
8162         I915_WRITE(VBLANK(cpu_transcoder),
8163                    (adjusted_mode->crtc_vblank_start - 1) |
8164                    ((crtc_vblank_end - 1) << 16));
8165         I915_WRITE(VSYNC(cpu_transcoder),
8166                    (adjusted_mode->crtc_vsync_start - 1) |
8167                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
8168
8169         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
8170          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
8171          * documented on the DDI_FUNC_CTL register description, EDP Input Select
8172          * bits. */
8173         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
8174             (pipe == PIPE_B || pipe == PIPE_C))
8175                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
8176
8177 }
8178
8179 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
8180 {
8181         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8182         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8183         enum pipe pipe = crtc->pipe;
8184
8185         /* pipesrc controls the size that is scaled from, which should
8186          * always be the user's requested size.
8187          */
8188         I915_WRITE(PIPESRC(pipe),
8189                    ((crtc_state->pipe_src_w - 1) << 16) |
8190                    (crtc_state->pipe_src_h - 1));
8191 }
8192
8193 static void intel_get_pipe_timings(struct intel_crtc *crtc,
8194                                    struct intel_crtc_state *pipe_config)
8195 {
8196         struct drm_device *dev = crtc->base.dev;
8197         struct drm_i915_private *dev_priv = to_i915(dev);
8198         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
8199         u32 tmp;
8200
8201         tmp = I915_READ(HTOTAL(cpu_transcoder));
8202         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
8203         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
8204
8205         if (!transcoder_is_dsi(cpu_transcoder)) {
8206                 tmp = I915_READ(HBLANK(cpu_transcoder));
8207                 pipe_config->base.adjusted_mode.crtc_hblank_start =
8208                                                         (tmp & 0xffff) + 1;
8209                 pipe_config->base.adjusted_mode.crtc_hblank_end =
8210                                                 ((tmp >> 16) & 0xffff) + 1;
8211         }
8212         tmp = I915_READ(HSYNC(cpu_transcoder));
8213         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
8214         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
8215
8216         tmp = I915_READ(VTOTAL(cpu_transcoder));
8217         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
8218         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
8219
8220         if (!transcoder_is_dsi(cpu_transcoder)) {
8221                 tmp = I915_READ(VBLANK(cpu_transcoder));
8222                 pipe_config->base.adjusted_mode.crtc_vblank_start =
8223                                                         (tmp & 0xffff) + 1;
8224                 pipe_config->base.adjusted_mode.crtc_vblank_end =
8225                                                 ((tmp >> 16) & 0xffff) + 1;
8226         }
8227         tmp = I915_READ(VSYNC(cpu_transcoder));
8228         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
8229         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
8230
8231         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
8232                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
8233                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
8234                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
8235         }
8236 }
8237
8238 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
8239                                     struct intel_crtc_state *pipe_config)
8240 {
8241         struct drm_device *dev = crtc->base.dev;
8242         struct drm_i915_private *dev_priv = to_i915(dev);
8243         u32 tmp;
8244
8245         tmp = I915_READ(PIPESRC(crtc->pipe));
8246         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
8247         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
8248
8249         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
8250         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
8251 }
8252
8253 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
8254                                  struct intel_crtc_state *pipe_config)
8255 {
8256         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
8257         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
8258         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
8259         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
8260
8261         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
8262         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
8263         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
8264         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
8265
8266         mode->flags = pipe_config->base.adjusted_mode.flags;
8267         mode->type = DRM_MODE_TYPE_DRIVER;
8268
8269         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
8270
8271         mode->hsync = drm_mode_hsync(mode);
8272         mode->vrefresh = drm_mode_vrefresh(mode);
8273         drm_mode_set_name(mode);
8274 }
8275
8276 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
8277 {
8278         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8279         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8280         u32 pipeconf;
8281
8282         pipeconf = 0;
8283
8284         /* we keep both pipes enabled on 830 */
8285         if (IS_I830(dev_priv))
8286                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
8287
8288         if (crtc_state->double_wide)
8289                 pipeconf |= PIPECONF_DOUBLE_WIDE;
8290
8291         /* only g4x and later have fancy bpc/dither controls */
8292         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8293             IS_CHERRYVIEW(dev_priv)) {
8294                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
8295                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
8296                         pipeconf |= PIPECONF_DITHER_EN |
8297                                     PIPECONF_DITHER_TYPE_SP;
8298
8299                 switch (crtc_state->pipe_bpp) {
8300                 case 18:
8301                         pipeconf |= PIPECONF_6BPC;
8302                         break;
8303                 case 24:
8304                         pipeconf |= PIPECONF_8BPC;
8305                         break;
8306                 case 30:
8307                         pipeconf |= PIPECONF_10BPC;
8308                         break;
8309                 default:
8310                         /* Case prevented by intel_choose_pipe_bpp_dither. */
8311                         BUG();
8312                 }
8313         }
8314
8315         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
8316                 if (INTEL_GEN(dev_priv) < 4 ||
8317                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8318                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
8319                 else
8320                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
8321         } else {
8322                 pipeconf |= PIPECONF_PROGRESSIVE;
8323         }
8324
8325         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8326              crtc_state->limited_color_range)
8327                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
8328
8329         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
8330
8331         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
8332         POSTING_READ(PIPECONF(crtc->pipe));
8333 }
8334
8335 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
8336                                    struct intel_crtc_state *crtc_state)
8337 {
8338         struct drm_device *dev = crtc->base.dev;
8339         struct drm_i915_private *dev_priv = to_i915(dev);
8340         const struct intel_limit *limit;
8341         int refclk = 48000;
8342
8343         memset(&crtc_state->dpll_hw_state, 0,
8344                sizeof(crtc_state->dpll_hw_state));
8345
8346         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8347                 if (intel_panel_use_ssc(dev_priv)) {
8348                         refclk = dev_priv->vbt.lvds_ssc_freq;
8349                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8350                 }
8351
8352                 limit = &intel_limits_i8xx_lvds;
8353         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
8354                 limit = &intel_limits_i8xx_dvo;
8355         } else {
8356                 limit = &intel_limits_i8xx_dac;
8357         }
8358
8359         if (!crtc_state->clock_set &&
8360             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8361                                  refclk, NULL, &crtc_state->dpll)) {
8362                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8363                 return -EINVAL;
8364         }
8365
8366         i8xx_compute_dpll(crtc, crtc_state, NULL);
8367
8368         return 0;
8369 }
8370
8371 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
8372                                   struct intel_crtc_state *crtc_state)
8373 {
8374         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8375         const struct intel_limit *limit;
8376         int refclk = 96000;
8377
8378         memset(&crtc_state->dpll_hw_state, 0,
8379                sizeof(crtc_state->dpll_hw_state));
8380
8381         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8382                 if (intel_panel_use_ssc(dev_priv)) {
8383                         refclk = dev_priv->vbt.lvds_ssc_freq;
8384                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8385                 }
8386
8387                 if (intel_is_dual_link_lvds(dev_priv))
8388                         limit = &intel_limits_g4x_dual_channel_lvds;
8389                 else
8390                         limit = &intel_limits_g4x_single_channel_lvds;
8391         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
8392                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
8393                 limit = &intel_limits_g4x_hdmi;
8394         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
8395                 limit = &intel_limits_g4x_sdvo;
8396         } else {
8397                 /* The option is for other outputs */
8398                 limit = &intel_limits_i9xx_sdvo;
8399         }
8400
8401         if (!crtc_state->clock_set &&
8402             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8403                                 refclk, NULL, &crtc_state->dpll)) {
8404                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8405                 return -EINVAL;
8406         }
8407
8408         i9xx_compute_dpll(crtc, crtc_state, NULL);
8409
8410         return 0;
8411 }
8412
8413 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
8414                                   struct intel_crtc_state *crtc_state)
8415 {
8416         struct drm_device *dev = crtc->base.dev;
8417         struct drm_i915_private *dev_priv = to_i915(dev);
8418         const struct intel_limit *limit;
8419         int refclk = 96000;
8420
8421         memset(&crtc_state->dpll_hw_state, 0,
8422                sizeof(crtc_state->dpll_hw_state));
8423
8424         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8425                 if (intel_panel_use_ssc(dev_priv)) {
8426                         refclk = dev_priv->vbt.lvds_ssc_freq;
8427                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8428                 }
8429
8430                 limit = &intel_limits_pineview_lvds;
8431         } else {
8432                 limit = &intel_limits_pineview_sdvo;
8433         }
8434
8435         if (!crtc_state->clock_set &&
8436             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8437                                 refclk, NULL, &crtc_state->dpll)) {
8438                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8439                 return -EINVAL;
8440         }
8441
8442         i9xx_compute_dpll(crtc, crtc_state, NULL);
8443
8444         return 0;
8445 }
8446
8447 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
8448                                    struct intel_crtc_state *crtc_state)
8449 {
8450         struct drm_device *dev = crtc->base.dev;
8451         struct drm_i915_private *dev_priv = to_i915(dev);
8452         const struct intel_limit *limit;
8453         int refclk = 96000;
8454
8455         memset(&crtc_state->dpll_hw_state, 0,
8456                sizeof(crtc_state->dpll_hw_state));
8457
8458         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8459                 if (intel_panel_use_ssc(dev_priv)) {
8460                         refclk = dev_priv->vbt.lvds_ssc_freq;
8461                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8462                 }
8463
8464                 limit = &intel_limits_i9xx_lvds;
8465         } else {
8466                 limit = &intel_limits_i9xx_sdvo;
8467         }
8468
8469         if (!crtc_state->clock_set &&
8470             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8471                                  refclk, NULL, &crtc_state->dpll)) {
8472                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8473                 return -EINVAL;
8474         }
8475
8476         i9xx_compute_dpll(crtc, crtc_state, NULL);
8477
8478         return 0;
8479 }
8480
8481 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
8482                                   struct intel_crtc_state *crtc_state)
8483 {
8484         int refclk = 100000;
8485         const struct intel_limit *limit = &intel_limits_chv;
8486
8487         memset(&crtc_state->dpll_hw_state, 0,
8488                sizeof(crtc_state->dpll_hw_state));
8489
8490         if (!crtc_state->clock_set &&
8491             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8492                                 refclk, NULL, &crtc_state->dpll)) {
8493                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8494                 return -EINVAL;
8495         }
8496
8497         chv_compute_dpll(crtc, crtc_state);
8498
8499         return 0;
8500 }
8501
8502 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
8503                                   struct intel_crtc_state *crtc_state)
8504 {
8505         int refclk = 100000;
8506         const struct intel_limit *limit = &intel_limits_vlv;
8507
8508         memset(&crtc_state->dpll_hw_state, 0,
8509                sizeof(crtc_state->dpll_hw_state));
8510
8511         if (!crtc_state->clock_set &&
8512             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8513                                 refclk, NULL, &crtc_state->dpll)) {
8514                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8515                 return -EINVAL;
8516         }
8517
8518         vlv_compute_dpll(crtc, crtc_state);
8519
8520         return 0;
8521 }
8522
8523 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
8524 {
8525         if (IS_I830(dev_priv))
8526                 return false;
8527
8528         return INTEL_GEN(dev_priv) >= 4 ||
8529                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
8530 }
8531
8532 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8533                                  struct intel_crtc_state *pipe_config)
8534 {
8535         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8536         u32 tmp;
8537
8538         if (!i9xx_has_pfit(dev_priv))
8539                 return;
8540
8541         tmp = I915_READ(PFIT_CONTROL);
8542         if (!(tmp & PFIT_ENABLE))
8543                 return;
8544
8545         /* Check whether the pfit is attached to our pipe. */
8546         if (INTEL_GEN(dev_priv) < 4) {
8547                 if (crtc->pipe != PIPE_B)
8548                         return;
8549         } else {
8550                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
8551                         return;
8552         }
8553
8554         pipe_config->gmch_pfit.control = tmp;
8555         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
8556 }
8557
8558 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8559                                struct intel_crtc_state *pipe_config)
8560 {
8561         struct drm_device *dev = crtc->base.dev;
8562         struct drm_i915_private *dev_priv = to_i915(dev);
8563         int pipe = pipe_config->cpu_transcoder;
8564         struct dpll clock;
8565         u32 mdiv;
8566         int refclk = 100000;
8567
8568         /* In case of DSI, DPLL will not be used */
8569         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8570                 return;
8571
8572         vlv_dpio_get(dev_priv);
8573         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8574         vlv_dpio_put(dev_priv);
8575
8576         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8577         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8578         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8579         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8580         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8581
8582         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8583 }
8584
8585 static void
8586 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8587                               struct intel_initial_plane_config *plane_config)
8588 {
8589         struct drm_device *dev = crtc->base.dev;
8590         struct drm_i915_private *dev_priv = to_i915(dev);
8591         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8592         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8593         enum pipe pipe;
8594         u32 val, base, offset;
8595         int fourcc, pixel_format;
8596         unsigned int aligned_height;
8597         struct drm_framebuffer *fb;
8598         struct intel_framebuffer *intel_fb;
8599
8600         if (!plane->get_hw_state(plane, &pipe))
8601                 return;
8602
8603         WARN_ON(pipe != crtc->pipe);
8604
8605         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8606         if (!intel_fb) {
8607                 DRM_DEBUG_KMS("failed to alloc fb\n");
8608                 return;
8609         }
8610
8611         fb = &intel_fb->base;
8612
8613         fb->dev = dev;
8614
8615         val = I915_READ(DSPCNTR(i9xx_plane));
8616
8617         if (INTEL_GEN(dev_priv) >= 4) {
8618                 if (val & DISPPLANE_TILED) {
8619                         plane_config->tiling = I915_TILING_X;
8620                         fb->modifier = I915_FORMAT_MOD_X_TILED;
8621                 }
8622
8623                 if (val & DISPPLANE_ROTATE_180)
8624                         plane_config->rotation = DRM_MODE_ROTATE_180;
8625         }
8626
8627         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
8628             val & DISPPLANE_MIRROR)
8629                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8630
8631         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8632         fourcc = i9xx_format_to_fourcc(pixel_format);
8633         fb->format = drm_format_info(fourcc);
8634
8635         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
8636                 offset = I915_READ(DSPOFFSET(i9xx_plane));
8637                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8638         } else if (INTEL_GEN(dev_priv) >= 4) {
8639                 if (plane_config->tiling)
8640                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
8641                 else
8642                         offset = I915_READ(DSPLINOFF(i9xx_plane));
8643                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8644         } else {
8645                 base = I915_READ(DSPADDR(i9xx_plane));
8646         }
8647         plane_config->base = base;
8648
8649         val = I915_READ(PIPESRC(pipe));
8650         fb->width = ((val >> 16) & 0xfff) + 1;
8651         fb->height = ((val >> 0) & 0xfff) + 1;
8652
8653         val = I915_READ(DSPSTRIDE(i9xx_plane));
8654         fb->pitches[0] = val & 0xffffffc0;
8655
8656         aligned_height = intel_fb_align_height(fb, 0, fb->height);
8657
8658         plane_config->size = fb->pitches[0] * aligned_height;
8659
8660         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8661                       crtc->base.name, plane->base.name, fb->width, fb->height,
8662                       fb->format->cpp[0] * 8, base, fb->pitches[0],
8663                       plane_config->size);
8664
8665         plane_config->fb = intel_fb;
8666 }
8667
8668 static void chv_crtc_clock_get(struct intel_crtc *crtc,
8669                                struct intel_crtc_state *pipe_config)
8670 {
8671         struct drm_device *dev = crtc->base.dev;
8672         struct drm_i915_private *dev_priv = to_i915(dev);
8673         int pipe = pipe_config->cpu_transcoder;
8674         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8675         struct dpll clock;
8676         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
8677         int refclk = 100000;
8678
8679         /* In case of DSI, DPLL will not be used */
8680         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8681                 return;
8682
8683         vlv_dpio_get(dev_priv);
8684         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
8685         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
8686         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
8687         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
8688         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8689         vlv_dpio_put(dev_priv);
8690
8691         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
8692         clock.m2 = (pll_dw0 & 0xff) << 22;
8693         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
8694                 clock.m2 |= pll_dw2 & 0x3fffff;
8695         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
8696         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
8697         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
8698
8699         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
8700 }
8701
8702 static void intel_get_crtc_ycbcr_config(struct intel_crtc *crtc,
8703                                         struct intel_crtc_state *pipe_config)
8704 {
8705         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8706         enum intel_output_format output = INTEL_OUTPUT_FORMAT_RGB;
8707
8708         pipe_config->lspcon_downsampling = false;
8709
8710         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
8711                 u32 tmp = I915_READ(PIPEMISC(crtc->pipe));
8712
8713                 if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
8714                         bool ycbcr420_enabled = tmp & PIPEMISC_YUV420_ENABLE;
8715                         bool blend = tmp & PIPEMISC_YUV420_MODE_FULL_BLEND;
8716
8717                         if (ycbcr420_enabled) {
8718                                 /* We support 4:2:0 in full blend mode only */
8719                                 if (!blend)
8720                                         output = INTEL_OUTPUT_FORMAT_INVALID;
8721                                 else if (!(IS_GEMINILAKE(dev_priv) ||
8722                                            INTEL_GEN(dev_priv) >= 10))
8723                                         output = INTEL_OUTPUT_FORMAT_INVALID;
8724                                 else
8725                                         output = INTEL_OUTPUT_FORMAT_YCBCR420;
8726                         } else {
8727                                 /*
8728                                  * Currently there is no interface defined to
8729                                  * check user preference between RGB/YCBCR444
8730                                  * or YCBCR420. So the only possible case for
8731                                  * YCBCR444 usage is driving YCBCR420 output
8732                                  * with LSPCON, when pipe is configured for
8733                                  * YCBCR444 output and LSPCON takes care of
8734                                  * downsampling it.
8735                                  */
8736                                 pipe_config->lspcon_downsampling = true;
8737                                 output = INTEL_OUTPUT_FORMAT_YCBCR444;
8738                         }
8739                 }
8740         }
8741
8742         pipe_config->output_format = output;
8743 }
8744
8745 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
8746 {
8747         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8748         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8749         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8750         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8751         u32 tmp;
8752
8753         tmp = I915_READ(DSPCNTR(i9xx_plane));
8754
8755         if (tmp & DISPPLANE_GAMMA_ENABLE)
8756                 crtc_state->gamma_enable = true;
8757
8758         if (!HAS_GMCH(dev_priv) &&
8759             tmp & DISPPLANE_PIPE_CSC_ENABLE)
8760                 crtc_state->csc_enable = true;
8761 }
8762
8763 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
8764                                  struct intel_crtc_state *pipe_config)
8765 {
8766         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8767         enum intel_display_power_domain power_domain;
8768         intel_wakeref_t wakeref;
8769         u32 tmp;
8770         bool ret;
8771
8772         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
8773         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
8774         if (!wakeref)
8775                 return false;
8776
8777         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
8778         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8779         pipe_config->shared_dpll = NULL;
8780
8781         ret = false;
8782
8783         tmp = I915_READ(PIPECONF(crtc->pipe));
8784         if (!(tmp & PIPECONF_ENABLE))
8785                 goto out;
8786
8787         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8788             IS_CHERRYVIEW(dev_priv)) {
8789                 switch (tmp & PIPECONF_BPC_MASK) {
8790                 case PIPECONF_6BPC:
8791                         pipe_config->pipe_bpp = 18;
8792                         break;
8793                 case PIPECONF_8BPC:
8794                         pipe_config->pipe_bpp = 24;
8795                         break;
8796                 case PIPECONF_10BPC:
8797                         pipe_config->pipe_bpp = 30;
8798                         break;
8799                 default:
8800                         break;
8801                 }
8802         }
8803
8804         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8805             (tmp & PIPECONF_COLOR_RANGE_SELECT))
8806                 pipe_config->limited_color_range = true;
8807
8808         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
8809                 PIPECONF_GAMMA_MODE_SHIFT;
8810
8811         if (IS_CHERRYVIEW(dev_priv))
8812                 pipe_config->cgm_mode = I915_READ(CGM_PIPE_MODE(crtc->pipe));
8813
8814         i9xx_get_pipe_color_config(pipe_config);
8815         intel_color_get_config(pipe_config);
8816
8817         if (INTEL_GEN(dev_priv) < 4)
8818                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
8819
8820         intel_get_pipe_timings(crtc, pipe_config);
8821         intel_get_pipe_src_size(crtc, pipe_config);
8822
8823         i9xx_get_pfit_config(crtc, pipe_config);
8824
8825         if (INTEL_GEN(dev_priv) >= 4) {
8826                 /* No way to read it out on pipes B and C */
8827                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
8828                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
8829                 else
8830                         tmp = I915_READ(DPLL_MD(crtc->pipe));
8831                 pipe_config->pixel_multiplier =
8832                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8833                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8834                 pipe_config->dpll_hw_state.dpll_md = tmp;
8835         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8836                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8837                 tmp = I915_READ(DPLL(crtc->pipe));
8838                 pipe_config->pixel_multiplier =
8839                         ((tmp & SDVO_MULTIPLIER_MASK)
8840                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8841         } else {
8842                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8843                  * port and will be fixed up in the encoder->get_config
8844                  * function. */
8845                 pipe_config->pixel_multiplier = 1;
8846         }
8847         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8848         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
8849                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8850                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8851         } else {
8852                 /* Mask out read-only status bits. */
8853                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8854                                                      DPLL_PORTC_READY_MASK |
8855                                                      DPLL_PORTB_READY_MASK);
8856         }
8857
8858         if (IS_CHERRYVIEW(dev_priv))
8859                 chv_crtc_clock_get(crtc, pipe_config);
8860         else if (IS_VALLEYVIEW(dev_priv))
8861                 vlv_crtc_clock_get(crtc, pipe_config);
8862         else
8863                 i9xx_crtc_clock_get(crtc, pipe_config);
8864
8865         /*
8866          * Normally the dotclock is filled in by the encoder .get_config()
8867          * but in case the pipe is enabled w/o any ports we need a sane
8868          * default.
8869          */
8870         pipe_config->base.adjusted_mode.crtc_clock =
8871                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8872
8873         ret = true;
8874
8875 out:
8876         intel_display_power_put(dev_priv, power_domain, wakeref);
8877
8878         return ret;
8879 }
8880
8881 static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
8882 {
8883         struct intel_encoder *encoder;
8884         int i;
8885         u32 val, final;
8886         bool has_lvds = false;
8887         bool has_cpu_edp = false;
8888         bool has_panel = false;
8889         bool has_ck505 = false;
8890         bool can_ssc = false;
8891         bool using_ssc_source = false;
8892
8893         /* We need to take the global config into account */
8894         for_each_intel_encoder(&dev_priv->drm, encoder) {
8895                 switch (encoder->type) {
8896                 case INTEL_OUTPUT_LVDS:
8897                         has_panel = true;
8898                         has_lvds = true;
8899                         break;
8900                 case INTEL_OUTPUT_EDP:
8901                         has_panel = true;
8902                         if (encoder->port == PORT_A)
8903                                 has_cpu_edp = true;
8904                         break;
8905                 default:
8906                         break;
8907                 }
8908         }
8909
8910         if (HAS_PCH_IBX(dev_priv)) {
8911                 has_ck505 = dev_priv->vbt.display_clock_mode;
8912                 can_ssc = has_ck505;
8913         } else {
8914                 has_ck505 = false;
8915                 can_ssc = true;
8916         }
8917
8918         /* Check if any DPLLs are using the SSC source */
8919         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
8920                 u32 temp = I915_READ(PCH_DPLL(i));
8921
8922                 if (!(temp & DPLL_VCO_ENABLE))
8923                         continue;
8924
8925                 if ((temp & PLL_REF_INPUT_MASK) ==
8926                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
8927                         using_ssc_source = true;
8928                         break;
8929                 }
8930         }
8931
8932         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
8933                       has_panel, has_lvds, has_ck505, using_ssc_source);
8934
8935         /* Ironlake: try to setup display ref clock before DPLL
8936          * enabling. This is only under driver's control after
8937          * PCH B stepping, previous chipset stepping should be
8938          * ignoring this setting.
8939          */
8940         val = I915_READ(PCH_DREF_CONTROL);
8941
8942         /* As we must carefully and slowly disable/enable each source in turn,
8943          * compute the final state we want first and check if we need to
8944          * make any changes at all.
8945          */
8946         final = val;
8947         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8948         if (has_ck505)
8949                 final |= DREF_NONSPREAD_CK505_ENABLE;
8950         else
8951                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8952
8953         final &= ~DREF_SSC_SOURCE_MASK;
8954         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8955         final &= ~DREF_SSC1_ENABLE;
8956
8957         if (has_panel) {
8958                 final |= DREF_SSC_SOURCE_ENABLE;
8959
8960                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8961                         final |= DREF_SSC1_ENABLE;
8962
8963                 if (has_cpu_edp) {
8964                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8965                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8966                         else
8967                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8968                 } else
8969                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8970         } else if (using_ssc_source) {
8971                 final |= DREF_SSC_SOURCE_ENABLE;
8972                 final |= DREF_SSC1_ENABLE;
8973         }
8974
8975         if (final == val)
8976                 return;
8977
8978         /* Always enable nonspread source */
8979         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8980
8981         if (has_ck505)
8982                 val |= DREF_NONSPREAD_CK505_ENABLE;
8983         else
8984                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8985
8986         if (has_panel) {
8987                 val &= ~DREF_SSC_SOURCE_MASK;
8988                 val |= DREF_SSC_SOURCE_ENABLE;
8989
8990                 /* SSC must be turned on before enabling the CPU output  */
8991                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8992                         DRM_DEBUG_KMS("Using SSC on panel\n");
8993                         val |= DREF_SSC1_ENABLE;
8994                 } else
8995                         val &= ~DREF_SSC1_ENABLE;
8996
8997                 /* Get SSC going before enabling the outputs */
8998                 I915_WRITE(PCH_DREF_CONTROL, val);
8999                 POSTING_READ(PCH_DREF_CONTROL);
9000                 udelay(200);
9001
9002                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9003
9004                 /* Enable CPU source on CPU attached eDP */
9005                 if (has_cpu_edp) {
9006                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9007                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
9008                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9009                         } else
9010                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9011                 } else
9012                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9013
9014                 I915_WRITE(PCH_DREF_CONTROL, val);
9015                 POSTING_READ(PCH_DREF_CONTROL);
9016                 udelay(200);
9017         } else {
9018                 DRM_DEBUG_KMS("Disabling CPU source output\n");
9019
9020                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9021
9022                 /* Turn off CPU output */
9023                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9024
9025                 I915_WRITE(PCH_DREF_CONTROL, val);
9026                 POSTING_READ(PCH_DREF_CONTROL);
9027                 udelay(200);
9028
9029                 if (!using_ssc_source) {
9030                         DRM_DEBUG_KMS("Disabling SSC source\n");
9031
9032                         /* Turn off the SSC source */
9033                         val &= ~DREF_SSC_SOURCE_MASK;
9034                         val |= DREF_SSC_SOURCE_DISABLE;
9035
9036                         /* Turn off SSC1 */
9037                         val &= ~DREF_SSC1_ENABLE;
9038
9039                         I915_WRITE(PCH_DREF_CONTROL, val);
9040                         POSTING_READ(PCH_DREF_CONTROL);
9041                         udelay(200);
9042                 }
9043         }
9044
9045         BUG_ON(val != final);
9046 }
9047
9048 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
9049 {
9050         u32 tmp;
9051
9052         tmp = I915_READ(SOUTH_CHICKEN2);
9053         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
9054         I915_WRITE(SOUTH_CHICKEN2, tmp);
9055
9056         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
9057                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
9058                 DRM_ERROR("FDI mPHY reset assert timeout\n");
9059
9060         tmp = I915_READ(SOUTH_CHICKEN2);
9061         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
9062         I915_WRITE(SOUTH_CHICKEN2, tmp);
9063
9064         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
9065                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
9066                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
9067 }
9068
9069 /* WaMPhyProgramming:hsw */
9070 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
9071 {
9072         u32 tmp;
9073
9074         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
9075         tmp &= ~(0xFF << 24);
9076         tmp |= (0x12 << 24);
9077         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
9078
9079         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
9080         tmp |= (1 << 11);
9081         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
9082
9083         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
9084         tmp |= (1 << 11);
9085         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
9086
9087         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
9088         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9089         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
9090
9091         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
9092         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9093         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
9094
9095         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
9096         tmp &= ~(7 << 13);
9097         tmp |= (5 << 13);
9098         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
9099
9100         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
9101         tmp &= ~(7 << 13);
9102         tmp |= (5 << 13);
9103         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
9104
9105         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
9106         tmp &= ~0xFF;
9107         tmp |= 0x1C;
9108         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
9109
9110         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
9111         tmp &= ~0xFF;
9112         tmp |= 0x1C;
9113         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
9114
9115         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
9116         tmp &= ~(0xFF << 16);
9117         tmp |= (0x1C << 16);
9118         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
9119
9120         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
9121         tmp &= ~(0xFF << 16);
9122         tmp |= (0x1C << 16);
9123         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
9124
9125         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
9126         tmp |= (1 << 27);
9127         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
9128
9129         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
9130         tmp |= (1 << 27);
9131         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
9132
9133         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
9134         tmp &= ~(0xF << 28);
9135         tmp |= (4 << 28);
9136         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
9137
9138         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
9139         tmp &= ~(0xF << 28);
9140         tmp |= (4 << 28);
9141         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
9142 }
9143
9144 /* Implements 3 different sequences from BSpec chapter "Display iCLK
9145  * Programming" based on the parameters passed:
9146  * - Sequence to enable CLKOUT_DP
9147  * - Sequence to enable CLKOUT_DP without spread
9148  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
9149  */
9150 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
9151                                  bool with_spread, bool with_fdi)
9152 {
9153         u32 reg, tmp;
9154
9155         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
9156                 with_spread = true;
9157         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
9158             with_fdi, "LP PCH doesn't have FDI\n"))
9159                 with_fdi = false;
9160
9161         mutex_lock(&dev_priv->sb_lock);
9162
9163         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9164         tmp &= ~SBI_SSCCTL_DISABLE;
9165         tmp |= SBI_SSCCTL_PATHALT;
9166         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9167
9168         udelay(24);
9169
9170         if (with_spread) {
9171                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9172                 tmp &= ~SBI_SSCCTL_PATHALT;
9173                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9174
9175                 if (with_fdi) {
9176                         lpt_reset_fdi_mphy(dev_priv);
9177                         lpt_program_fdi_mphy(dev_priv);
9178                 }
9179         }
9180
9181         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9182         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9183         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9184         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9185
9186         mutex_unlock(&dev_priv->sb_lock);
9187 }
9188
9189 /* Sequence to disable CLKOUT_DP */
9190 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
9191 {
9192         u32 reg, tmp;
9193
9194         mutex_lock(&dev_priv->sb_lock);
9195
9196         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9197         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9198         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9199         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9200
9201         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9202         if (!(tmp & SBI_SSCCTL_DISABLE)) {
9203                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
9204                         tmp |= SBI_SSCCTL_PATHALT;
9205                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9206                         udelay(32);
9207                 }
9208                 tmp |= SBI_SSCCTL_DISABLE;
9209                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9210         }
9211
9212         mutex_unlock(&dev_priv->sb_lock);
9213 }
9214
9215 #define BEND_IDX(steps) ((50 + (steps)) / 5)
9216
9217 static const u16 sscdivintphase[] = {
9218         [BEND_IDX( 50)] = 0x3B23,
9219         [BEND_IDX( 45)] = 0x3B23,
9220         [BEND_IDX( 40)] = 0x3C23,
9221         [BEND_IDX( 35)] = 0x3C23,
9222         [BEND_IDX( 30)] = 0x3D23,
9223         [BEND_IDX( 25)] = 0x3D23,
9224         [BEND_IDX( 20)] = 0x3E23,
9225         [BEND_IDX( 15)] = 0x3E23,
9226         [BEND_IDX( 10)] = 0x3F23,
9227         [BEND_IDX(  5)] = 0x3F23,
9228         [BEND_IDX(  0)] = 0x0025,
9229         [BEND_IDX( -5)] = 0x0025,
9230         [BEND_IDX(-10)] = 0x0125,
9231         [BEND_IDX(-15)] = 0x0125,
9232         [BEND_IDX(-20)] = 0x0225,
9233         [BEND_IDX(-25)] = 0x0225,
9234         [BEND_IDX(-30)] = 0x0325,
9235         [BEND_IDX(-35)] = 0x0325,
9236         [BEND_IDX(-40)] = 0x0425,
9237         [BEND_IDX(-45)] = 0x0425,
9238         [BEND_IDX(-50)] = 0x0525,
9239 };
9240
9241 /*
9242  * Bend CLKOUT_DP
9243  * steps -50 to 50 inclusive, in steps of 5
9244  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
9245  * change in clock period = -(steps / 10) * 5.787 ps
9246  */
9247 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
9248 {
9249         u32 tmp;
9250         int idx = BEND_IDX(steps);
9251
9252         if (WARN_ON(steps % 5 != 0))
9253                 return;
9254
9255         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
9256                 return;
9257
9258         mutex_lock(&dev_priv->sb_lock);
9259
9260         if (steps % 10 != 0)
9261                 tmp = 0xAAAAAAAB;
9262         else
9263                 tmp = 0x00000000;
9264         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
9265
9266         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
9267         tmp &= 0xffff0000;
9268         tmp |= sscdivintphase[idx];
9269         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
9270
9271         mutex_unlock(&dev_priv->sb_lock);
9272 }
9273
9274 #undef BEND_IDX
9275
9276 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
9277 {
9278         u32 fuse_strap = I915_READ(FUSE_STRAP);
9279         u32 ctl = I915_READ(SPLL_CTL);
9280
9281         if ((ctl & SPLL_PLL_ENABLE) == 0)
9282                 return false;
9283
9284         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
9285             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9286                 return true;
9287
9288         if (IS_BROADWELL(dev_priv) &&
9289             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
9290                 return true;
9291
9292         return false;
9293 }
9294
9295 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
9296                                enum intel_dpll_id id)
9297 {
9298         u32 fuse_strap = I915_READ(FUSE_STRAP);
9299         u32 ctl = I915_READ(WRPLL_CTL(id));
9300
9301         if ((ctl & WRPLL_PLL_ENABLE) == 0)
9302                 return false;
9303
9304         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
9305                 return true;
9306
9307         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
9308             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
9309             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9310                 return true;
9311
9312         return false;
9313 }
9314
9315 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
9316 {
9317         struct intel_encoder *encoder;
9318         bool pch_ssc_in_use = false;
9319         bool has_fdi = false;
9320
9321         for_each_intel_encoder(&dev_priv->drm, encoder) {
9322                 switch (encoder->type) {
9323                 case INTEL_OUTPUT_ANALOG:
9324                         has_fdi = true;
9325                         break;
9326                 default:
9327                         break;
9328                 }
9329         }
9330
9331         /*
9332          * The BIOS may have decided to use the PCH SSC
9333          * reference so we must not disable it until the
9334          * relevant PLLs have stopped relying on it. We'll
9335          * just leave the PCH SSC reference enabled in case
9336          * any active PLL is using it. It will get disabled
9337          * after runtime suspend if we don't have FDI.
9338          *
9339          * TODO: Move the whole reference clock handling
9340          * to the modeset sequence proper so that we can
9341          * actually enable/disable/reconfigure these things
9342          * safely. To do that we need to introduce a real
9343          * clock hierarchy. That would also allow us to do
9344          * clock bending finally.
9345          */
9346         if (spll_uses_pch_ssc(dev_priv)) {
9347                 DRM_DEBUG_KMS("SPLL using PCH SSC\n");
9348                 pch_ssc_in_use = true;
9349         }
9350
9351         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
9352                 DRM_DEBUG_KMS("WRPLL1 using PCH SSC\n");
9353                 pch_ssc_in_use = true;
9354         }
9355
9356         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
9357                 DRM_DEBUG_KMS("WRPLL2 using PCH SSC\n");
9358                 pch_ssc_in_use = true;
9359         }
9360
9361         if (pch_ssc_in_use)
9362                 return;
9363
9364         if (has_fdi) {
9365                 lpt_bend_clkout_dp(dev_priv, 0);
9366                 lpt_enable_clkout_dp(dev_priv, true, true);
9367         } else {
9368                 lpt_disable_clkout_dp(dev_priv);
9369         }
9370 }
9371
9372 /*
9373  * Initialize reference clocks when the driver loads
9374  */
9375 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
9376 {
9377         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
9378                 ironlake_init_pch_refclk(dev_priv);
9379         else if (HAS_PCH_LPT(dev_priv))
9380                 lpt_init_pch_refclk(dev_priv);
9381 }
9382
9383 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state)
9384 {
9385         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9386         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9387         enum pipe pipe = crtc->pipe;
9388         u32 val;
9389
9390         val = 0;
9391
9392         switch (crtc_state->pipe_bpp) {
9393         case 18:
9394                 val |= PIPECONF_6BPC;
9395                 break;
9396         case 24:
9397                 val |= PIPECONF_8BPC;
9398                 break;
9399         case 30:
9400                 val |= PIPECONF_10BPC;
9401                 break;
9402         case 36:
9403                 val |= PIPECONF_12BPC;
9404                 break;
9405         default:
9406                 /* Case prevented by intel_choose_pipe_bpp_dither. */
9407                 BUG();
9408         }
9409
9410         if (crtc_state->dither)
9411                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9412
9413         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9414                 val |= PIPECONF_INTERLACED_ILK;
9415         else
9416                 val |= PIPECONF_PROGRESSIVE;
9417
9418         if (crtc_state->limited_color_range)
9419                 val |= PIPECONF_COLOR_RANGE_SELECT;
9420
9421         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
9422
9423         I915_WRITE(PIPECONF(pipe), val);
9424         POSTING_READ(PIPECONF(pipe));
9425 }
9426
9427 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state)
9428 {
9429         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9430         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9431         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
9432         u32 val = 0;
9433
9434         if (IS_HASWELL(dev_priv) && crtc_state->dither)
9435                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9436
9437         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9438                 val |= PIPECONF_INTERLACED_ILK;
9439         else
9440                 val |= PIPECONF_PROGRESSIVE;
9441
9442         I915_WRITE(PIPECONF(cpu_transcoder), val);
9443         POSTING_READ(PIPECONF(cpu_transcoder));
9444 }
9445
9446 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
9447 {
9448         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9449         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9450         u32 val = 0;
9451
9452         switch (crtc_state->pipe_bpp) {
9453         case 18:
9454                 val |= PIPEMISC_DITHER_6_BPC;
9455                 break;
9456         case 24:
9457                 val |= PIPEMISC_DITHER_8_BPC;
9458                 break;
9459         case 30:
9460                 val |= PIPEMISC_DITHER_10_BPC;
9461                 break;
9462         case 36:
9463                 val |= PIPEMISC_DITHER_12_BPC;
9464                 break;
9465         default:
9466                 MISSING_CASE(crtc_state->pipe_bpp);
9467                 break;
9468         }
9469
9470         if (crtc_state->dither)
9471                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
9472
9473         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
9474             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
9475                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
9476
9477         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
9478                 val |= PIPEMISC_YUV420_ENABLE |
9479                         PIPEMISC_YUV420_MODE_FULL_BLEND;
9480
9481         if (INTEL_GEN(dev_priv) >= 11 &&
9482             (crtc_state->active_planes & ~(icl_hdr_plane_mask() |
9483                                            BIT(PLANE_CURSOR))) == 0)
9484                 val |= PIPEMISC_HDR_MODE_PRECISION;
9485
9486         I915_WRITE(PIPEMISC(crtc->pipe), val);
9487 }
9488
9489 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
9490 {
9491         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9492         u32 tmp;
9493
9494         tmp = I915_READ(PIPEMISC(crtc->pipe));
9495
9496         switch (tmp & PIPEMISC_DITHER_BPC_MASK) {
9497         case PIPEMISC_DITHER_6_BPC:
9498                 return 18;
9499         case PIPEMISC_DITHER_8_BPC:
9500                 return 24;
9501         case PIPEMISC_DITHER_10_BPC:
9502                 return 30;
9503         case PIPEMISC_DITHER_12_BPC:
9504                 return 36;
9505         default:
9506                 MISSING_CASE(tmp);
9507                 return 0;
9508         }
9509 }
9510
9511 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
9512 {
9513         /*
9514          * Account for spread spectrum to avoid
9515          * oversubscribing the link. Max center spread
9516          * is 2.5%; use 5% for safety's sake.
9517          */
9518         u32 bps = target_clock * bpp * 21 / 20;
9519         return DIV_ROUND_UP(bps, link_bw * 8);
9520 }
9521
9522 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
9523 {
9524         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
9525 }
9526
9527 static void ironlake_compute_dpll(struct intel_crtc *crtc,
9528                                   struct intel_crtc_state *crtc_state,
9529                                   struct dpll *reduced_clock)
9530 {
9531         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9532         u32 dpll, fp, fp2;
9533         int factor;
9534
9535         /* Enable autotuning of the PLL clock (if permissible) */
9536         factor = 21;
9537         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9538                 if ((intel_panel_use_ssc(dev_priv) &&
9539                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
9540                     (HAS_PCH_IBX(dev_priv) &&
9541                      intel_is_dual_link_lvds(dev_priv)))
9542                         factor = 25;
9543         } else if (crtc_state->sdvo_tv_clock) {
9544                 factor = 20;
9545         }
9546
9547         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
9548
9549         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
9550                 fp |= FP_CB_TUNE;
9551
9552         if (reduced_clock) {
9553                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
9554
9555                 if (reduced_clock->m < factor * reduced_clock->n)
9556                         fp2 |= FP_CB_TUNE;
9557         } else {
9558                 fp2 = fp;
9559         }
9560
9561         dpll = 0;
9562
9563         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
9564                 dpll |= DPLLB_MODE_LVDS;
9565         else
9566                 dpll |= DPLLB_MODE_DAC_SERIAL;
9567
9568         dpll |= (crtc_state->pixel_multiplier - 1)
9569                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
9570
9571         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
9572             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
9573                 dpll |= DPLL_SDVO_HIGH_SPEED;
9574
9575         if (intel_crtc_has_dp_encoder(crtc_state))
9576                 dpll |= DPLL_SDVO_HIGH_SPEED;
9577
9578         /*
9579          * The high speed IO clock is only really required for
9580          * SDVO/HDMI/DP, but we also enable it for CRT to make it
9581          * possible to share the DPLL between CRT and HDMI. Enabling
9582          * the clock needlessly does no real harm, except use up a
9583          * bit of power potentially.
9584          *
9585          * We'll limit this to IVB with 3 pipes, since it has only two
9586          * DPLLs and so DPLL sharing is the only way to get three pipes
9587          * driving PCH ports at the same time. On SNB we could do this,
9588          * and potentially avoid enabling the second DPLL, but it's not
9589          * clear if it''s a win or loss power wise. No point in doing
9590          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
9591          */
9592         if (INTEL_INFO(dev_priv)->num_pipes == 3 &&
9593             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
9594                 dpll |= DPLL_SDVO_HIGH_SPEED;
9595
9596         /* compute bitmask from p1 value */
9597         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
9598         /* also FPA1 */
9599         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
9600
9601         switch (crtc_state->dpll.p2) {
9602         case 5:
9603                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
9604                 break;
9605         case 7:
9606                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
9607                 break;
9608         case 10:
9609                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
9610                 break;
9611         case 14:
9612                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
9613                 break;
9614         }
9615
9616         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
9617             intel_panel_use_ssc(dev_priv))
9618                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
9619         else
9620                 dpll |= PLL_REF_INPUT_DREFCLK;
9621
9622         dpll |= DPLL_VCO_ENABLE;
9623
9624         crtc_state->dpll_hw_state.dpll = dpll;
9625         crtc_state->dpll_hw_state.fp0 = fp;
9626         crtc_state->dpll_hw_state.fp1 = fp2;
9627 }
9628
9629 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
9630                                        struct intel_crtc_state *crtc_state)
9631 {
9632         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9633         struct intel_atomic_state *state =
9634                 to_intel_atomic_state(crtc_state->base.state);
9635         const struct intel_limit *limit;
9636         int refclk = 120000;
9637
9638         memset(&crtc_state->dpll_hw_state, 0,
9639                sizeof(crtc_state->dpll_hw_state));
9640
9641         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
9642         if (!crtc_state->has_pch_encoder)
9643                 return 0;
9644
9645         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9646                 if (intel_panel_use_ssc(dev_priv)) {
9647                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
9648                                       dev_priv->vbt.lvds_ssc_freq);
9649                         refclk = dev_priv->vbt.lvds_ssc_freq;
9650                 }
9651
9652                 if (intel_is_dual_link_lvds(dev_priv)) {
9653                         if (refclk == 100000)
9654                                 limit = &intel_limits_ironlake_dual_lvds_100m;
9655                         else
9656                                 limit = &intel_limits_ironlake_dual_lvds;
9657                 } else {
9658                         if (refclk == 100000)
9659                                 limit = &intel_limits_ironlake_single_lvds_100m;
9660                         else
9661                                 limit = &intel_limits_ironlake_single_lvds;
9662                 }
9663         } else {
9664                 limit = &intel_limits_ironlake_dac;
9665         }
9666
9667         if (!crtc_state->clock_set &&
9668             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9669                                 refclk, NULL, &crtc_state->dpll)) {
9670                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
9671                 return -EINVAL;
9672         }
9673
9674         ironlake_compute_dpll(crtc, crtc_state, NULL);
9675
9676         if (!intel_reserve_shared_dplls(state, crtc, NULL)) {
9677                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
9678                               pipe_name(crtc->pipe));
9679                 return -EINVAL;
9680         }
9681
9682         return 0;
9683 }
9684
9685 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
9686                                          struct intel_link_m_n *m_n)
9687 {
9688         struct drm_device *dev = crtc->base.dev;
9689         struct drm_i915_private *dev_priv = to_i915(dev);
9690         enum pipe pipe = crtc->pipe;
9691
9692         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
9693         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
9694         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
9695                 & ~TU_SIZE_MASK;
9696         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
9697         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
9698                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9699 }
9700
9701 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
9702                                          enum transcoder transcoder,
9703                                          struct intel_link_m_n *m_n,
9704                                          struct intel_link_m_n *m2_n2)
9705 {
9706         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9707         enum pipe pipe = crtc->pipe;
9708
9709         if (INTEL_GEN(dev_priv) >= 5) {
9710                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
9711                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
9712                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
9713                         & ~TU_SIZE_MASK;
9714                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
9715                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
9716                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9717
9718                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
9719                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
9720                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
9721                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
9722                                         & ~TU_SIZE_MASK;
9723                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
9724                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
9725                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9726                 }
9727         } else {
9728                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
9729                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
9730                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
9731                         & ~TU_SIZE_MASK;
9732                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
9733                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
9734                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9735         }
9736 }
9737
9738 void intel_dp_get_m_n(struct intel_crtc *crtc,
9739                       struct intel_crtc_state *pipe_config)
9740 {
9741         if (pipe_config->has_pch_encoder)
9742                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
9743         else
9744                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9745                                              &pipe_config->dp_m_n,
9746                                              &pipe_config->dp_m2_n2);
9747 }
9748
9749 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
9750                                         struct intel_crtc_state *pipe_config)
9751 {
9752         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9753                                      &pipe_config->fdi_m_n, NULL);
9754 }
9755
9756 static void skylake_get_pfit_config(struct intel_crtc *crtc,
9757                                     struct intel_crtc_state *pipe_config)
9758 {
9759         struct drm_device *dev = crtc->base.dev;
9760         struct drm_i915_private *dev_priv = to_i915(dev);
9761         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
9762         u32 ps_ctrl = 0;
9763         int id = -1;
9764         int i;
9765
9766         /* find scaler attached to this pipe */
9767         for (i = 0; i < crtc->num_scalers; i++) {
9768                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
9769                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
9770                         id = i;
9771                         pipe_config->pch_pfit.enabled = true;
9772                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
9773                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
9774                         scaler_state->scalers[i].in_use = true;
9775                         break;
9776                 }
9777         }
9778
9779         scaler_state->scaler_id = id;
9780         if (id >= 0) {
9781                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
9782         } else {
9783                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
9784         }
9785 }
9786
9787 static void
9788 skylake_get_initial_plane_config(struct intel_crtc *crtc,
9789                                  struct intel_initial_plane_config *plane_config)
9790 {
9791         struct drm_device *dev = crtc->base.dev;
9792         struct drm_i915_private *dev_priv = to_i915(dev);
9793         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
9794         enum plane_id plane_id = plane->id;
9795         enum pipe pipe;
9796         u32 val, base, offset, stride_mult, tiling, alpha;
9797         int fourcc, pixel_format;
9798         unsigned int aligned_height;
9799         struct drm_framebuffer *fb;
9800         struct intel_framebuffer *intel_fb;
9801
9802         if (!plane->get_hw_state(plane, &pipe))
9803                 return;
9804
9805         WARN_ON(pipe != crtc->pipe);
9806
9807         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9808         if (!intel_fb) {
9809                 DRM_DEBUG_KMS("failed to alloc fb\n");
9810                 return;
9811         }
9812
9813         fb = &intel_fb->base;
9814
9815         fb->dev = dev;
9816
9817         val = I915_READ(PLANE_CTL(pipe, plane_id));
9818
9819         if (INTEL_GEN(dev_priv) >= 11)
9820                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
9821         else
9822                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
9823
9824         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
9825                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
9826                 alpha &= PLANE_COLOR_ALPHA_MASK;
9827         } else {
9828                 alpha = val & PLANE_CTL_ALPHA_MASK;
9829         }
9830
9831         fourcc = skl_format_to_fourcc(pixel_format,
9832                                       val & PLANE_CTL_ORDER_RGBX, alpha);
9833         fb->format = drm_format_info(fourcc);
9834
9835         tiling = val & PLANE_CTL_TILED_MASK;
9836         switch (tiling) {
9837         case PLANE_CTL_TILED_LINEAR:
9838                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
9839                 break;
9840         case PLANE_CTL_TILED_X:
9841                 plane_config->tiling = I915_TILING_X;
9842                 fb->modifier = I915_FORMAT_MOD_X_TILED;
9843                 break;
9844         case PLANE_CTL_TILED_Y:
9845                 plane_config->tiling = I915_TILING_Y;
9846                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
9847                         fb->modifier = I915_FORMAT_MOD_Y_TILED_CCS;
9848                 else
9849                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
9850                 break;
9851         case PLANE_CTL_TILED_YF:
9852                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
9853                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
9854                 else
9855                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
9856                 break;
9857         default:
9858                 MISSING_CASE(tiling);
9859                 goto error;
9860         }
9861
9862         /*
9863          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
9864          * while i915 HW rotation is clockwise, thats why this swapping.
9865          */
9866         switch (val & PLANE_CTL_ROTATE_MASK) {
9867         case PLANE_CTL_ROTATE_0:
9868                 plane_config->rotation = DRM_MODE_ROTATE_0;
9869                 break;
9870         case PLANE_CTL_ROTATE_90:
9871                 plane_config->rotation = DRM_MODE_ROTATE_270;
9872                 break;
9873         case PLANE_CTL_ROTATE_180:
9874                 plane_config->rotation = DRM_MODE_ROTATE_180;
9875                 break;
9876         case PLANE_CTL_ROTATE_270:
9877                 plane_config->rotation = DRM_MODE_ROTATE_90;
9878                 break;
9879         }
9880
9881         if (INTEL_GEN(dev_priv) >= 10 &&
9882             val & PLANE_CTL_FLIP_HORIZONTAL)
9883                 plane_config->rotation |= DRM_MODE_REFLECT_X;
9884
9885         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
9886         plane_config->base = base;
9887
9888         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
9889
9890         val = I915_READ(PLANE_SIZE(pipe, plane_id));
9891         fb->height = ((val >> 16) & 0xfff) + 1;
9892         fb->width = ((val >> 0) & 0x1fff) + 1;
9893
9894         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
9895         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
9896         fb->pitches[0] = (val & 0x3ff) * stride_mult;
9897
9898         aligned_height = intel_fb_align_height(fb, 0, fb->height);
9899
9900         plane_config->size = fb->pitches[0] * aligned_height;
9901
9902         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9903                       crtc->base.name, plane->base.name, fb->width, fb->height,
9904                       fb->format->cpp[0] * 8, base, fb->pitches[0],
9905                       plane_config->size);
9906
9907         plane_config->fb = intel_fb;
9908         return;
9909
9910 error:
9911         kfree(intel_fb);
9912 }
9913
9914 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
9915                                      struct intel_crtc_state *pipe_config)
9916 {
9917         struct drm_device *dev = crtc->base.dev;
9918         struct drm_i915_private *dev_priv = to_i915(dev);
9919         u32 tmp;
9920
9921         tmp = I915_READ(PF_CTL(crtc->pipe));
9922
9923         if (tmp & PF_ENABLE) {
9924                 pipe_config->pch_pfit.enabled = true;
9925                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
9926                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
9927
9928                 /* We currently do not free assignements of panel fitters on
9929                  * ivb/hsw (since we don't use the higher upscaling modes which
9930                  * differentiates them) so just WARN about this case for now. */
9931                 if (IS_GEN(dev_priv, 7)) {
9932                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9933                                 PF_PIPE_SEL_IVB(crtc->pipe));
9934                 }
9935         }
9936 }
9937
9938 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9939                                      struct intel_crtc_state *pipe_config)
9940 {
9941         struct drm_device *dev = crtc->base.dev;
9942         struct drm_i915_private *dev_priv = to_i915(dev);
9943         enum intel_display_power_domain power_domain;
9944         intel_wakeref_t wakeref;
9945         u32 tmp;
9946         bool ret;
9947
9948         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9949         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
9950         if (!wakeref)
9951                 return false;
9952
9953         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9954         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9955         pipe_config->shared_dpll = NULL;
9956
9957         ret = false;
9958         tmp = I915_READ(PIPECONF(crtc->pipe));
9959         if (!(tmp & PIPECONF_ENABLE))
9960                 goto out;
9961
9962         switch (tmp & PIPECONF_BPC_MASK) {
9963         case PIPECONF_6BPC:
9964                 pipe_config->pipe_bpp = 18;
9965                 break;
9966         case PIPECONF_8BPC:
9967                 pipe_config->pipe_bpp = 24;
9968                 break;
9969         case PIPECONF_10BPC:
9970                 pipe_config->pipe_bpp = 30;
9971                 break;
9972         case PIPECONF_12BPC:
9973                 pipe_config->pipe_bpp = 36;
9974                 break;
9975         default:
9976                 break;
9977         }
9978
9979         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9980                 pipe_config->limited_color_range = true;
9981
9982         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
9983                 PIPECONF_GAMMA_MODE_SHIFT;
9984
9985         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
9986
9987         i9xx_get_pipe_color_config(pipe_config);
9988         intel_color_get_config(pipe_config);
9989
9990         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9991                 struct intel_shared_dpll *pll;
9992                 enum intel_dpll_id pll_id;
9993
9994                 pipe_config->has_pch_encoder = true;
9995
9996                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
9997                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9998                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9999
10000                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10001
10002                 if (HAS_PCH_IBX(dev_priv)) {
10003                         /*
10004                          * The pipe->pch transcoder and pch transcoder->pll
10005                          * mapping is fixed.
10006                          */
10007                         pll_id = (enum intel_dpll_id) crtc->pipe;
10008                 } else {
10009                         tmp = I915_READ(PCH_DPLL_SEL);
10010                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
10011                                 pll_id = DPLL_ID_PCH_PLL_B;
10012                         else
10013                                 pll_id= DPLL_ID_PCH_PLL_A;
10014                 }
10015
10016                 pipe_config->shared_dpll =
10017                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
10018                 pll = pipe_config->shared_dpll;
10019
10020                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10021                                                 &pipe_config->dpll_hw_state));
10022
10023                 tmp = pipe_config->dpll_hw_state.dpll;
10024                 pipe_config->pixel_multiplier =
10025                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
10026                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
10027
10028                 ironlake_pch_clock_get(crtc, pipe_config);
10029         } else {
10030                 pipe_config->pixel_multiplier = 1;
10031         }
10032
10033         intel_get_pipe_timings(crtc, pipe_config);
10034         intel_get_pipe_src_size(crtc, pipe_config);
10035
10036         ironlake_get_pfit_config(crtc, pipe_config);
10037
10038         ret = true;
10039
10040 out:
10041         intel_display_power_put(dev_priv, power_domain, wakeref);
10042
10043         return ret;
10044 }
10045 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
10046                                       struct intel_crtc_state *crtc_state)
10047 {
10048         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10049         struct intel_atomic_state *state =
10050                 to_intel_atomic_state(crtc_state->base.state);
10051
10052         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
10053             INTEL_GEN(dev_priv) >= 11) {
10054                 struct intel_encoder *encoder =
10055                         intel_get_crtc_new_encoder(state, crtc_state);
10056
10057                 if (!intel_reserve_shared_dplls(state, crtc, encoder)) {
10058                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10059                                       pipe_name(crtc->pipe));
10060                         return -EINVAL;
10061                 }
10062         }
10063
10064         return 0;
10065 }
10066
10067 static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
10068                                    enum port port,
10069                                    struct intel_crtc_state *pipe_config)
10070 {
10071         enum intel_dpll_id id;
10072         u32 temp;
10073
10074         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
10075         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
10076
10077         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
10078                 return;
10079
10080         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10081 }
10082
10083 static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
10084                                 enum port port,
10085                                 struct intel_crtc_state *pipe_config)
10086 {
10087         enum phy phy = intel_port_to_phy(dev_priv, port);
10088         enum icl_port_dpll_id port_dpll_id;
10089         enum intel_dpll_id id;
10090         u32 temp;
10091
10092         if (intel_phy_is_combo(dev_priv, phy)) {
10093                 temp = I915_READ(ICL_DPCLKA_CFGCR0) &
10094                         ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
10095                 id = temp >> ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
10096                 port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10097         } else if (intel_phy_is_tc(dev_priv, phy)) {
10098                 u32 clk_sel = I915_READ(DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK;
10099
10100                 if (clk_sel == DDI_CLK_SEL_MG) {
10101                         id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv,
10102                                                                     port));
10103                         port_dpll_id = ICL_PORT_DPLL_MG_PHY;
10104                 } else {
10105                         WARN_ON(clk_sel < DDI_CLK_SEL_TBT_162);
10106                         id = DPLL_ID_ICL_TBTPLL;
10107                         port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10108                 }
10109         } else {
10110                 WARN(1, "Invalid port %x\n", port);
10111                 return;
10112         }
10113
10114         pipe_config->icl_port_dplls[port_dpll_id].pll =
10115                 intel_get_shared_dpll_by_id(dev_priv, id);
10116
10117         icl_set_active_port_dpll(pipe_config, port_dpll_id);
10118 }
10119
10120 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
10121                                 enum port port,
10122                                 struct intel_crtc_state *pipe_config)
10123 {
10124         enum intel_dpll_id id;
10125
10126         switch (port) {
10127         case PORT_A:
10128                 id = DPLL_ID_SKL_DPLL0;
10129                 break;
10130         case PORT_B:
10131                 id = DPLL_ID_SKL_DPLL1;
10132                 break;
10133         case PORT_C:
10134                 id = DPLL_ID_SKL_DPLL2;
10135                 break;
10136         default:
10137                 DRM_ERROR("Incorrect port type\n");
10138                 return;
10139         }
10140
10141         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10142 }
10143
10144 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
10145                                 enum port port,
10146                                 struct intel_crtc_state *pipe_config)
10147 {
10148         enum intel_dpll_id id;
10149         u32 temp;
10150
10151         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
10152         id = temp >> (port * 3 + 1);
10153
10154         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
10155                 return;
10156
10157         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10158 }
10159
10160 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
10161                                 enum port port,
10162                                 struct intel_crtc_state *pipe_config)
10163 {
10164         enum intel_dpll_id id;
10165         u32 ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
10166
10167         switch (ddi_pll_sel) {
10168         case PORT_CLK_SEL_WRPLL1:
10169                 id = DPLL_ID_WRPLL1;
10170                 break;
10171         case PORT_CLK_SEL_WRPLL2:
10172                 id = DPLL_ID_WRPLL2;
10173                 break;
10174         case PORT_CLK_SEL_SPLL:
10175                 id = DPLL_ID_SPLL;
10176                 break;
10177         case PORT_CLK_SEL_LCPLL_810:
10178                 id = DPLL_ID_LCPLL_810;
10179                 break;
10180         case PORT_CLK_SEL_LCPLL_1350:
10181                 id = DPLL_ID_LCPLL_1350;
10182                 break;
10183         case PORT_CLK_SEL_LCPLL_2700:
10184                 id = DPLL_ID_LCPLL_2700;
10185                 break;
10186         default:
10187                 MISSING_CASE(ddi_pll_sel);
10188                 /* fall through */
10189         case PORT_CLK_SEL_NONE:
10190                 return;
10191         }
10192
10193         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10194 }
10195
10196 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
10197                                      struct intel_crtc_state *pipe_config,
10198                                      u64 *power_domain_mask,
10199                                      intel_wakeref_t *wakerefs)
10200 {
10201         struct drm_device *dev = crtc->base.dev;
10202         struct drm_i915_private *dev_priv = to_i915(dev);
10203         enum intel_display_power_domain power_domain;
10204         unsigned long panel_transcoder_mask = 0;
10205         unsigned long enabled_panel_transcoders = 0;
10206         enum transcoder panel_transcoder;
10207         intel_wakeref_t wf;
10208         u32 tmp;
10209
10210         if (INTEL_GEN(dev_priv) >= 11)
10211                 panel_transcoder_mask |=
10212                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
10213
10214         if (HAS_TRANSCODER_EDP(dev_priv))
10215                 panel_transcoder_mask |= BIT(TRANSCODER_EDP);
10216
10217         /*
10218          * The pipe->transcoder mapping is fixed with the exception of the eDP
10219          * and DSI transcoders handled below.
10220          */
10221         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10222
10223         /*
10224          * XXX: Do intel_display_power_get_if_enabled before reading this (for
10225          * consistency and less surprising code; it's in always on power).
10226          */
10227         for_each_set_bit(panel_transcoder,
10228                          &panel_transcoder_mask,
10229                          ARRAY_SIZE(INTEL_INFO(dev_priv)->trans_offsets)) {
10230                 bool force_thru = false;
10231                 enum pipe trans_pipe;
10232
10233                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder));
10234                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
10235                         continue;
10236
10237                 /*
10238                  * Log all enabled ones, only use the first one.
10239                  *
10240                  * FIXME: This won't work for two separate DSI displays.
10241                  */
10242                 enabled_panel_transcoders |= BIT(panel_transcoder);
10243                 if (enabled_panel_transcoders != BIT(panel_transcoder))
10244                         continue;
10245
10246                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
10247                 default:
10248                         WARN(1, "unknown pipe linked to transcoder %s\n",
10249                              transcoder_name(panel_transcoder));
10250                         /* fall through */
10251                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
10252                         force_thru = true;
10253                         /* fall through */
10254                 case TRANS_DDI_EDP_INPUT_A_ON:
10255                         trans_pipe = PIPE_A;
10256                         break;
10257                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
10258                         trans_pipe = PIPE_B;
10259                         break;
10260                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
10261                         trans_pipe = PIPE_C;
10262                         break;
10263                 }
10264
10265                 if (trans_pipe == crtc->pipe) {
10266                         pipe_config->cpu_transcoder = panel_transcoder;
10267                         pipe_config->pch_pfit.force_thru = force_thru;
10268                 }
10269         }
10270
10271         /*
10272          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
10273          */
10274         WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
10275                 enabled_panel_transcoders != BIT(TRANSCODER_EDP));
10276
10277         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
10278         WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10279
10280         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10281         if (!wf)
10282                 return false;
10283
10284         wakerefs[power_domain] = wf;
10285         *power_domain_mask |= BIT_ULL(power_domain);
10286
10287         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10288
10289         return tmp & PIPECONF_ENABLE;
10290 }
10291
10292 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
10293                                          struct intel_crtc_state *pipe_config,
10294                                          u64 *power_domain_mask,
10295                                          intel_wakeref_t *wakerefs)
10296 {
10297         struct drm_device *dev = crtc->base.dev;
10298         struct drm_i915_private *dev_priv = to_i915(dev);
10299         enum intel_display_power_domain power_domain;
10300         enum transcoder cpu_transcoder;
10301         intel_wakeref_t wf;
10302         enum port port;
10303         u32 tmp;
10304
10305         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
10306                 if (port == PORT_A)
10307                         cpu_transcoder = TRANSCODER_DSI_A;
10308                 else
10309                         cpu_transcoder = TRANSCODER_DSI_C;
10310
10311                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10312                 WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10313
10314                 wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10315                 if (!wf)
10316                         continue;
10317
10318                 wakerefs[power_domain] = wf;
10319                 *power_domain_mask |= BIT_ULL(power_domain);
10320
10321                 /*
10322                  * The PLL needs to be enabled with a valid divider
10323                  * configuration, otherwise accessing DSI registers will hang
10324                  * the machine. See BSpec North Display Engine
10325                  * registers/MIPI[BXT]. We can break out here early, since we
10326                  * need the same DSI PLL to be enabled for both DSI ports.
10327                  */
10328                 if (!bxt_dsi_pll_is_enabled(dev_priv))
10329                         break;
10330
10331                 /* XXX: this works for video mode only */
10332                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
10333                 if (!(tmp & DPI_ENABLE))
10334                         continue;
10335
10336                 tmp = I915_READ(MIPI_CTRL(port));
10337                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
10338                         continue;
10339
10340                 pipe_config->cpu_transcoder = cpu_transcoder;
10341                 break;
10342         }
10343
10344         return transcoder_is_dsi(pipe_config->cpu_transcoder);
10345 }
10346
10347 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
10348                                        struct intel_crtc_state *pipe_config)
10349 {
10350         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10351         struct intel_shared_dpll *pll;
10352         enum port port;
10353         u32 tmp;
10354
10355         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
10356
10357         if (INTEL_GEN(dev_priv) >= 12)
10358                 port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10359         else
10360                 port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10361
10362         if (INTEL_GEN(dev_priv) >= 11)
10363                 icelake_get_ddi_pll(dev_priv, port, pipe_config);
10364         else if (IS_CANNONLAKE(dev_priv))
10365                 cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
10366         else if (IS_GEN9_BC(dev_priv))
10367                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
10368         else if (IS_GEN9_LP(dev_priv))
10369                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
10370         else
10371                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
10372
10373         pll = pipe_config->shared_dpll;
10374         if (pll) {
10375                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10376                                                 &pipe_config->dpll_hw_state));
10377         }
10378
10379         /*
10380          * Haswell has only FDI/PCH transcoder A. It is which is connected to
10381          * DDI E. So just check whether this pipe is wired to DDI E and whether
10382          * the PCH transcoder is on.
10383          */
10384         if (INTEL_GEN(dev_priv) < 9 &&
10385             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
10386                 pipe_config->has_pch_encoder = true;
10387
10388                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
10389                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10390                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10391
10392                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10393         }
10394 }
10395
10396 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
10397                                     struct intel_crtc_state *pipe_config)
10398 {
10399         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10400         intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf;
10401         enum intel_display_power_domain power_domain;
10402         u64 power_domain_mask;
10403         bool active;
10404
10405         intel_crtc_init_scalers(crtc, pipe_config);
10406
10407         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10408         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10409         if (!wf)
10410                 return false;
10411
10412         wakerefs[power_domain] = wf;
10413         power_domain_mask = BIT_ULL(power_domain);
10414
10415         pipe_config->shared_dpll = NULL;
10416
10417         active = hsw_get_transcoder_state(crtc, pipe_config,
10418                                           &power_domain_mask, wakerefs);
10419
10420         if (IS_GEN9_LP(dev_priv) &&
10421             bxt_get_dsi_transcoder_state(crtc, pipe_config,
10422                                          &power_domain_mask, wakerefs)) {
10423                 WARN_ON(active);
10424                 active = true;
10425         }
10426
10427         if (!active)
10428                 goto out;
10429
10430         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
10431             INTEL_GEN(dev_priv) >= 11) {
10432                 haswell_get_ddi_port_state(crtc, pipe_config);
10433                 intel_get_pipe_timings(crtc, pipe_config);
10434         }
10435
10436         intel_get_pipe_src_size(crtc, pipe_config);
10437         intel_get_crtc_ycbcr_config(crtc, pipe_config);
10438
10439         pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe));
10440
10441         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10442
10443         if (INTEL_GEN(dev_priv) >= 9) {
10444                 u32 tmp = I915_READ(SKL_BOTTOM_COLOR(crtc->pipe));
10445
10446                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
10447                         pipe_config->gamma_enable = true;
10448
10449                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
10450                         pipe_config->csc_enable = true;
10451         } else {
10452                 i9xx_get_pipe_color_config(pipe_config);
10453         }
10454
10455         intel_color_get_config(pipe_config);
10456
10457         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
10458         WARN_ON(power_domain_mask & BIT_ULL(power_domain));
10459
10460         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10461         if (wf) {
10462                 wakerefs[power_domain] = wf;
10463                 power_domain_mask |= BIT_ULL(power_domain);
10464
10465                 if (INTEL_GEN(dev_priv) >= 9)
10466                         skylake_get_pfit_config(crtc, pipe_config);
10467                 else
10468                         ironlake_get_pfit_config(crtc, pipe_config);
10469         }
10470
10471         if (hsw_crtc_supports_ips(crtc)) {
10472                 if (IS_HASWELL(dev_priv))
10473                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
10474                 else {
10475                         /*
10476                          * We cannot readout IPS state on broadwell, set to
10477                          * true so we can set it to a defined state on first
10478                          * commit.
10479                          */
10480                         pipe_config->ips_enabled = true;
10481                 }
10482         }
10483
10484         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
10485             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
10486                 pipe_config->pixel_multiplier =
10487                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
10488         } else {
10489                 pipe_config->pixel_multiplier = 1;
10490         }
10491
10492 out:
10493         for_each_power_domain(power_domain, power_domain_mask)
10494                 intel_display_power_put(dev_priv,
10495                                         power_domain, wakerefs[power_domain]);
10496
10497         return active;
10498 }
10499
10500 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
10501 {
10502         struct drm_i915_private *dev_priv =
10503                 to_i915(plane_state->base.plane->dev);
10504         const struct drm_framebuffer *fb = plane_state->base.fb;
10505         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
10506         u32 base;
10507
10508         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
10509                 base = obj->phys_handle->busaddr;
10510         else
10511                 base = intel_plane_ggtt_offset(plane_state);
10512
10513         base += plane_state->color_plane[0].offset;
10514
10515         /* ILK+ do this automagically */
10516         if (HAS_GMCH(dev_priv) &&
10517             plane_state->base.rotation & DRM_MODE_ROTATE_180)
10518                 base += (plane_state->base.crtc_h *
10519                          plane_state->base.crtc_w - 1) * fb->format->cpp[0];
10520
10521         return base;
10522 }
10523
10524 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
10525 {
10526         int x = plane_state->base.crtc_x;
10527         int y = plane_state->base.crtc_y;
10528         u32 pos = 0;
10529
10530         if (x < 0) {
10531                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10532                 x = -x;
10533         }
10534         pos |= x << CURSOR_X_SHIFT;
10535
10536         if (y < 0) {
10537                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10538                 y = -y;
10539         }
10540         pos |= y << CURSOR_Y_SHIFT;
10541
10542         return pos;
10543 }
10544
10545 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
10546 {
10547         const struct drm_mode_config *config =
10548                 &plane_state->base.plane->dev->mode_config;
10549         int width = plane_state->base.crtc_w;
10550         int height = plane_state->base.crtc_h;
10551
10552         return width > 0 && width <= config->cursor_width &&
10553                 height > 0 && height <= config->cursor_height;
10554 }
10555
10556 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
10557 {
10558         int src_x, src_y;
10559         u32 offset;
10560         int ret;
10561
10562         ret = intel_plane_compute_gtt(plane_state);
10563         if (ret)
10564                 return ret;
10565
10566         if (!plane_state->base.visible)
10567                 return 0;
10568
10569         src_x = plane_state->base.src_x >> 16;
10570         src_y = plane_state->base.src_y >> 16;
10571
10572         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
10573         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
10574                                                     plane_state, 0);
10575
10576         if (src_x != 0 || src_y != 0) {
10577                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
10578                 return -EINVAL;
10579         }
10580
10581         plane_state->color_plane[0].offset = offset;
10582
10583         return 0;
10584 }
10585
10586 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
10587                               struct intel_plane_state *plane_state)
10588 {
10589         const struct drm_framebuffer *fb = plane_state->base.fb;
10590         int ret;
10591
10592         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
10593                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
10594                 return -EINVAL;
10595         }
10596
10597         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
10598                                                   &crtc_state->base,
10599                                                   DRM_PLANE_HELPER_NO_SCALING,
10600                                                   DRM_PLANE_HELPER_NO_SCALING,
10601                                                   true, true);
10602         if (ret)
10603                 return ret;
10604
10605         ret = intel_cursor_check_surface(plane_state);
10606         if (ret)
10607                 return ret;
10608
10609         if (!plane_state->base.visible)
10610                 return 0;
10611
10612         ret = intel_plane_check_src_coordinates(plane_state);
10613         if (ret)
10614                 return ret;
10615
10616         return 0;
10617 }
10618
10619 static unsigned int
10620 i845_cursor_max_stride(struct intel_plane *plane,
10621                        u32 pixel_format, u64 modifier,
10622                        unsigned int rotation)
10623 {
10624         return 2048;
10625 }
10626
10627 static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
10628 {
10629         u32 cntl = 0;
10630
10631         if (crtc_state->gamma_enable)
10632                 cntl |= CURSOR_GAMMA_ENABLE;
10633
10634         return cntl;
10635 }
10636
10637 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
10638                            const struct intel_plane_state *plane_state)
10639 {
10640         return CURSOR_ENABLE |
10641                 CURSOR_FORMAT_ARGB |
10642                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
10643 }
10644
10645 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
10646 {
10647         int width = plane_state->base.crtc_w;
10648
10649         /*
10650          * 845g/865g are only limited by the width of their cursors,
10651          * the height is arbitrary up to the precision of the register.
10652          */
10653         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
10654 }
10655
10656 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
10657                              struct intel_plane_state *plane_state)
10658 {
10659         const struct drm_framebuffer *fb = plane_state->base.fb;
10660         int ret;
10661
10662         ret = intel_check_cursor(crtc_state, plane_state);
10663         if (ret)
10664                 return ret;
10665
10666         /* if we want to turn off the cursor ignore width and height */
10667         if (!fb)
10668                 return 0;
10669
10670         /* Check for which cursor types we support */
10671         if (!i845_cursor_size_ok(plane_state)) {
10672                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10673                           plane_state->base.crtc_w,
10674                           plane_state->base.crtc_h);
10675                 return -EINVAL;
10676         }
10677
10678         WARN_ON(plane_state->base.visible &&
10679                 plane_state->color_plane[0].stride != fb->pitches[0]);
10680
10681         switch (fb->pitches[0]) {
10682         case 256:
10683         case 512:
10684         case 1024:
10685         case 2048:
10686                 break;
10687         default:
10688                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
10689                               fb->pitches[0]);
10690                 return -EINVAL;
10691         }
10692
10693         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
10694
10695         return 0;
10696 }
10697
10698 static void i845_update_cursor(struct intel_plane *plane,
10699                                const struct intel_crtc_state *crtc_state,
10700                                const struct intel_plane_state *plane_state)
10701 {
10702         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10703         u32 cntl = 0, base = 0, pos = 0, size = 0;
10704         unsigned long irqflags;
10705
10706         if (plane_state && plane_state->base.visible) {
10707                 unsigned int width = plane_state->base.crtc_w;
10708                 unsigned int height = plane_state->base.crtc_h;
10709
10710                 cntl = plane_state->ctl |
10711                         i845_cursor_ctl_crtc(crtc_state);
10712
10713                 size = (height << 12) | width;
10714
10715                 base = intel_cursor_base(plane_state);
10716                 pos = intel_cursor_position(plane_state);
10717         }
10718
10719         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10720
10721         /* On these chipsets we can only modify the base/size/stride
10722          * whilst the cursor is disabled.
10723          */
10724         if (plane->cursor.base != base ||
10725             plane->cursor.size != size ||
10726             plane->cursor.cntl != cntl) {
10727                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
10728                 I915_WRITE_FW(CURBASE(PIPE_A), base);
10729                 I915_WRITE_FW(CURSIZE, size);
10730                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
10731                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
10732
10733                 plane->cursor.base = base;
10734                 plane->cursor.size = size;
10735                 plane->cursor.cntl = cntl;
10736         } else {
10737                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
10738         }
10739
10740         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10741 }
10742
10743 static void i845_disable_cursor(struct intel_plane *plane,
10744                                 const struct intel_crtc_state *crtc_state)
10745 {
10746         i845_update_cursor(plane, crtc_state, NULL);
10747 }
10748
10749 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
10750                                      enum pipe *pipe)
10751 {
10752         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10753         enum intel_display_power_domain power_domain;
10754         intel_wakeref_t wakeref;
10755         bool ret;
10756
10757         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
10758         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10759         if (!wakeref)
10760                 return false;
10761
10762         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
10763
10764         *pipe = PIPE_A;
10765
10766         intel_display_power_put(dev_priv, power_domain, wakeref);
10767
10768         return ret;
10769 }
10770
10771 static unsigned int
10772 i9xx_cursor_max_stride(struct intel_plane *plane,
10773                        u32 pixel_format, u64 modifier,
10774                        unsigned int rotation)
10775 {
10776         return plane->base.dev->mode_config.cursor_width * 4;
10777 }
10778
10779 static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
10780 {
10781         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
10782         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10783         u32 cntl = 0;
10784
10785         if (INTEL_GEN(dev_priv) >= 11)
10786                 return cntl;
10787
10788         if (crtc_state->gamma_enable)
10789                 cntl = MCURSOR_GAMMA_ENABLE;
10790
10791         if (crtc_state->csc_enable)
10792                 cntl |= MCURSOR_PIPE_CSC_ENABLE;
10793
10794         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10795                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
10796
10797         return cntl;
10798 }
10799
10800 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
10801                            const struct intel_plane_state *plane_state)
10802 {
10803         struct drm_i915_private *dev_priv =
10804                 to_i915(plane_state->base.plane->dev);
10805         u32 cntl = 0;
10806
10807         if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
10808                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
10809
10810         switch (plane_state->base.crtc_w) {
10811         case 64:
10812                 cntl |= MCURSOR_MODE_64_ARGB_AX;
10813                 break;
10814         case 128:
10815                 cntl |= MCURSOR_MODE_128_ARGB_AX;
10816                 break;
10817         case 256:
10818                 cntl |= MCURSOR_MODE_256_ARGB_AX;
10819                 break;
10820         default:
10821                 MISSING_CASE(plane_state->base.crtc_w);
10822                 return 0;
10823         }
10824
10825         if (plane_state->base.rotation & DRM_MODE_ROTATE_180)
10826                 cntl |= MCURSOR_ROTATE_180;
10827
10828         return cntl;
10829 }
10830
10831 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
10832 {
10833         struct drm_i915_private *dev_priv =
10834                 to_i915(plane_state->base.plane->dev);
10835         int width = plane_state->base.crtc_w;
10836         int height = plane_state->base.crtc_h;
10837
10838         if (!intel_cursor_size_ok(plane_state))
10839                 return false;
10840
10841         /* Cursor width is limited to a few power-of-two sizes */
10842         switch (width) {
10843         case 256:
10844         case 128:
10845         case 64:
10846                 break;
10847         default:
10848                 return false;
10849         }
10850
10851         /*
10852          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
10853          * height from 8 lines up to the cursor width, when the
10854          * cursor is not rotated. Everything else requires square
10855          * cursors.
10856          */
10857         if (HAS_CUR_FBC(dev_priv) &&
10858             plane_state->base.rotation & DRM_MODE_ROTATE_0) {
10859                 if (height < 8 || height > width)
10860                         return false;
10861         } else {
10862                 if (height != width)
10863                         return false;
10864         }
10865
10866         return true;
10867 }
10868
10869 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
10870                              struct intel_plane_state *plane_state)
10871 {
10872         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
10873         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10874         const struct drm_framebuffer *fb = plane_state->base.fb;
10875         enum pipe pipe = plane->pipe;
10876         int ret;
10877
10878         ret = intel_check_cursor(crtc_state, plane_state);
10879         if (ret)
10880                 return ret;
10881
10882         /* if we want to turn off the cursor ignore width and height */
10883         if (!fb)
10884                 return 0;
10885
10886         /* Check for which cursor types we support */
10887         if (!i9xx_cursor_size_ok(plane_state)) {
10888                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10889                           plane_state->base.crtc_w,
10890                           plane_state->base.crtc_h);
10891                 return -EINVAL;
10892         }
10893
10894         WARN_ON(plane_state->base.visible &&
10895                 plane_state->color_plane[0].stride != fb->pitches[0]);
10896
10897         if (fb->pitches[0] != plane_state->base.crtc_w * fb->format->cpp[0]) {
10898                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
10899                               fb->pitches[0], plane_state->base.crtc_w);
10900                 return -EINVAL;
10901         }
10902
10903         /*
10904          * There's something wrong with the cursor on CHV pipe C.
10905          * If it straddles the left edge of the screen then
10906          * moving it away from the edge or disabling it often
10907          * results in a pipe underrun, and often that can lead to
10908          * dead pipe (constant underrun reported, and it scans
10909          * out just a solid color). To recover from that, the
10910          * display power well must be turned off and on again.
10911          * Refuse the put the cursor into that compromised position.
10912          */
10913         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
10914             plane_state->base.visible && plane_state->base.crtc_x < 0) {
10915                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
10916                 return -EINVAL;
10917         }
10918
10919         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
10920
10921         return 0;
10922 }
10923
10924 static void i9xx_update_cursor(struct intel_plane *plane,
10925                                const struct intel_crtc_state *crtc_state,
10926                                const struct intel_plane_state *plane_state)
10927 {
10928         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10929         enum pipe pipe = plane->pipe;
10930         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
10931         unsigned long irqflags;
10932
10933         if (plane_state && plane_state->base.visible) {
10934                 cntl = plane_state->ctl |
10935                         i9xx_cursor_ctl_crtc(crtc_state);
10936
10937                 if (plane_state->base.crtc_h != plane_state->base.crtc_w)
10938                         fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1);
10939
10940                 base = intel_cursor_base(plane_state);
10941                 pos = intel_cursor_position(plane_state);
10942         }
10943
10944         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10945
10946         /*
10947          * On some platforms writing CURCNTR first will also
10948          * cause CURPOS to be armed by the CURBASE write.
10949          * Without the CURCNTR write the CURPOS write would
10950          * arm itself. Thus we always update CURCNTR before
10951          * CURPOS.
10952          *
10953          * On other platforms CURPOS always requires the
10954          * CURBASE write to arm the update. Additonally
10955          * a write to any of the cursor register will cancel
10956          * an already armed cursor update. Thus leaving out
10957          * the CURBASE write after CURPOS could lead to a
10958          * cursor that doesn't appear to move, or even change
10959          * shape. Thus we always write CURBASE.
10960          *
10961          * The other registers are armed by by the CURBASE write
10962          * except when the plane is getting enabled at which time
10963          * the CURCNTR write arms the update.
10964          */
10965
10966         if (INTEL_GEN(dev_priv) >= 9)
10967                 skl_write_cursor_wm(plane, crtc_state);
10968
10969         if (plane->cursor.base != base ||
10970             plane->cursor.size != fbc_ctl ||
10971             plane->cursor.cntl != cntl) {
10972                 if (HAS_CUR_FBC(dev_priv))
10973                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
10974                 I915_WRITE_FW(CURCNTR(pipe), cntl);
10975                 I915_WRITE_FW(CURPOS(pipe), pos);
10976                 I915_WRITE_FW(CURBASE(pipe), base);
10977
10978                 plane->cursor.base = base;
10979                 plane->cursor.size = fbc_ctl;
10980                 plane->cursor.cntl = cntl;
10981         } else {
10982                 I915_WRITE_FW(CURPOS(pipe), pos);
10983                 I915_WRITE_FW(CURBASE(pipe), base);
10984         }
10985
10986         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10987 }
10988
10989 static void i9xx_disable_cursor(struct intel_plane *plane,
10990                                 const struct intel_crtc_state *crtc_state)
10991 {
10992         i9xx_update_cursor(plane, crtc_state, NULL);
10993 }
10994
10995 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
10996                                      enum pipe *pipe)
10997 {
10998         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10999         enum intel_display_power_domain power_domain;
11000         intel_wakeref_t wakeref;
11001         bool ret;
11002         u32 val;
11003
11004         /*
11005          * Not 100% correct for planes that can move between pipes,
11006          * but that's only the case for gen2-3 which don't have any
11007          * display power wells.
11008          */
11009         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
11010         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11011         if (!wakeref)
11012                 return false;
11013
11014         val = I915_READ(CURCNTR(plane->pipe));
11015
11016         ret = val & MCURSOR_MODE;
11017
11018         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
11019                 *pipe = plane->pipe;
11020         else
11021                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
11022                         MCURSOR_PIPE_SELECT_SHIFT;
11023
11024         intel_display_power_put(dev_priv, power_domain, wakeref);
11025
11026         return ret;
11027 }
11028
11029 /* VESA 640x480x72Hz mode to set on the pipe */
11030 static const struct drm_display_mode load_detect_mode = {
11031         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
11032                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
11033 };
11034
11035 struct drm_framebuffer *
11036 intel_framebuffer_create(struct drm_i915_gem_object *obj,
11037                          struct drm_mode_fb_cmd2 *mode_cmd)
11038 {
11039         struct intel_framebuffer *intel_fb;
11040         int ret;
11041
11042         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
11043         if (!intel_fb)
11044                 return ERR_PTR(-ENOMEM);
11045
11046         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
11047         if (ret)
11048                 goto err;
11049
11050         return &intel_fb->base;
11051
11052 err:
11053         kfree(intel_fb);
11054         return ERR_PTR(ret);
11055 }
11056
11057 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
11058                                         struct drm_crtc *crtc)
11059 {
11060         struct drm_plane *plane;
11061         struct drm_plane_state *plane_state;
11062         int ret, i;
11063
11064         ret = drm_atomic_add_affected_planes(state, crtc);
11065         if (ret)
11066                 return ret;
11067
11068         for_each_new_plane_in_state(state, plane, plane_state, i) {
11069                 if (plane_state->crtc != crtc)
11070                         continue;
11071
11072                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
11073                 if (ret)
11074                         return ret;
11075
11076                 drm_atomic_set_fb_for_plane(plane_state, NULL);
11077         }
11078
11079         return 0;
11080 }
11081
11082 int intel_get_load_detect_pipe(struct drm_connector *connector,
11083                                const struct drm_display_mode *mode,
11084                                struct intel_load_detect_pipe *old,
11085                                struct drm_modeset_acquire_ctx *ctx)
11086 {
11087         struct intel_crtc *intel_crtc;
11088         struct intel_encoder *intel_encoder =
11089                 intel_attached_encoder(connector);
11090         struct drm_crtc *possible_crtc;
11091         struct drm_encoder *encoder = &intel_encoder->base;
11092         struct drm_crtc *crtc = NULL;
11093         struct drm_device *dev = encoder->dev;
11094         struct drm_i915_private *dev_priv = to_i915(dev);
11095         struct drm_mode_config *config = &dev->mode_config;
11096         struct drm_atomic_state *state = NULL, *restore_state = NULL;
11097         struct drm_connector_state *connector_state;
11098         struct intel_crtc_state *crtc_state;
11099         int ret, i = -1;
11100
11101         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11102                       connector->base.id, connector->name,
11103                       encoder->base.id, encoder->name);
11104
11105         old->restore_state = NULL;
11106
11107         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
11108
11109         /*
11110          * Algorithm gets a little messy:
11111          *
11112          *   - if the connector already has an assigned crtc, use it (but make
11113          *     sure it's on first)
11114          *
11115          *   - try to find the first unused crtc that can drive this connector,
11116          *     and use that if we find one
11117          */
11118
11119         /* See if we already have a CRTC for this connector */
11120         if (connector->state->crtc) {
11121                 crtc = connector->state->crtc;
11122
11123                 ret = drm_modeset_lock(&crtc->mutex, ctx);
11124                 if (ret)
11125                         goto fail;
11126
11127                 /* Make sure the crtc and connector are running */
11128                 goto found;
11129         }
11130
11131         /* Find an unused one (if possible) */
11132         for_each_crtc(dev, possible_crtc) {
11133                 i++;
11134                 if (!(encoder->possible_crtcs & (1 << i)))
11135                         continue;
11136
11137                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
11138                 if (ret)
11139                         goto fail;
11140
11141                 if (possible_crtc->state->enable) {
11142                         drm_modeset_unlock(&possible_crtc->mutex);
11143                         continue;
11144                 }
11145
11146                 crtc = possible_crtc;
11147                 break;
11148         }
11149
11150         /*
11151          * If we didn't find an unused CRTC, don't use any.
11152          */
11153         if (!crtc) {
11154                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
11155                 ret = -ENODEV;
11156                 goto fail;
11157         }
11158
11159 found:
11160         intel_crtc = to_intel_crtc(crtc);
11161
11162         state = drm_atomic_state_alloc(dev);
11163         restore_state = drm_atomic_state_alloc(dev);
11164         if (!state || !restore_state) {
11165                 ret = -ENOMEM;
11166                 goto fail;
11167         }
11168
11169         state->acquire_ctx = ctx;
11170         restore_state->acquire_ctx = ctx;
11171
11172         connector_state = drm_atomic_get_connector_state(state, connector);
11173         if (IS_ERR(connector_state)) {
11174                 ret = PTR_ERR(connector_state);
11175                 goto fail;
11176         }
11177
11178         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
11179         if (ret)
11180                 goto fail;
11181
11182         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
11183         if (IS_ERR(crtc_state)) {
11184                 ret = PTR_ERR(crtc_state);
11185                 goto fail;
11186         }
11187
11188         crtc_state->base.active = crtc_state->base.enable = true;
11189
11190         if (!mode)
11191                 mode = &load_detect_mode;
11192
11193         ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
11194         if (ret)
11195                 goto fail;
11196
11197         ret = intel_modeset_disable_planes(state, crtc);
11198         if (ret)
11199                 goto fail;
11200
11201         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
11202         if (!ret)
11203                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
11204         if (!ret)
11205                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
11206         if (ret) {
11207                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
11208                 goto fail;
11209         }
11210
11211         ret = drm_atomic_commit(state);
11212         if (ret) {
11213                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
11214                 goto fail;
11215         }
11216
11217         old->restore_state = restore_state;
11218         drm_atomic_state_put(state);
11219
11220         /* let the connector get through one full cycle before testing */
11221         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
11222         return true;
11223
11224 fail:
11225         if (state) {
11226                 drm_atomic_state_put(state);
11227                 state = NULL;
11228         }
11229         if (restore_state) {
11230                 drm_atomic_state_put(restore_state);
11231                 restore_state = NULL;
11232         }
11233
11234         if (ret == -EDEADLK)
11235                 return ret;
11236
11237         return false;
11238 }
11239
11240 void intel_release_load_detect_pipe(struct drm_connector *connector,
11241                                     struct intel_load_detect_pipe *old,
11242                                     struct drm_modeset_acquire_ctx *ctx)
11243 {
11244         struct intel_encoder *intel_encoder =
11245                 intel_attached_encoder(connector);
11246         struct drm_encoder *encoder = &intel_encoder->base;
11247         struct drm_atomic_state *state = old->restore_state;
11248         int ret;
11249
11250         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11251                       connector->base.id, connector->name,
11252                       encoder->base.id, encoder->name);
11253
11254         if (!state)
11255                 return;
11256
11257         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
11258         if (ret)
11259                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
11260         drm_atomic_state_put(state);
11261 }
11262
11263 static int i9xx_pll_refclk(struct drm_device *dev,
11264                            const struct intel_crtc_state *pipe_config)
11265 {
11266         struct drm_i915_private *dev_priv = to_i915(dev);
11267         u32 dpll = pipe_config->dpll_hw_state.dpll;
11268
11269         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
11270                 return dev_priv->vbt.lvds_ssc_freq;
11271         else if (HAS_PCH_SPLIT(dev_priv))
11272                 return 120000;
11273         else if (!IS_GEN(dev_priv, 2))
11274                 return 96000;
11275         else
11276                 return 48000;
11277 }
11278
11279 /* Returns the clock of the currently programmed mode of the given pipe. */
11280 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
11281                                 struct intel_crtc_state *pipe_config)
11282 {
11283         struct drm_device *dev = crtc->base.dev;
11284         struct drm_i915_private *dev_priv = to_i915(dev);
11285         int pipe = pipe_config->cpu_transcoder;
11286         u32 dpll = pipe_config->dpll_hw_state.dpll;
11287         u32 fp;
11288         struct dpll clock;
11289         int port_clock;
11290         int refclk = i9xx_pll_refclk(dev, pipe_config);
11291
11292         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
11293                 fp = pipe_config->dpll_hw_state.fp0;
11294         else
11295                 fp = pipe_config->dpll_hw_state.fp1;
11296
11297         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
11298         if (IS_PINEVIEW(dev_priv)) {
11299                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
11300                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
11301         } else {
11302                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
11303                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
11304         }
11305
11306         if (!IS_GEN(dev_priv, 2)) {
11307                 if (IS_PINEVIEW(dev_priv))
11308                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
11309                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
11310                 else
11311                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
11312                                DPLL_FPA01_P1_POST_DIV_SHIFT);
11313
11314                 switch (dpll & DPLL_MODE_MASK) {
11315                 case DPLLB_MODE_DAC_SERIAL:
11316                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
11317                                 5 : 10;
11318                         break;
11319                 case DPLLB_MODE_LVDS:
11320                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
11321                                 7 : 14;
11322                         break;
11323                 default:
11324                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
11325                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
11326                         return;
11327                 }
11328
11329                 if (IS_PINEVIEW(dev_priv))
11330                         port_clock = pnv_calc_dpll_params(refclk, &clock);
11331                 else
11332                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
11333         } else {
11334                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
11335                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
11336
11337                 if (is_lvds) {
11338                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
11339                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
11340
11341                         if (lvds & LVDS_CLKB_POWER_UP)
11342                                 clock.p2 = 7;
11343                         else
11344                                 clock.p2 = 14;
11345                 } else {
11346                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
11347                                 clock.p1 = 2;
11348                         else {
11349                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
11350                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
11351                         }
11352                         if (dpll & PLL_P2_DIVIDE_BY_4)
11353                                 clock.p2 = 4;
11354                         else
11355                                 clock.p2 = 2;
11356                 }
11357
11358                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
11359         }
11360
11361         /*
11362          * This value includes pixel_multiplier. We will use
11363          * port_clock to compute adjusted_mode.crtc_clock in the
11364          * encoder's get_config() function.
11365          */
11366         pipe_config->port_clock = port_clock;
11367 }
11368
11369 int intel_dotclock_calculate(int link_freq,
11370                              const struct intel_link_m_n *m_n)
11371 {
11372         /*
11373          * The calculation for the data clock is:
11374          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
11375          * But we want to avoid losing precison if possible, so:
11376          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
11377          *
11378          * and the link clock is simpler:
11379          * link_clock = (m * link_clock) / n
11380          */
11381
11382         if (!m_n->link_n)
11383                 return 0;
11384
11385         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
11386 }
11387
11388 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
11389                                    struct intel_crtc_state *pipe_config)
11390 {
11391         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11392
11393         /* read out port_clock from the DPLL */
11394         i9xx_crtc_clock_get(crtc, pipe_config);
11395
11396         /*
11397          * In case there is an active pipe without active ports,
11398          * we may need some idea for the dotclock anyway.
11399          * Calculate one based on the FDI configuration.
11400          */
11401         pipe_config->base.adjusted_mode.crtc_clock =
11402                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11403                                          &pipe_config->fdi_m_n);
11404 }
11405
11406 /* Returns the currently programmed mode of the given encoder. */
11407 struct drm_display_mode *
11408 intel_encoder_current_mode(struct intel_encoder *encoder)
11409 {
11410         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
11411         struct intel_crtc_state *crtc_state;
11412         struct drm_display_mode *mode;
11413         struct intel_crtc *crtc;
11414         enum pipe pipe;
11415
11416         if (!encoder->get_hw_state(encoder, &pipe))
11417                 return NULL;
11418
11419         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11420
11421         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
11422         if (!mode)
11423                 return NULL;
11424
11425         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
11426         if (!crtc_state) {
11427                 kfree(mode);
11428                 return NULL;
11429         }
11430
11431         crtc_state->base.crtc = &crtc->base;
11432
11433         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
11434                 kfree(crtc_state);
11435                 kfree(mode);
11436                 return NULL;
11437         }
11438
11439         encoder->get_config(encoder, crtc_state);
11440
11441         intel_mode_from_pipe_config(mode, crtc_state);
11442
11443         kfree(crtc_state);
11444
11445         return mode;
11446 }
11447
11448 static void intel_crtc_destroy(struct drm_crtc *crtc)
11449 {
11450         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11451
11452         drm_crtc_cleanup(crtc);
11453         kfree(intel_crtc);
11454 }
11455
11456 /**
11457  * intel_wm_need_update - Check whether watermarks need updating
11458  * @cur: current plane state
11459  * @new: new plane state
11460  *
11461  * Check current plane state versus the new one to determine whether
11462  * watermarks need to be recalculated.
11463  *
11464  * Returns true or false.
11465  */
11466 static bool intel_wm_need_update(const struct intel_plane_state *cur,
11467                                  struct intel_plane_state *new)
11468 {
11469         /* Update watermarks on tiling or size changes. */
11470         if (new->base.visible != cur->base.visible)
11471                 return true;
11472
11473         if (!cur->base.fb || !new->base.fb)
11474                 return false;
11475
11476         if (cur->base.fb->modifier != new->base.fb->modifier ||
11477             cur->base.rotation != new->base.rotation ||
11478             drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) ||
11479             drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) ||
11480             drm_rect_width(&new->base.dst) != drm_rect_width(&cur->base.dst) ||
11481             drm_rect_height(&new->base.dst) != drm_rect_height(&cur->base.dst))
11482                 return true;
11483
11484         return false;
11485 }
11486
11487 static bool needs_scaling(const struct intel_plane_state *state)
11488 {
11489         int src_w = drm_rect_width(&state->base.src) >> 16;
11490         int src_h = drm_rect_height(&state->base.src) >> 16;
11491         int dst_w = drm_rect_width(&state->base.dst);
11492         int dst_h = drm_rect_height(&state->base.dst);
11493
11494         return (src_w != dst_w || src_h != dst_h);
11495 }
11496
11497 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
11498                                     struct intel_crtc_state *crtc_state,
11499                                     const struct intel_plane_state *old_plane_state,
11500                                     struct intel_plane_state *plane_state)
11501 {
11502         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
11503         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
11504         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11505         bool mode_changed = needs_modeset(crtc_state);
11506         bool was_crtc_enabled = old_crtc_state->base.active;
11507         bool is_crtc_enabled = crtc_state->base.active;
11508         bool turn_off, turn_on, visible, was_visible;
11509         struct drm_framebuffer *fb = plane_state->base.fb;
11510         int ret;
11511
11512         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
11513                 ret = skl_update_scaler_plane(crtc_state, plane_state);
11514                 if (ret)
11515                         return ret;
11516         }
11517
11518         was_visible = old_plane_state->base.visible;
11519         visible = plane_state->base.visible;
11520
11521         if (!was_crtc_enabled && WARN_ON(was_visible))
11522                 was_visible = false;
11523
11524         /*
11525          * Visibility is calculated as if the crtc was on, but
11526          * after scaler setup everything depends on it being off
11527          * when the crtc isn't active.
11528          *
11529          * FIXME this is wrong for watermarks. Watermarks should also
11530          * be computed as if the pipe would be active. Perhaps move
11531          * per-plane wm computation to the .check_plane() hook, and
11532          * only combine the results from all planes in the current place?
11533          */
11534         if (!is_crtc_enabled) {
11535                 plane_state->base.visible = visible = false;
11536                 crtc_state->active_planes &= ~BIT(plane->id);
11537                 crtc_state->data_rate[plane->id] = 0;
11538         }
11539
11540         if (!was_visible && !visible)
11541                 return 0;
11542
11543         if (fb != old_plane_state->base.fb)
11544                 crtc_state->fb_changed = true;
11545
11546         turn_off = was_visible && (!visible || mode_changed);
11547         turn_on = visible && (!was_visible || mode_changed);
11548
11549         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n",
11550                          crtc->base.base.id, crtc->base.name,
11551                          plane->base.base.id, plane->base.name,
11552                          fb ? fb->base.id : -1);
11553
11554         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
11555                          plane->base.base.id, plane->base.name,
11556                          was_visible, visible,
11557                          turn_off, turn_on, mode_changed);
11558
11559         if (turn_on) {
11560                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11561                         crtc_state->update_wm_pre = true;
11562
11563                 /* must disable cxsr around plane enable/disable */
11564                 if (plane->id != PLANE_CURSOR)
11565                         crtc_state->disable_cxsr = true;
11566         } else if (turn_off) {
11567                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11568                         crtc_state->update_wm_post = true;
11569
11570                 /* must disable cxsr around plane enable/disable */
11571                 if (plane->id != PLANE_CURSOR)
11572                         crtc_state->disable_cxsr = true;
11573         } else if (intel_wm_need_update(old_plane_state, plane_state)) {
11574                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
11575                         /* FIXME bollocks */
11576                         crtc_state->update_wm_pre = true;
11577                         crtc_state->update_wm_post = true;
11578                 }
11579         }
11580
11581         if (visible || was_visible)
11582                 crtc_state->fb_bits |= plane->frontbuffer_bit;
11583
11584         /*
11585          * ILK/SNB DVSACNTR/Sprite Enable
11586          * IVB SPR_CTL/Sprite Enable
11587          * "When in Self Refresh Big FIFO mode, a write to enable the
11588          *  plane will be internally buffered and delayed while Big FIFO
11589          *  mode is exiting."
11590          *
11591          * Which means that enabling the sprite can take an extra frame
11592          * when we start in big FIFO mode (LP1+). Thus we need to drop
11593          * down to LP0 and wait for vblank in order to make sure the
11594          * sprite gets enabled on the next vblank after the register write.
11595          * Doing otherwise would risk enabling the sprite one frame after
11596          * we've already signalled flip completion. We can resume LP1+
11597          * once the sprite has been enabled.
11598          *
11599          *
11600          * WaCxSRDisabledForSpriteScaling:ivb
11601          * IVB SPR_SCALE/Scaling Enable
11602          * "Low Power watermarks must be disabled for at least one
11603          *  frame before enabling sprite scaling, and kept disabled
11604          *  until sprite scaling is disabled."
11605          *
11606          * ILK/SNB DVSASCALE/Scaling Enable
11607          * "When in Self Refresh Big FIFO mode, scaling enable will be
11608          *  masked off while Big FIFO mode is exiting."
11609          *
11610          * Despite the w/a only being listed for IVB we assume that
11611          * the ILK/SNB note has similar ramifications, hence we apply
11612          * the w/a on all three platforms.
11613          *
11614          * With experimental results seems this is needed also for primary
11615          * plane, not only sprite plane.
11616          */
11617         if (plane->id != PLANE_CURSOR &&
11618             (IS_GEN_RANGE(dev_priv, 5, 6) ||
11619              IS_IVYBRIDGE(dev_priv)) &&
11620             (turn_on || (!needs_scaling(old_plane_state) &&
11621                          needs_scaling(plane_state))))
11622                 crtc_state->disable_lp_wm = true;
11623
11624         return 0;
11625 }
11626
11627 static bool encoders_cloneable(const struct intel_encoder *a,
11628                                const struct intel_encoder *b)
11629 {
11630         /* masks could be asymmetric, so check both ways */
11631         return a == b || (a->cloneable & (1 << b->type) &&
11632                           b->cloneable & (1 << a->type));
11633 }
11634
11635 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
11636                                          struct intel_crtc *crtc,
11637                                          struct intel_encoder *encoder)
11638 {
11639         struct intel_encoder *source_encoder;
11640         struct drm_connector *connector;
11641         struct drm_connector_state *connector_state;
11642         int i;
11643
11644         for_each_new_connector_in_state(state, connector, connector_state, i) {
11645                 if (connector_state->crtc != &crtc->base)
11646                         continue;
11647
11648                 source_encoder =
11649                         to_intel_encoder(connector_state->best_encoder);
11650                 if (!encoders_cloneable(encoder, source_encoder))
11651                         return false;
11652         }
11653
11654         return true;
11655 }
11656
11657 static int icl_add_linked_planes(struct intel_atomic_state *state)
11658 {
11659         struct intel_plane *plane, *linked;
11660         struct intel_plane_state *plane_state, *linked_plane_state;
11661         int i;
11662
11663         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11664                 linked = plane_state->linked_plane;
11665
11666                 if (!linked)
11667                         continue;
11668
11669                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
11670                 if (IS_ERR(linked_plane_state))
11671                         return PTR_ERR(linked_plane_state);
11672
11673                 WARN_ON(linked_plane_state->linked_plane != plane);
11674                 WARN_ON(linked_plane_state->slave == plane_state->slave);
11675         }
11676
11677         return 0;
11678 }
11679
11680 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
11681 {
11682         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
11683         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11684         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->base.state);
11685         struct intel_plane *plane, *linked;
11686         struct intel_plane_state *plane_state;
11687         int i;
11688
11689         if (INTEL_GEN(dev_priv) < 11)
11690                 return 0;
11691
11692         /*
11693          * Destroy all old plane links and make the slave plane invisible
11694          * in the crtc_state->active_planes mask.
11695          */
11696         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11697                 if (plane->pipe != crtc->pipe || !plane_state->linked_plane)
11698                         continue;
11699
11700                 plane_state->linked_plane = NULL;
11701                 if (plane_state->slave && !plane_state->base.visible) {
11702                         crtc_state->active_planes &= ~BIT(plane->id);
11703                         crtc_state->update_planes |= BIT(plane->id);
11704                 }
11705
11706                 plane_state->slave = false;
11707         }
11708
11709         if (!crtc_state->nv12_planes)
11710                 return 0;
11711
11712         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11713                 struct intel_plane_state *linked_state = NULL;
11714
11715                 if (plane->pipe != crtc->pipe ||
11716                     !(crtc_state->nv12_planes & BIT(plane->id)))
11717                         continue;
11718
11719                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
11720                         if (!icl_is_nv12_y_plane(linked->id))
11721                                 continue;
11722
11723                         if (crtc_state->active_planes & BIT(linked->id))
11724                                 continue;
11725
11726                         linked_state = intel_atomic_get_plane_state(state, linked);
11727                         if (IS_ERR(linked_state))
11728                                 return PTR_ERR(linked_state);
11729
11730                         break;
11731                 }
11732
11733                 if (!linked_state) {
11734                         DRM_DEBUG_KMS("Need %d free Y planes for planar YUV\n",
11735                                       hweight8(crtc_state->nv12_planes));
11736
11737                         return -EINVAL;
11738                 }
11739
11740                 plane_state->linked_plane = linked;
11741
11742                 linked_state->slave = true;
11743                 linked_state->linked_plane = plane;
11744                 crtc_state->active_planes |= BIT(linked->id);
11745                 crtc_state->update_planes |= BIT(linked->id);
11746                 DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name);
11747         }
11748
11749         return 0;
11750 }
11751
11752 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
11753 {
11754         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
11755         struct intel_atomic_state *state =
11756                 to_intel_atomic_state(new_crtc_state->base.state);
11757         const struct intel_crtc_state *old_crtc_state =
11758                 intel_atomic_get_old_crtc_state(state, crtc);
11759
11760         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
11761 }
11762
11763 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
11764                                    struct drm_crtc_state *crtc_state)
11765 {
11766         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
11767         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11768         struct intel_crtc_state *pipe_config =
11769                 to_intel_crtc_state(crtc_state);
11770         int ret;
11771         bool mode_changed = needs_modeset(pipe_config);
11772
11773         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) &&
11774             mode_changed && !crtc_state->active)
11775                 pipe_config->update_wm_post = true;
11776
11777         if (mode_changed && crtc_state->enable &&
11778             dev_priv->display.crtc_compute_clock &&
11779             !WARN_ON(pipe_config->shared_dpll)) {
11780                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
11781                                                            pipe_config);
11782                 if (ret)
11783                         return ret;
11784         }
11785
11786         /*
11787          * May need to update pipe gamma enable bits
11788          * when C8 planes are getting enabled/disabled.
11789          */
11790         if (c8_planes_changed(pipe_config))
11791                 crtc_state->color_mgmt_changed = true;
11792
11793         if (mode_changed || pipe_config->update_pipe ||
11794             crtc_state->color_mgmt_changed) {
11795                 ret = intel_color_check(pipe_config);
11796                 if (ret)
11797                         return ret;
11798         }
11799
11800         ret = 0;
11801         if (dev_priv->display.compute_pipe_wm) {
11802                 ret = dev_priv->display.compute_pipe_wm(pipe_config);
11803                 if (ret) {
11804                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
11805                         return ret;
11806                 }
11807         }
11808
11809         if (dev_priv->display.compute_intermediate_wm) {
11810                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
11811                         return 0;
11812
11813                 /*
11814                  * Calculate 'intermediate' watermarks that satisfy both the
11815                  * old state and the new state.  We can program these
11816                  * immediately.
11817                  */
11818                 ret = dev_priv->display.compute_intermediate_wm(pipe_config);
11819                 if (ret) {
11820                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
11821                         return ret;
11822                 }
11823         }
11824
11825         if (INTEL_GEN(dev_priv) >= 9) {
11826                 if (mode_changed || pipe_config->update_pipe)
11827                         ret = skl_update_scaler_crtc(pipe_config);
11828
11829                 if (!ret)
11830                         ret = icl_check_nv12_planes(pipe_config);
11831                 if (!ret)
11832                         ret = skl_check_pipe_max_pixel_rate(intel_crtc,
11833                                                             pipe_config);
11834                 if (!ret)
11835                         ret = intel_atomic_setup_scalers(dev_priv, intel_crtc,
11836                                                          pipe_config);
11837         }
11838
11839         if (HAS_IPS(dev_priv))
11840                 pipe_config->ips_enabled = hsw_compute_ips_config(pipe_config);
11841
11842         return ret;
11843 }
11844
11845 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
11846         .atomic_check = intel_crtc_atomic_check,
11847 };
11848
11849 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
11850 {
11851         struct intel_connector *connector;
11852         struct drm_connector_list_iter conn_iter;
11853
11854         drm_connector_list_iter_begin(dev, &conn_iter);
11855         for_each_intel_connector_iter(connector, &conn_iter) {
11856                 if (connector->base.state->crtc)
11857                         drm_connector_put(&connector->base);
11858
11859                 if (connector->base.encoder) {
11860                         connector->base.state->best_encoder =
11861                                 connector->base.encoder;
11862                         connector->base.state->crtc =
11863                                 connector->base.encoder->crtc;
11864
11865                         drm_connector_get(&connector->base);
11866                 } else {
11867                         connector->base.state->best_encoder = NULL;
11868                         connector->base.state->crtc = NULL;
11869                 }
11870         }
11871         drm_connector_list_iter_end(&conn_iter);
11872 }
11873
11874 static int
11875 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
11876                       struct intel_crtc_state *pipe_config)
11877 {
11878         struct drm_connector *connector = conn_state->connector;
11879         const struct drm_display_info *info = &connector->display_info;
11880         int bpp;
11881
11882         switch (conn_state->max_bpc) {
11883         case 6 ... 7:
11884                 bpp = 6 * 3;
11885                 break;
11886         case 8 ... 9:
11887                 bpp = 8 * 3;
11888                 break;
11889         case 10 ... 11:
11890                 bpp = 10 * 3;
11891                 break;
11892         case 12:
11893                 bpp = 12 * 3;
11894                 break;
11895         default:
11896                 return -EINVAL;
11897         }
11898
11899         if (bpp < pipe_config->pipe_bpp) {
11900                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
11901                               "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
11902                               connector->base.id, connector->name,
11903                               bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc,
11904                               pipe_config->pipe_bpp);
11905
11906                 pipe_config->pipe_bpp = bpp;
11907         }
11908
11909         return 0;
11910 }
11911
11912 static int
11913 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
11914                           struct intel_crtc_state *pipe_config)
11915 {
11916         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11917         struct drm_atomic_state *state = pipe_config->base.state;
11918         struct drm_connector *connector;
11919         struct drm_connector_state *connector_state;
11920         int bpp, i;
11921
11922         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
11923             IS_CHERRYVIEW(dev_priv)))
11924                 bpp = 10*3;
11925         else if (INTEL_GEN(dev_priv) >= 5)
11926                 bpp = 12*3;
11927         else
11928                 bpp = 8*3;
11929
11930         pipe_config->pipe_bpp = bpp;
11931
11932         /* Clamp display bpp to connector max bpp */
11933         for_each_new_connector_in_state(state, connector, connector_state, i) {
11934                 int ret;
11935
11936                 if (connector_state->crtc != &crtc->base)
11937                         continue;
11938
11939                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
11940                 if (ret)
11941                         return ret;
11942         }
11943
11944         return 0;
11945 }
11946
11947 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
11948 {
11949         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
11950                       "type: 0x%x flags: 0x%x\n",
11951                       mode->crtc_clock,
11952                       mode->crtc_hdisplay, mode->crtc_hsync_start,
11953                       mode->crtc_hsync_end, mode->crtc_htotal,
11954                       mode->crtc_vdisplay, mode->crtc_vsync_start,
11955                       mode->crtc_vsync_end, mode->crtc_vtotal,
11956                       mode->type, mode->flags);
11957 }
11958
11959 static inline void
11960 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
11961                       const char *id, unsigned int lane_count,
11962                       const struct intel_link_m_n *m_n)
11963 {
11964         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
11965                       id, lane_count,
11966                       m_n->gmch_m, m_n->gmch_n,
11967                       m_n->link_m, m_n->link_n, m_n->tu);
11968 }
11969
11970 static void
11971 intel_dump_infoframe(struct drm_i915_private *dev_priv,
11972                      const union hdmi_infoframe *frame)
11973 {
11974         if ((drm_debug & DRM_UT_KMS) == 0)
11975                 return;
11976
11977         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
11978 }
11979
11980 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
11981
11982 static const char * const output_type_str[] = {
11983         OUTPUT_TYPE(UNUSED),
11984         OUTPUT_TYPE(ANALOG),
11985         OUTPUT_TYPE(DVO),
11986         OUTPUT_TYPE(SDVO),
11987         OUTPUT_TYPE(LVDS),
11988         OUTPUT_TYPE(TVOUT),
11989         OUTPUT_TYPE(HDMI),
11990         OUTPUT_TYPE(DP),
11991         OUTPUT_TYPE(EDP),
11992         OUTPUT_TYPE(DSI),
11993         OUTPUT_TYPE(DDI),
11994         OUTPUT_TYPE(DP_MST),
11995 };
11996
11997 #undef OUTPUT_TYPE
11998
11999 static void snprintf_output_types(char *buf, size_t len,
12000                                   unsigned int output_types)
12001 {
12002         char *str = buf;
12003         int i;
12004
12005         str[0] = '\0';
12006
12007         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
12008                 int r;
12009
12010                 if ((output_types & BIT(i)) == 0)
12011                         continue;
12012
12013                 r = snprintf(str, len, "%s%s",
12014                              str != buf ? "," : "", output_type_str[i]);
12015                 if (r >= len)
12016                         break;
12017                 str += r;
12018                 len -= r;
12019
12020                 output_types &= ~BIT(i);
12021         }
12022
12023         WARN_ON_ONCE(output_types != 0);
12024 }
12025
12026 static const char * const output_format_str[] = {
12027         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
12028         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
12029         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
12030         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
12031 };
12032
12033 static const char *output_formats(enum intel_output_format format)
12034 {
12035         if (format >= ARRAY_SIZE(output_format_str))
12036                 format = INTEL_OUTPUT_FORMAT_INVALID;
12037         return output_format_str[format];
12038 }
12039
12040 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
12041 {
12042         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
12043         const struct drm_framebuffer *fb = plane_state->base.fb;
12044         struct drm_format_name_buf format_name;
12045
12046         if (!fb) {
12047                 DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
12048                               plane->base.base.id, plane->base.name,
12049                               yesno(plane_state->base.visible));
12050                 return;
12051         }
12052
12053         DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s, visible: %s\n",
12054                       plane->base.base.id, plane->base.name,
12055                       fb->base.id, fb->width, fb->height,
12056                       drm_get_format_name(fb->format->format, &format_name),
12057                       yesno(plane_state->base.visible));
12058         DRM_DEBUG_KMS("\trotation: 0x%x, scaler: %d\n",
12059                       plane_state->base.rotation, plane_state->scaler_id);
12060         if (plane_state->base.visible)
12061                 DRM_DEBUG_KMS("\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
12062                               DRM_RECT_FP_ARG(&plane_state->base.src),
12063                               DRM_RECT_ARG(&plane_state->base.dst));
12064 }
12065
12066 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
12067                                    struct intel_atomic_state *state,
12068                                    const char *context)
12069 {
12070         struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
12071         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12072         const struct intel_plane_state *plane_state;
12073         struct intel_plane *plane;
12074         char buf[64];
12075         int i;
12076
12077         DRM_DEBUG_KMS("[CRTC:%d:%s] enable: %s %s\n",
12078                       crtc->base.base.id, crtc->base.name,
12079                       yesno(pipe_config->base.enable), context);
12080
12081         if (!pipe_config->base.enable)
12082                 goto dump_planes;
12083
12084         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
12085         DRM_DEBUG_KMS("active: %s, output_types: %s (0x%x), output format: %s\n",
12086                       yesno(pipe_config->base.active),
12087                       buf, pipe_config->output_types,
12088                       output_formats(pipe_config->output_format));
12089
12090         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
12091                       transcoder_name(pipe_config->cpu_transcoder),
12092                       pipe_config->pipe_bpp, pipe_config->dither);
12093
12094         if (pipe_config->has_pch_encoder)
12095                 intel_dump_m_n_config(pipe_config, "fdi",
12096                                       pipe_config->fdi_lanes,
12097                                       &pipe_config->fdi_m_n);
12098
12099         if (intel_crtc_has_dp_encoder(pipe_config)) {
12100                 intel_dump_m_n_config(pipe_config, "dp m_n",
12101                                 pipe_config->lane_count, &pipe_config->dp_m_n);
12102                 if (pipe_config->has_drrs)
12103                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
12104                                               pipe_config->lane_count,
12105                                               &pipe_config->dp_m2_n2);
12106         }
12107
12108         DRM_DEBUG_KMS("audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
12109                       pipe_config->has_audio, pipe_config->has_infoframe,
12110                       pipe_config->infoframes.enable);
12111
12112         if (pipe_config->infoframes.enable &
12113             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
12114                 DRM_DEBUG_KMS("GCP: 0x%x\n", pipe_config->infoframes.gcp);
12115         if (pipe_config->infoframes.enable &
12116             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
12117                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
12118         if (pipe_config->infoframes.enable &
12119             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
12120                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
12121         if (pipe_config->infoframes.enable &
12122             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
12123                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
12124
12125         DRM_DEBUG_KMS("requested mode:\n");
12126         drm_mode_debug_printmodeline(&pipe_config->base.mode);
12127         DRM_DEBUG_KMS("adjusted mode:\n");
12128         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
12129         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
12130         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
12131                       pipe_config->port_clock,
12132                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
12133                       pipe_config->pixel_rate);
12134
12135         if (INTEL_GEN(dev_priv) >= 9)
12136                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12137                               crtc->num_scalers,
12138                               pipe_config->scaler_state.scaler_users,
12139                               pipe_config->scaler_state.scaler_id);
12140
12141         if (HAS_GMCH(dev_priv))
12142                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12143                               pipe_config->gmch_pfit.control,
12144                               pipe_config->gmch_pfit.pgm_ratios,
12145                               pipe_config->gmch_pfit.lvds_border_bits);
12146         else
12147                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s, force thru: %s\n",
12148                               pipe_config->pch_pfit.pos,
12149                               pipe_config->pch_pfit.size,
12150                               enableddisabled(pipe_config->pch_pfit.enabled),
12151                               yesno(pipe_config->pch_pfit.force_thru));
12152
12153         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
12154                       pipe_config->ips_enabled, pipe_config->double_wide);
12155
12156         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
12157
12158 dump_planes:
12159         if (!state)
12160                 return;
12161
12162         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12163                 if (plane->pipe == crtc->pipe)
12164                         intel_dump_plane_state(plane_state);
12165         }
12166 }
12167
12168 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
12169 {
12170         struct drm_device *dev = state->base.dev;
12171         struct drm_connector *connector;
12172         struct drm_connector_list_iter conn_iter;
12173         unsigned int used_ports = 0;
12174         unsigned int used_mst_ports = 0;
12175         bool ret = true;
12176
12177         /*
12178          * Walk the connector list instead of the encoder
12179          * list to detect the problem on ddi platforms
12180          * where there's just one encoder per digital port.
12181          */
12182         drm_connector_list_iter_begin(dev, &conn_iter);
12183         drm_for_each_connector_iter(connector, &conn_iter) {
12184                 struct drm_connector_state *connector_state;
12185                 struct intel_encoder *encoder;
12186
12187                 connector_state =
12188                         drm_atomic_get_new_connector_state(&state->base,
12189                                                            connector);
12190                 if (!connector_state)
12191                         connector_state = connector->state;
12192
12193                 if (!connector_state->best_encoder)
12194                         continue;
12195
12196                 encoder = to_intel_encoder(connector_state->best_encoder);
12197
12198                 WARN_ON(!connector_state->crtc);
12199
12200                 switch (encoder->type) {
12201                         unsigned int port_mask;
12202                 case INTEL_OUTPUT_DDI:
12203                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
12204                                 break;
12205                         /* else, fall through */
12206                 case INTEL_OUTPUT_DP:
12207                 case INTEL_OUTPUT_HDMI:
12208                 case INTEL_OUTPUT_EDP:
12209                         port_mask = 1 << encoder->port;
12210
12211                         /* the same port mustn't appear more than once */
12212                         if (used_ports & port_mask)
12213                                 ret = false;
12214
12215                         used_ports |= port_mask;
12216                         break;
12217                 case INTEL_OUTPUT_DP_MST:
12218                         used_mst_ports |=
12219                                 1 << encoder->port;
12220                         break;
12221                 default:
12222                         break;
12223                 }
12224         }
12225         drm_connector_list_iter_end(&conn_iter);
12226
12227         /* can't mix MST and SST/HDMI on the same port */
12228         if (used_ports & used_mst_ports)
12229                 return false;
12230
12231         return ret;
12232 }
12233
12234 static int
12235 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
12236 {
12237         struct drm_i915_private *dev_priv =
12238                 to_i915(crtc_state->base.crtc->dev);
12239         struct intel_crtc_state *saved_state;
12240
12241         saved_state = kzalloc(sizeof(*saved_state), GFP_KERNEL);
12242         if (!saved_state)
12243                 return -ENOMEM;
12244
12245         /* FIXME: before the switch to atomic started, a new pipe_config was
12246          * kzalloc'd. Code that depends on any field being zero should be
12247          * fixed, so that the crtc_state can be safely duplicated. For now,
12248          * only fields that are know to not cause problems are preserved. */
12249
12250         saved_state->scaler_state = crtc_state->scaler_state;
12251         saved_state->shared_dpll = crtc_state->shared_dpll;
12252         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
12253         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
12254                sizeof(saved_state->icl_port_dplls));
12255         saved_state->crc_enabled = crtc_state->crc_enabled;
12256         if (IS_G4X(dev_priv) ||
12257             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12258                 saved_state->wm = crtc_state->wm;
12259
12260         /* Keep base drm_crtc_state intact, only clear our extended struct */
12261         BUILD_BUG_ON(offsetof(struct intel_crtc_state, base));
12262         memcpy(&crtc_state->base + 1, &saved_state->base + 1,
12263                sizeof(*crtc_state) - sizeof(crtc_state->base));
12264
12265         kfree(saved_state);
12266         return 0;
12267 }
12268
12269 static int
12270 intel_modeset_pipe_config(struct intel_crtc_state *pipe_config)
12271 {
12272         struct drm_crtc *crtc = pipe_config->base.crtc;
12273         struct drm_atomic_state *state = pipe_config->base.state;
12274         struct intel_encoder *encoder;
12275         struct drm_connector *connector;
12276         struct drm_connector_state *connector_state;
12277         int base_bpp, ret;
12278         int i;
12279         bool retry = true;
12280
12281         ret = clear_intel_crtc_state(pipe_config);
12282         if (ret)
12283                 return ret;
12284
12285         pipe_config->cpu_transcoder =
12286                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12287
12288         /*
12289          * Sanitize sync polarity flags based on requested ones. If neither
12290          * positive or negative polarity is requested, treat this as meaning
12291          * negative polarity.
12292          */
12293         if (!(pipe_config->base.adjusted_mode.flags &
12294               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12295                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12296
12297         if (!(pipe_config->base.adjusted_mode.flags &
12298               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12299                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12300
12301         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12302                                         pipe_config);
12303         if (ret)
12304                 return ret;
12305
12306         base_bpp = pipe_config->pipe_bpp;
12307
12308         /*
12309          * Determine the real pipe dimensions. Note that stereo modes can
12310          * increase the actual pipe size due to the frame doubling and
12311          * insertion of additional space for blanks between the frame. This
12312          * is stored in the crtc timings. We use the requested mode to do this
12313          * computation to clearly distinguish it from the adjusted mode, which
12314          * can be changed by the connectors in the below retry loop.
12315          */
12316         drm_mode_get_hv_timing(&pipe_config->base.mode,
12317                                &pipe_config->pipe_src_w,
12318                                &pipe_config->pipe_src_h);
12319
12320         for_each_new_connector_in_state(state, connector, connector_state, i) {
12321                 if (connector_state->crtc != crtc)
12322                         continue;
12323
12324                 encoder = to_intel_encoder(connector_state->best_encoder);
12325
12326                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
12327                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
12328                         return -EINVAL;
12329                 }
12330
12331                 /*
12332                  * Determine output_types before calling the .compute_config()
12333                  * hooks so that the hooks can use this information safely.
12334                  */
12335                 if (encoder->compute_output_type)
12336                         pipe_config->output_types |=
12337                                 BIT(encoder->compute_output_type(encoder, pipe_config,
12338                                                                  connector_state));
12339                 else
12340                         pipe_config->output_types |= BIT(encoder->type);
12341         }
12342
12343 encoder_retry:
12344         /* Ensure the port clock defaults are reset when retrying. */
12345         pipe_config->port_clock = 0;
12346         pipe_config->pixel_multiplier = 1;
12347
12348         /* Fill in default crtc timings, allow encoders to overwrite them. */
12349         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
12350                               CRTC_STEREO_DOUBLE);
12351
12352         /* Pass our mode to the connectors and the CRTC to give them a chance to
12353          * adjust it according to limitations or connector properties, and also
12354          * a chance to reject the mode entirely.
12355          */
12356         for_each_new_connector_in_state(state, connector, connector_state, i) {
12357                 if (connector_state->crtc != crtc)
12358                         continue;
12359
12360                 encoder = to_intel_encoder(connector_state->best_encoder);
12361                 ret = encoder->compute_config(encoder, pipe_config,
12362                                               connector_state);
12363                 if (ret < 0) {
12364                         if (ret != -EDEADLK)
12365                                 DRM_DEBUG_KMS("Encoder config failure: %d\n",
12366                                               ret);
12367                         return ret;
12368                 }
12369         }
12370
12371         /* Set default port clock if not overwritten by the encoder. Needs to be
12372          * done afterwards in case the encoder adjusts the mode. */
12373         if (!pipe_config->port_clock)
12374                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
12375                         * pipe_config->pixel_multiplier;
12376
12377         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
12378         if (ret == -EDEADLK)
12379                 return ret;
12380         if (ret < 0) {
12381                 DRM_DEBUG_KMS("CRTC fixup failed\n");
12382                 return ret;
12383         }
12384
12385         if (ret == RETRY) {
12386                 if (WARN(!retry, "loop in pipe configuration computation\n"))
12387                         return -EINVAL;
12388
12389                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
12390                 retry = false;
12391                 goto encoder_retry;
12392         }
12393
12394         /* Dithering seems to not pass-through bits correctly when it should, so
12395          * only enable it on 6bpc panels and when its not a compliance
12396          * test requesting 6bpc video pattern.
12397          */
12398         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
12399                 !pipe_config->dither_force_disable;
12400         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
12401                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
12402
12403         return 0;
12404 }
12405
12406 bool intel_fuzzy_clock_check(int clock1, int clock2)
12407 {
12408         int diff;
12409
12410         if (clock1 == clock2)
12411                 return true;
12412
12413         if (!clock1 || !clock2)
12414                 return false;
12415
12416         diff = abs(clock1 - clock2);
12417
12418         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
12419                 return true;
12420
12421         return false;
12422 }
12423
12424 static bool
12425 intel_compare_m_n(unsigned int m, unsigned int n,
12426                   unsigned int m2, unsigned int n2,
12427                   bool exact)
12428 {
12429         if (m == m2 && n == n2)
12430                 return true;
12431
12432         if (exact || !m || !n || !m2 || !n2)
12433                 return false;
12434
12435         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
12436
12437         if (n > n2) {
12438                 while (n > n2) {
12439                         m2 <<= 1;
12440                         n2 <<= 1;
12441                 }
12442         } else if (n < n2) {
12443                 while (n < n2) {
12444                         m <<= 1;
12445                         n <<= 1;
12446                 }
12447         }
12448
12449         if (n != n2)
12450                 return false;
12451
12452         return intel_fuzzy_clock_check(m, m2);
12453 }
12454
12455 static bool
12456 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
12457                        const struct intel_link_m_n *m2_n2,
12458                        bool exact)
12459 {
12460         return m_n->tu == m2_n2->tu &&
12461                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
12462                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
12463                 intel_compare_m_n(m_n->link_m, m_n->link_n,
12464                                   m2_n2->link_m, m2_n2->link_n, exact);
12465 }
12466
12467 static bool
12468 intel_compare_infoframe(const union hdmi_infoframe *a,
12469                         const union hdmi_infoframe *b)
12470 {
12471         return memcmp(a, b, sizeof(*a)) == 0;
12472 }
12473
12474 static void
12475 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
12476                                bool fastset, const char *name,
12477                                const union hdmi_infoframe *a,
12478                                const union hdmi_infoframe *b)
12479 {
12480         if (fastset) {
12481                 if ((drm_debug & DRM_UT_KMS) == 0)
12482                         return;
12483
12484                 drm_dbg(DRM_UT_KMS, "fastset mismatch in %s infoframe", name);
12485                 drm_dbg(DRM_UT_KMS, "expected:");
12486                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
12487                 drm_dbg(DRM_UT_KMS, "found");
12488                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
12489         } else {
12490                 drm_err("mismatch in %s infoframe", name);
12491                 drm_err("expected:");
12492                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
12493                 drm_err("found");
12494                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
12495         }
12496 }
12497
12498 static void __printf(3, 4)
12499 pipe_config_mismatch(bool fastset, const char *name, const char *format, ...)
12500 {
12501         struct va_format vaf;
12502         va_list args;
12503
12504         va_start(args, format);
12505         vaf.fmt = format;
12506         vaf.va = &args;
12507
12508         if (fastset)
12509                 drm_dbg(DRM_UT_KMS, "fastset mismatch in %s %pV", name, &vaf);
12510         else
12511                 drm_err("mismatch in %s %pV", name, &vaf);
12512
12513         va_end(args);
12514 }
12515
12516 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
12517 {
12518         if (i915_modparams.fastboot != -1)
12519                 return i915_modparams.fastboot;
12520
12521         /* Enable fastboot by default on Skylake and newer */
12522         if (INTEL_GEN(dev_priv) >= 9)
12523                 return true;
12524
12525         /* Enable fastboot by default on VLV and CHV */
12526         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12527                 return true;
12528
12529         /* Disabled by default on all others */
12530         return false;
12531 }
12532
12533 static bool
12534 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
12535                           const struct intel_crtc_state *pipe_config,
12536                           bool fastset)
12537 {
12538         struct drm_i915_private *dev_priv = to_i915(current_config->base.crtc->dev);
12539         bool ret = true;
12540         bool fixup_inherited = fastset &&
12541                 (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
12542                 !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
12543
12544         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
12545                 DRM_DEBUG_KMS("initial modeset and fastboot not set\n");
12546                 ret = false;
12547         }
12548
12549 #define PIPE_CONF_CHECK_X(name) do { \
12550         if (current_config->name != pipe_config->name) { \
12551                 pipe_config_mismatch(fastset, __stringify(name), \
12552                                      "(expected 0x%08x, found 0x%08x)\n", \
12553                                      current_config->name, \
12554                                      pipe_config->name); \
12555                 ret = false; \
12556         } \
12557 } while (0)
12558
12559 #define PIPE_CONF_CHECK_I(name) do { \
12560         if (current_config->name != pipe_config->name) { \
12561                 pipe_config_mismatch(fastset, __stringify(name), \
12562                                      "(expected %i, found %i)\n", \
12563                                      current_config->name, \
12564                                      pipe_config->name); \
12565                 ret = false; \
12566         } \
12567 } while (0)
12568
12569 #define PIPE_CONF_CHECK_BOOL(name) do { \
12570         if (current_config->name != pipe_config->name) { \
12571                 pipe_config_mismatch(fastset, __stringify(name), \
12572                                      "(expected %s, found %s)\n", \
12573                                      yesno(current_config->name), \
12574                                      yesno(pipe_config->name)); \
12575                 ret = false; \
12576         } \
12577 } while (0)
12578
12579 /*
12580  * Checks state where we only read out the enabling, but not the entire
12581  * state itself (like full infoframes or ELD for audio). These states
12582  * require a full modeset on bootup to fix up.
12583  */
12584 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
12585         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
12586                 PIPE_CONF_CHECK_BOOL(name); \
12587         } else { \
12588                 pipe_config_mismatch(fastset, __stringify(name), \
12589                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \
12590                                      yesno(current_config->name), \
12591                                      yesno(pipe_config->name)); \
12592                 ret = false; \
12593         } \
12594 } while (0)
12595
12596 #define PIPE_CONF_CHECK_P(name) do { \
12597         if (current_config->name != pipe_config->name) { \
12598                 pipe_config_mismatch(fastset, __stringify(name), \
12599                                      "(expected %p, found %p)\n", \
12600                                      current_config->name, \
12601                                      pipe_config->name); \
12602                 ret = false; \
12603         } \
12604 } while (0)
12605
12606 #define PIPE_CONF_CHECK_M_N(name) do { \
12607         if (!intel_compare_link_m_n(&current_config->name, \
12608                                     &pipe_config->name,\
12609                                     !fastset)) { \
12610                 pipe_config_mismatch(fastset, __stringify(name), \
12611                                      "(expected tu %i gmch %i/%i link %i/%i, " \
12612                                      "found tu %i, gmch %i/%i link %i/%i)\n", \
12613                                      current_config->name.tu, \
12614                                      current_config->name.gmch_m, \
12615                                      current_config->name.gmch_n, \
12616                                      current_config->name.link_m, \
12617                                      current_config->name.link_n, \
12618                                      pipe_config->name.tu, \
12619                                      pipe_config->name.gmch_m, \
12620                                      pipe_config->name.gmch_n, \
12621                                      pipe_config->name.link_m, \
12622                                      pipe_config->name.link_n); \
12623                 ret = false; \
12624         } \
12625 } while (0)
12626
12627 /* This is required for BDW+ where there is only one set of registers for
12628  * switching between high and low RR.
12629  * This macro can be used whenever a comparison has to be made between one
12630  * hw state and multiple sw state variables.
12631  */
12632 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
12633         if (!intel_compare_link_m_n(&current_config->name, \
12634                                     &pipe_config->name, !fastset) && \
12635             !intel_compare_link_m_n(&current_config->alt_name, \
12636                                     &pipe_config->name, !fastset)) { \
12637                 pipe_config_mismatch(fastset, __stringify(name), \
12638                                      "(expected tu %i gmch %i/%i link %i/%i, " \
12639                                      "or tu %i gmch %i/%i link %i/%i, " \
12640                                      "found tu %i, gmch %i/%i link %i/%i)\n", \
12641                                      current_config->name.tu, \
12642                                      current_config->name.gmch_m, \
12643                                      current_config->name.gmch_n, \
12644                                      current_config->name.link_m, \
12645                                      current_config->name.link_n, \
12646                                      current_config->alt_name.tu, \
12647                                      current_config->alt_name.gmch_m, \
12648                                      current_config->alt_name.gmch_n, \
12649                                      current_config->alt_name.link_m, \
12650                                      current_config->alt_name.link_n, \
12651                                      pipe_config->name.tu, \
12652                                      pipe_config->name.gmch_m, \
12653                                      pipe_config->name.gmch_n, \
12654                                      pipe_config->name.link_m, \
12655                                      pipe_config->name.link_n); \
12656                 ret = false; \
12657         } \
12658 } while (0)
12659
12660 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
12661         if ((current_config->name ^ pipe_config->name) & (mask)) { \
12662                 pipe_config_mismatch(fastset, __stringify(name), \
12663                                      "(%x) (expected %i, found %i)\n", \
12664                                      (mask), \
12665                                      current_config->name & (mask), \
12666                                      pipe_config->name & (mask)); \
12667                 ret = false; \
12668         } \
12669 } while (0)
12670
12671 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
12672         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
12673                 pipe_config_mismatch(fastset, __stringify(name), \
12674                                      "(expected %i, found %i)\n", \
12675                                      current_config->name, \
12676                                      pipe_config->name); \
12677                 ret = false; \
12678         } \
12679 } while (0)
12680
12681 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
12682         if (!intel_compare_infoframe(&current_config->infoframes.name, \
12683                                      &pipe_config->infoframes.name)) { \
12684                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
12685                                                &current_config->infoframes.name, \
12686                                                &pipe_config->infoframes.name); \
12687                 ret = false; \
12688         } \
12689 } while (0)
12690
12691 #define PIPE_CONF_QUIRK(quirk) \
12692         ((current_config->quirks | pipe_config->quirks) & (quirk))
12693
12694         PIPE_CONF_CHECK_I(cpu_transcoder);
12695
12696         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
12697         PIPE_CONF_CHECK_I(fdi_lanes);
12698         PIPE_CONF_CHECK_M_N(fdi_m_n);
12699
12700         PIPE_CONF_CHECK_I(lane_count);
12701         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
12702
12703         if (INTEL_GEN(dev_priv) < 8) {
12704                 PIPE_CONF_CHECK_M_N(dp_m_n);
12705
12706                 if (current_config->has_drrs)
12707                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
12708         } else
12709                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
12710
12711         PIPE_CONF_CHECK_X(output_types);
12712
12713         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
12714         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
12715         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
12716         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
12717         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
12718         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
12719
12720         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
12721         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
12722         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
12723         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
12724         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
12725         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
12726
12727         PIPE_CONF_CHECK_I(pixel_multiplier);
12728         PIPE_CONF_CHECK_I(output_format);
12729         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
12730         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
12731             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12732                 PIPE_CONF_CHECK_BOOL(limited_color_range);
12733
12734         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
12735         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
12736         PIPE_CONF_CHECK_BOOL(has_infoframe);
12737
12738         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
12739
12740         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12741                               DRM_MODE_FLAG_INTERLACE);
12742
12743         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
12744                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12745                                       DRM_MODE_FLAG_PHSYNC);
12746                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12747                                       DRM_MODE_FLAG_NHSYNC);
12748                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12749                                       DRM_MODE_FLAG_PVSYNC);
12750                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12751                                       DRM_MODE_FLAG_NVSYNC);
12752         }
12753
12754         PIPE_CONF_CHECK_X(gmch_pfit.control);
12755         /* pfit ratios are autocomputed by the hw on gen4+ */
12756         if (INTEL_GEN(dev_priv) < 4)
12757                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
12758         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
12759
12760         /*
12761          * Changing the EDP transcoder input mux
12762          * (A_ONOFF vs. A_ON) requires a full modeset.
12763          */
12764         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
12765
12766         if (!fastset) {
12767                 PIPE_CONF_CHECK_I(pipe_src_w);
12768                 PIPE_CONF_CHECK_I(pipe_src_h);
12769
12770                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
12771                 if (current_config->pch_pfit.enabled) {
12772                         PIPE_CONF_CHECK_X(pch_pfit.pos);
12773                         PIPE_CONF_CHECK_X(pch_pfit.size);
12774                 }
12775
12776                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
12777                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
12778
12779                 PIPE_CONF_CHECK_X(gamma_mode);
12780                 if (IS_CHERRYVIEW(dev_priv))
12781                         PIPE_CONF_CHECK_X(cgm_mode);
12782                 else
12783                         PIPE_CONF_CHECK_X(csc_mode);
12784                 PIPE_CONF_CHECK_BOOL(gamma_enable);
12785                 PIPE_CONF_CHECK_BOOL(csc_enable);
12786         }
12787
12788         PIPE_CONF_CHECK_BOOL(double_wide);
12789
12790         PIPE_CONF_CHECK_P(shared_dpll);
12791         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
12792         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
12793         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
12794         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
12795         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
12796         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
12797         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
12798         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
12799         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
12800         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
12801         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
12802         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
12803         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
12804         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
12805         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
12806         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
12807         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
12808         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
12809         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
12810         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
12811         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
12812         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
12813         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
12814         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
12815         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
12816         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
12817         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
12818         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
12819         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
12820         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
12821         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
12822
12823         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
12824         PIPE_CONF_CHECK_X(dsi_pll.div);
12825
12826         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
12827                 PIPE_CONF_CHECK_I(pipe_bpp);
12828
12829         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
12830         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
12831
12832         PIPE_CONF_CHECK_I(min_voltage_level);
12833
12834         PIPE_CONF_CHECK_X(infoframes.enable);
12835         PIPE_CONF_CHECK_X(infoframes.gcp);
12836         PIPE_CONF_CHECK_INFOFRAME(avi);
12837         PIPE_CONF_CHECK_INFOFRAME(spd);
12838         PIPE_CONF_CHECK_INFOFRAME(hdmi);
12839         PIPE_CONF_CHECK_INFOFRAME(drm);
12840
12841 #undef PIPE_CONF_CHECK_X
12842 #undef PIPE_CONF_CHECK_I
12843 #undef PIPE_CONF_CHECK_BOOL
12844 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
12845 #undef PIPE_CONF_CHECK_P
12846 #undef PIPE_CONF_CHECK_FLAGS
12847 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
12848 #undef PIPE_CONF_QUIRK
12849
12850         return ret;
12851 }
12852
12853 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
12854                                            const struct intel_crtc_state *pipe_config)
12855 {
12856         if (pipe_config->has_pch_encoder) {
12857                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
12858                                                             &pipe_config->fdi_m_n);
12859                 int dotclock = pipe_config->base.adjusted_mode.crtc_clock;
12860
12861                 /*
12862                  * FDI already provided one idea for the dotclock.
12863                  * Yell if the encoder disagrees.
12864                  */
12865                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
12866                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
12867                      fdi_dotclock, dotclock);
12868         }
12869 }
12870
12871 static void verify_wm_state(struct intel_crtc *crtc,
12872                             struct intel_crtc_state *new_crtc_state)
12873 {
12874         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12875         struct skl_hw_state {
12876                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
12877                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
12878                 struct skl_ddb_allocation ddb;
12879                 struct skl_pipe_wm wm;
12880         } *hw;
12881         struct skl_ddb_allocation *sw_ddb;
12882         struct skl_pipe_wm *sw_wm;
12883         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
12884         const enum pipe pipe = crtc->pipe;
12885         int plane, level, max_level = ilk_wm_max_level(dev_priv);
12886
12887         if (INTEL_GEN(dev_priv) < 9 || !new_crtc_state->base.active)
12888                 return;
12889
12890         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
12891         if (!hw)
12892                 return;
12893
12894         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
12895         sw_wm = &new_crtc_state->wm.skl.optimal;
12896
12897         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
12898
12899         skl_ddb_get_hw_state(dev_priv, &hw->ddb);
12900         sw_ddb = &dev_priv->wm.skl_hw.ddb;
12901
12902         if (INTEL_GEN(dev_priv) >= 11 &&
12903             hw->ddb.enabled_slices != sw_ddb->enabled_slices)
12904                 DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
12905                           sw_ddb->enabled_slices,
12906                           hw->ddb.enabled_slices);
12907
12908         /* planes */
12909         for_each_universal_plane(dev_priv, pipe, plane) {
12910                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
12911
12912                 hw_plane_wm = &hw->wm.planes[plane];
12913                 sw_plane_wm = &sw_wm->planes[plane];
12914
12915                 /* Watermarks */
12916                 for (level = 0; level <= max_level; level++) {
12917                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
12918                                                 &sw_plane_wm->wm[level]))
12919                                 continue;
12920
12921                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12922                                   pipe_name(pipe), plane + 1, level,
12923                                   sw_plane_wm->wm[level].plane_en,
12924                                   sw_plane_wm->wm[level].plane_res_b,
12925                                   sw_plane_wm->wm[level].plane_res_l,
12926                                   hw_plane_wm->wm[level].plane_en,
12927                                   hw_plane_wm->wm[level].plane_res_b,
12928                                   hw_plane_wm->wm[level].plane_res_l);
12929                 }
12930
12931                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12932                                          &sw_plane_wm->trans_wm)) {
12933                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12934                                   pipe_name(pipe), plane + 1,
12935                                   sw_plane_wm->trans_wm.plane_en,
12936                                   sw_plane_wm->trans_wm.plane_res_b,
12937                                   sw_plane_wm->trans_wm.plane_res_l,
12938                                   hw_plane_wm->trans_wm.plane_en,
12939                                   hw_plane_wm->trans_wm.plane_res_b,
12940                                   hw_plane_wm->trans_wm.plane_res_l);
12941                 }
12942
12943                 /* DDB */
12944                 hw_ddb_entry = &hw->ddb_y[plane];
12945                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane];
12946
12947                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
12948                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
12949                                   pipe_name(pipe), plane + 1,
12950                                   sw_ddb_entry->start, sw_ddb_entry->end,
12951                                   hw_ddb_entry->start, hw_ddb_entry->end);
12952                 }
12953         }
12954
12955         /*
12956          * cursor
12957          * If the cursor plane isn't active, we may not have updated it's ddb
12958          * allocation. In that case since the ddb allocation will be updated
12959          * once the plane becomes visible, we can skip this check
12960          */
12961         if (1) {
12962                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
12963
12964                 hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
12965                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
12966
12967                 /* Watermarks */
12968                 for (level = 0; level <= max_level; level++) {
12969                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
12970                                                 &sw_plane_wm->wm[level]))
12971                                 continue;
12972
12973                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12974                                   pipe_name(pipe), level,
12975                                   sw_plane_wm->wm[level].plane_en,
12976                                   sw_plane_wm->wm[level].plane_res_b,
12977                                   sw_plane_wm->wm[level].plane_res_l,
12978                                   hw_plane_wm->wm[level].plane_en,
12979                                   hw_plane_wm->wm[level].plane_res_b,
12980                                   hw_plane_wm->wm[level].plane_res_l);
12981                 }
12982
12983                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12984                                          &sw_plane_wm->trans_wm)) {
12985                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12986                                   pipe_name(pipe),
12987                                   sw_plane_wm->trans_wm.plane_en,
12988                                   sw_plane_wm->trans_wm.plane_res_b,
12989                                   sw_plane_wm->trans_wm.plane_res_l,
12990                                   hw_plane_wm->trans_wm.plane_en,
12991                                   hw_plane_wm->trans_wm.plane_res_b,
12992                                   hw_plane_wm->trans_wm.plane_res_l);
12993                 }
12994
12995                 /* DDB */
12996                 hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
12997                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR];
12998
12999                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
13000                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
13001                                   pipe_name(pipe),
13002                                   sw_ddb_entry->start, sw_ddb_entry->end,
13003                                   hw_ddb_entry->start, hw_ddb_entry->end);
13004                 }
13005         }
13006
13007         kfree(hw);
13008 }
13009
13010 static void
13011 verify_connector_state(struct intel_atomic_state *state,
13012                        struct intel_crtc *crtc)
13013 {
13014         struct drm_connector *connector;
13015         struct drm_connector_state *new_conn_state;
13016         int i;
13017
13018         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
13019                 struct drm_encoder *encoder = connector->encoder;
13020                 struct intel_crtc_state *crtc_state = NULL;
13021
13022                 if (new_conn_state->crtc != &crtc->base)
13023                         continue;
13024
13025                 if (crtc)
13026                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13027
13028                 intel_connector_verify_state(crtc_state, new_conn_state);
13029
13030                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
13031                      "connector's atomic encoder doesn't match legacy encoder\n");
13032         }
13033 }
13034
13035 static void
13036 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
13037 {
13038         struct intel_encoder *encoder;
13039         struct drm_connector *connector;
13040         struct drm_connector_state *old_conn_state, *new_conn_state;
13041         int i;
13042
13043         for_each_intel_encoder(&dev_priv->drm, encoder) {
13044                 bool enabled = false, found = false;
13045                 enum pipe pipe;
13046
13047                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
13048                               encoder->base.base.id,
13049                               encoder->base.name);
13050
13051                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
13052                                                    new_conn_state, i) {
13053                         if (old_conn_state->best_encoder == &encoder->base)
13054                                 found = true;
13055
13056                         if (new_conn_state->best_encoder != &encoder->base)
13057                                 continue;
13058                         found = enabled = true;
13059
13060                         I915_STATE_WARN(new_conn_state->crtc !=
13061                                         encoder->base.crtc,
13062                              "connector's crtc doesn't match encoder crtc\n");
13063                 }
13064
13065                 if (!found)
13066                         continue;
13067
13068                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
13069                      "encoder's enabled state mismatch "
13070                      "(expected %i, found %i)\n",
13071                      !!encoder->base.crtc, enabled);
13072
13073                 if (!encoder->base.crtc) {
13074                         bool active;
13075
13076                         active = encoder->get_hw_state(encoder, &pipe);
13077                         I915_STATE_WARN(active,
13078                              "encoder detached but still enabled on pipe %c.\n",
13079                              pipe_name(pipe));
13080                 }
13081         }
13082 }
13083
13084 static void
13085 verify_crtc_state(struct intel_crtc *crtc,
13086                   struct intel_crtc_state *old_crtc_state,
13087                   struct intel_crtc_state *new_crtc_state)
13088 {
13089         struct drm_device *dev = crtc->base.dev;
13090         struct drm_i915_private *dev_priv = to_i915(dev);
13091         struct intel_encoder *encoder;
13092         struct intel_crtc_state *pipe_config;
13093         struct drm_atomic_state *state;
13094         bool active;
13095
13096         state = old_crtc_state->base.state;
13097         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->base);
13098         pipe_config = old_crtc_state;
13099         memset(pipe_config, 0, sizeof(*pipe_config));
13100         pipe_config->base.crtc = &crtc->base;
13101         pipe_config->base.state = state;
13102
13103         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.base.id, crtc->base.name);
13104
13105         active = dev_priv->display.get_pipe_config(crtc, pipe_config);
13106
13107         /* we keep both pipes enabled on 830 */
13108         if (IS_I830(dev_priv))
13109                 active = new_crtc_state->base.active;
13110
13111         I915_STATE_WARN(new_crtc_state->base.active != active,
13112              "crtc active state doesn't match with hw state "
13113              "(expected %i, found %i)\n", new_crtc_state->base.active, active);
13114
13115         I915_STATE_WARN(crtc->active != new_crtc_state->base.active,
13116              "transitional active state does not match atomic hw state "
13117              "(expected %i, found %i)\n", new_crtc_state->base.active, crtc->active);
13118
13119         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
13120                 enum pipe pipe;
13121
13122                 active = encoder->get_hw_state(encoder, &pipe);
13123                 I915_STATE_WARN(active != new_crtc_state->base.active,
13124                         "[ENCODER:%i] active %i with crtc active %i\n",
13125                         encoder->base.base.id, active, new_crtc_state->base.active);
13126
13127                 I915_STATE_WARN(active && crtc->pipe != pipe,
13128                                 "Encoder connected to wrong pipe %c\n",
13129                                 pipe_name(pipe));
13130
13131                 if (active)
13132                         encoder->get_config(encoder, pipe_config);
13133         }
13134
13135         intel_crtc_compute_pixel_rate(pipe_config);
13136
13137         if (!new_crtc_state->base.active)
13138                 return;
13139
13140         intel_pipe_config_sanity_check(dev_priv, pipe_config);
13141
13142         if (!intel_pipe_config_compare(new_crtc_state,
13143                                        pipe_config, false)) {
13144                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
13145                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
13146                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
13147         }
13148 }
13149
13150 static void
13151 intel_verify_planes(struct intel_atomic_state *state)
13152 {
13153         struct intel_plane *plane;
13154         const struct intel_plane_state *plane_state;
13155         int i;
13156
13157         for_each_new_intel_plane_in_state(state, plane,
13158                                           plane_state, i)
13159                 assert_plane(plane, plane_state->slave ||
13160                              plane_state->base.visible);
13161 }
13162
13163 static void
13164 verify_single_dpll_state(struct drm_i915_private *dev_priv,
13165                          struct intel_shared_dpll *pll,
13166                          struct intel_crtc *crtc,
13167                          struct intel_crtc_state *new_crtc_state)
13168 {
13169         struct intel_dpll_hw_state dpll_hw_state;
13170         unsigned int crtc_mask;
13171         bool active;
13172
13173         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
13174
13175         DRM_DEBUG_KMS("%s\n", pll->info->name);
13176
13177         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
13178
13179         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
13180                 I915_STATE_WARN(!pll->on && pll->active_mask,
13181                      "pll in active use but not on in sw tracking\n");
13182                 I915_STATE_WARN(pll->on && !pll->active_mask,
13183                      "pll is on but not used by any active crtc\n");
13184                 I915_STATE_WARN(pll->on != active,
13185                      "pll on state mismatch (expected %i, found %i)\n",
13186                      pll->on, active);
13187         }
13188
13189         if (!crtc) {
13190                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
13191                                 "more active pll users than references: %x vs %x\n",
13192                                 pll->active_mask, pll->state.crtc_mask);
13193
13194                 return;
13195         }
13196
13197         crtc_mask = drm_crtc_mask(&crtc->base);
13198
13199         if (new_crtc_state->base.active)
13200                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
13201                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
13202                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13203         else
13204                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13205                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
13206                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13207
13208         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
13209                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
13210                         crtc_mask, pll->state.crtc_mask);
13211
13212         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
13213                                           &dpll_hw_state,
13214                                           sizeof(dpll_hw_state)),
13215                         "pll hw state mismatch\n");
13216 }
13217
13218 static void
13219 verify_shared_dpll_state(struct intel_crtc *crtc,
13220                          struct intel_crtc_state *old_crtc_state,
13221                          struct intel_crtc_state *new_crtc_state)
13222 {
13223         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13224
13225         if (new_crtc_state->shared_dpll)
13226                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
13227
13228         if (old_crtc_state->shared_dpll &&
13229             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
13230                 unsigned int crtc_mask = drm_crtc_mask(&crtc->base);
13231                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
13232
13233                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13234                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
13235                                 pipe_name(drm_crtc_index(&crtc->base)));
13236                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
13237                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
13238                                 pipe_name(drm_crtc_index(&crtc->base)));
13239         }
13240 }
13241
13242 static void
13243 intel_modeset_verify_crtc(struct intel_crtc *crtc,
13244                           struct intel_atomic_state *state,
13245                           struct intel_crtc_state *old_crtc_state,
13246                           struct intel_crtc_state *new_crtc_state)
13247 {
13248         if (!needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
13249                 return;
13250
13251         verify_wm_state(crtc, new_crtc_state);
13252         verify_connector_state(state, crtc);
13253         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
13254         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
13255 }
13256
13257 static void
13258 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
13259 {
13260         int i;
13261
13262         for (i = 0; i < dev_priv->num_shared_dpll; i++)
13263                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
13264 }
13265
13266 static void
13267 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
13268                               struct intel_atomic_state *state)
13269 {
13270         verify_encoder_state(dev_priv, state);
13271         verify_connector_state(state, NULL);
13272         verify_disabled_dpll_state(dev_priv);
13273 }
13274
13275 static void update_scanline_offset(const struct intel_crtc_state *crtc_state)
13276 {
13277         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
13278         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13279
13280         /*
13281          * The scanline counter increments at the leading edge of hsync.
13282          *
13283          * On most platforms it starts counting from vtotal-1 on the
13284          * first active line. That means the scanline counter value is
13285          * always one less than what we would expect. Ie. just after
13286          * start of vblank, which also occurs at start of hsync (on the
13287          * last active line), the scanline counter will read vblank_start-1.
13288          *
13289          * On gen2 the scanline counter starts counting from 1 instead
13290          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
13291          * to keep the value positive), instead of adding one.
13292          *
13293          * On HSW+ the behaviour of the scanline counter depends on the output
13294          * type. For DP ports it behaves like most other platforms, but on HDMI
13295          * there's an extra 1 line difference. So we need to add two instead of
13296          * one to the value.
13297          *
13298          * On VLV/CHV DSI the scanline counter would appear to increment
13299          * approx. 1/3 of a scanline before start of vblank. Unfortunately
13300          * that means we can't tell whether we're in vblank or not while
13301          * we're on that particular line. We must still set scanline_offset
13302          * to 1 so that the vblank timestamps come out correct when we query
13303          * the scanline counter from within the vblank interrupt handler.
13304          * However if queried just before the start of vblank we'll get an
13305          * answer that's slightly in the future.
13306          */
13307         if (IS_GEN(dev_priv, 2)) {
13308                 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
13309                 int vtotal;
13310
13311                 vtotal = adjusted_mode->crtc_vtotal;
13312                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
13313                         vtotal /= 2;
13314
13315                 crtc->scanline_offset = vtotal - 1;
13316         } else if (HAS_DDI(dev_priv) &&
13317                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
13318                 crtc->scanline_offset = 2;
13319         } else
13320                 crtc->scanline_offset = 1;
13321 }
13322
13323 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
13324 {
13325         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13326         struct intel_crtc_state *new_crtc_state;
13327         struct intel_crtc *crtc;
13328         int i;
13329
13330         if (!dev_priv->display.crtc_compute_clock)
13331                 return;
13332
13333         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13334                 if (!needs_modeset(new_crtc_state))
13335                         continue;
13336
13337                 intel_release_shared_dplls(state, crtc);
13338         }
13339 }
13340
13341 /*
13342  * This implements the workaround described in the "notes" section of the mode
13343  * set sequence documentation. When going from no pipes or single pipe to
13344  * multiple pipes, and planes are enabled after the pipe, we need to wait at
13345  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
13346  */
13347 static int haswell_mode_set_planes_workaround(struct intel_atomic_state *state)
13348 {
13349         struct intel_crtc_state *crtc_state;
13350         struct intel_crtc *crtc;
13351         struct intel_crtc_state *first_crtc_state = NULL;
13352         struct intel_crtc_state *other_crtc_state = NULL;
13353         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
13354         int i;
13355
13356         /* look at all crtc's that are going to be enabled in during modeset */
13357         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
13358                 if (!crtc_state->base.active ||
13359                     !needs_modeset(crtc_state))
13360                         continue;
13361
13362                 if (first_crtc_state) {
13363                         other_crtc_state = crtc_state;
13364                         break;
13365                 } else {
13366                         first_crtc_state = crtc_state;
13367                         first_pipe = crtc->pipe;
13368                 }
13369         }
13370
13371         /* No workaround needed? */
13372         if (!first_crtc_state)
13373                 return 0;
13374
13375         /* w/a possibly needed, check how many crtc's are already enabled. */
13376         for_each_intel_crtc(state->base.dev, crtc) {
13377                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
13378                 if (IS_ERR(crtc_state))
13379                         return PTR_ERR(crtc_state);
13380
13381                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
13382
13383                 if (!crtc_state->base.active ||
13384                     needs_modeset(crtc_state))
13385                         continue;
13386
13387                 /* 2 or more enabled crtcs means no need for w/a */
13388                 if (enabled_pipe != INVALID_PIPE)
13389                         return 0;
13390
13391                 enabled_pipe = crtc->pipe;
13392         }
13393
13394         if (enabled_pipe != INVALID_PIPE)
13395                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
13396         else if (other_crtc_state)
13397                 other_crtc_state->hsw_workaround_pipe = first_pipe;
13398
13399         return 0;
13400 }
13401
13402 static int intel_lock_all_pipes(struct intel_atomic_state *state)
13403 {
13404         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13405         struct intel_crtc *crtc;
13406
13407         /* Add all pipes to the state */
13408         for_each_intel_crtc(&dev_priv->drm, crtc) {
13409                 struct intel_crtc_state *crtc_state;
13410
13411                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
13412                 if (IS_ERR(crtc_state))
13413                         return PTR_ERR(crtc_state);
13414         }
13415
13416         return 0;
13417 }
13418
13419 static int intel_modeset_all_pipes(struct intel_atomic_state *state)
13420 {
13421         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13422         struct intel_crtc *crtc;
13423
13424         /*
13425          * Add all pipes to the state, and force
13426          * a modeset on all the active ones.
13427          */
13428         for_each_intel_crtc(&dev_priv->drm, crtc) {
13429                 struct intel_crtc_state *crtc_state;
13430                 int ret;
13431
13432                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
13433                 if (IS_ERR(crtc_state))
13434                         return PTR_ERR(crtc_state);
13435
13436                 if (!crtc_state->base.active || needs_modeset(crtc_state))
13437                         continue;
13438
13439                 crtc_state->base.mode_changed = true;
13440
13441                 ret = drm_atomic_add_affected_connectors(&state->base,
13442                                                          &crtc->base);
13443                 if (ret)
13444                         return ret;
13445
13446                 ret = drm_atomic_add_affected_planes(&state->base,
13447                                                      &crtc->base);
13448                 if (ret)
13449                         return ret;
13450         }
13451
13452         return 0;
13453 }
13454
13455 static int intel_modeset_checks(struct intel_atomic_state *state)
13456 {
13457         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13458         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13459         struct intel_crtc *crtc;
13460         int ret = 0, i;
13461
13462         if (!check_digital_port_conflicts(state)) {
13463                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
13464                 return -EINVAL;
13465         }
13466
13467         /* keep the current setting */
13468         if (!state->cdclk.force_min_cdclk_changed)
13469                 state->cdclk.force_min_cdclk = dev_priv->cdclk.force_min_cdclk;
13470
13471         state->modeset = true;
13472         state->active_crtcs = dev_priv->active_crtcs;
13473         state->cdclk.logical = dev_priv->cdclk.logical;
13474         state->cdclk.actual = dev_priv->cdclk.actual;
13475         state->cdclk.pipe = INVALID_PIPE;
13476
13477         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13478                                             new_crtc_state, i) {
13479                 if (new_crtc_state->base.active)
13480                         state->active_crtcs |= 1 << i;
13481                 else
13482                         state->active_crtcs &= ~(1 << i);
13483
13484                 if (old_crtc_state->base.active != new_crtc_state->base.active)
13485                         state->active_pipe_changes |= drm_crtc_mask(&crtc->base);
13486         }
13487
13488         /*
13489          * See if the config requires any additional preparation, e.g.
13490          * to adjust global state with pipes off.  We need to do this
13491          * here so we can get the modeset_pipe updated config for the new
13492          * mode set on this crtc.  For other crtcs we need to use the
13493          * adjusted_mode bits in the crtc directly.
13494          */
13495         if (dev_priv->display.modeset_calc_cdclk) {
13496                 enum pipe pipe;
13497
13498                 ret = dev_priv->display.modeset_calc_cdclk(state);
13499                 if (ret < 0)
13500                         return ret;
13501
13502                 /*
13503                  * Writes to dev_priv->cdclk.logical must protected by
13504                  * holding all the crtc locks, even if we don't end up
13505                  * touching the hardware
13506                  */
13507                 if (intel_cdclk_changed(&dev_priv->cdclk.logical,
13508                                         &state->cdclk.logical)) {
13509                         ret = intel_lock_all_pipes(state);
13510                         if (ret < 0)
13511                                 return ret;
13512                 }
13513
13514                 if (is_power_of_2(state->active_crtcs)) {
13515                         struct intel_crtc *crtc;
13516                         struct intel_crtc_state *crtc_state;
13517
13518                         pipe = ilog2(state->active_crtcs);
13519                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
13520                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13521                         if (crtc_state && needs_modeset(crtc_state))
13522                                 pipe = INVALID_PIPE;
13523                 } else {
13524                         pipe = INVALID_PIPE;
13525                 }
13526
13527                 /* All pipes must be switched off while we change the cdclk. */
13528                 if (pipe != INVALID_PIPE &&
13529                     intel_cdclk_needs_cd2x_update(dev_priv,
13530                                                   &dev_priv->cdclk.actual,
13531                                                   &state->cdclk.actual)) {
13532                         ret = intel_lock_all_pipes(state);
13533                         if (ret < 0)
13534                                 return ret;
13535
13536                         state->cdclk.pipe = pipe;
13537                 } else if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
13538                                                      &state->cdclk.actual)) {
13539                         ret = intel_modeset_all_pipes(state);
13540                         if (ret < 0)
13541                                 return ret;
13542
13543                         state->cdclk.pipe = INVALID_PIPE;
13544                 }
13545
13546                 DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
13547                               state->cdclk.logical.cdclk,
13548                               state->cdclk.actual.cdclk);
13549                 DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
13550                               state->cdclk.logical.voltage_level,
13551                               state->cdclk.actual.voltage_level);
13552         }
13553
13554         intel_modeset_clear_plls(state);
13555
13556         if (IS_HASWELL(dev_priv))
13557                 return haswell_mode_set_planes_workaround(state);
13558
13559         return 0;
13560 }
13561
13562 /*
13563  * Handle calculation of various watermark data at the end of the atomic check
13564  * phase.  The code here should be run after the per-crtc and per-plane 'check'
13565  * handlers to ensure that all derived state has been updated.
13566  */
13567 static int calc_watermark_data(struct intel_atomic_state *state)
13568 {
13569         struct drm_device *dev = state->base.dev;
13570         struct drm_i915_private *dev_priv = to_i915(dev);
13571
13572         /* Is there platform-specific watermark information to calculate? */
13573         if (dev_priv->display.compute_global_watermarks)
13574                 return dev_priv->display.compute_global_watermarks(state);
13575
13576         return 0;
13577 }
13578
13579 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
13580                                      struct intel_crtc_state *new_crtc_state)
13581 {
13582         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
13583                 return;
13584
13585         new_crtc_state->base.mode_changed = false;
13586         new_crtc_state->update_pipe = true;
13587
13588         /*
13589          * If we're not doing the full modeset we want to
13590          * keep the current M/N values as they may be
13591          * sufficiently different to the computed values
13592          * to cause problems.
13593          *
13594          * FIXME: should really copy more fuzzy state here
13595          */
13596         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
13597         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
13598         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
13599         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
13600 }
13601
13602 /**
13603  * intel_atomic_check - validate state object
13604  * @dev: drm device
13605  * @_state: state to validate
13606  */
13607 static int intel_atomic_check(struct drm_device *dev,
13608                               struct drm_atomic_state *_state)
13609 {
13610         struct drm_i915_private *dev_priv = to_i915(dev);
13611         struct intel_atomic_state *state = to_intel_atomic_state(_state);
13612         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13613         struct intel_crtc *crtc;
13614         int ret, i;
13615         bool any_ms = state->cdclk.force_min_cdclk_changed;
13616
13617         /* Catch I915_MODE_FLAG_INHERITED */
13618         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13619                                             new_crtc_state, i) {
13620                 if (new_crtc_state->base.mode.private_flags !=
13621                     old_crtc_state->base.mode.private_flags)
13622                         new_crtc_state->base.mode_changed = true;
13623         }
13624
13625         ret = drm_atomic_helper_check_modeset(dev, &state->base);
13626         if (ret)
13627                 goto fail;
13628
13629         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13630                                             new_crtc_state, i) {
13631                 if (!needs_modeset(new_crtc_state))
13632                         continue;
13633
13634                 if (!new_crtc_state->base.enable) {
13635                         any_ms = true;
13636                         continue;
13637                 }
13638
13639                 ret = intel_modeset_pipe_config(new_crtc_state);
13640                 if (ret)
13641                         goto fail;
13642
13643                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
13644
13645                 if (needs_modeset(new_crtc_state))
13646                         any_ms = true;
13647         }
13648
13649         ret = drm_dp_mst_atomic_check(&state->base);
13650         if (ret)
13651                 goto fail;
13652
13653         if (any_ms) {
13654                 ret = intel_modeset_checks(state);
13655                 if (ret)
13656                         goto fail;
13657         } else {
13658                 state->cdclk.logical = dev_priv->cdclk.logical;
13659         }
13660
13661         ret = icl_add_linked_planes(state);
13662         if (ret)
13663                 goto fail;
13664
13665         ret = drm_atomic_helper_check_planes(dev, &state->base);
13666         if (ret)
13667                 goto fail;
13668
13669         intel_fbc_choose_crtc(dev_priv, state);
13670         ret = calc_watermark_data(state);
13671         if (ret)
13672                 goto fail;
13673
13674         ret = intel_bw_atomic_check(state);
13675         if (ret)
13676                 goto fail;
13677
13678         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13679                                             new_crtc_state, i) {
13680                 if (!needs_modeset(new_crtc_state) &&
13681                     !new_crtc_state->update_pipe)
13682                         continue;
13683
13684                 intel_dump_pipe_config(new_crtc_state, state,
13685                                        needs_modeset(new_crtc_state) ?
13686                                        "[modeset]" : "[fastset]");
13687         }
13688
13689         return 0;
13690
13691  fail:
13692         if (ret == -EDEADLK)
13693                 return ret;
13694
13695         /*
13696          * FIXME would probably be nice to know which crtc specifically
13697          * caused the failure, in cases where we can pinpoint it.
13698          */
13699         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13700                                             new_crtc_state, i)
13701                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
13702
13703         return ret;
13704 }
13705
13706 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
13707 {
13708         return drm_atomic_helper_prepare_planes(state->base.dev,
13709                                                 &state->base);
13710 }
13711
13712 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
13713 {
13714         struct drm_device *dev = crtc->base.dev;
13715         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
13716
13717         if (!vblank->max_vblank_count)
13718                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
13719
13720         return crtc->base.funcs->get_vblank_counter(&crtc->base);
13721 }
13722
13723 static void intel_update_crtc(struct intel_crtc *crtc,
13724                               struct intel_atomic_state *state,
13725                               struct intel_crtc_state *old_crtc_state,
13726                               struct intel_crtc_state *new_crtc_state)
13727 {
13728         struct drm_device *dev = state->base.dev;
13729         struct drm_i915_private *dev_priv = to_i915(dev);
13730         bool modeset = needs_modeset(new_crtc_state);
13731         struct intel_plane_state *new_plane_state =
13732                 intel_atomic_get_new_plane_state(state,
13733                                                  to_intel_plane(crtc->base.primary));
13734
13735         if (modeset) {
13736                 update_scanline_offset(new_crtc_state);
13737                 dev_priv->display.crtc_enable(new_crtc_state, state);
13738
13739                 /* vblanks work again, re-enable pipe CRC. */
13740                 intel_crtc_enable_pipe_crc(crtc);
13741         } else {
13742                 intel_pre_plane_update(old_crtc_state, new_crtc_state);
13743
13744                 if (new_crtc_state->update_pipe)
13745                         intel_encoders_update_pipe(crtc, new_crtc_state, state);
13746         }
13747
13748         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
13749                 intel_fbc_disable(crtc);
13750         else if (new_plane_state)
13751                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
13752
13753         intel_begin_crtc_commit(state, crtc);
13754
13755         if (INTEL_GEN(dev_priv) >= 9)
13756                 skl_update_planes_on_crtc(state, crtc);
13757         else
13758                 i9xx_update_planes_on_crtc(state, crtc);
13759
13760         intel_finish_crtc_commit(state, crtc);
13761 }
13762
13763 static void intel_update_crtcs(struct intel_atomic_state *state)
13764 {
13765         struct intel_crtc *crtc;
13766         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13767         int i;
13768
13769         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13770                 if (!new_crtc_state->base.active)
13771                         continue;
13772
13773                 intel_update_crtc(crtc, state, old_crtc_state,
13774                                   new_crtc_state);
13775         }
13776 }
13777
13778 static void skl_update_crtcs(struct intel_atomic_state *state)
13779 {
13780         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13781         struct intel_crtc *crtc;
13782         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13783         unsigned int updated = 0;
13784         bool progress;
13785         enum pipe pipe;
13786         int i;
13787         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
13788         u8 required_slices = state->wm_results.ddb.enabled_slices;
13789         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
13790
13791         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i)
13792                 /* ignore allocations for crtc's that have been turned off. */
13793                 if (new_crtc_state->base.active)
13794                         entries[i] = old_crtc_state->wm.skl.ddb;
13795
13796         /* If 2nd DBuf slice required, enable it here */
13797         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
13798                 icl_dbuf_slices_update(dev_priv, required_slices);
13799
13800         /*
13801          * Whenever the number of active pipes changes, we need to make sure we
13802          * update the pipes in the right order so that their ddb allocations
13803          * never overlap with eachother inbetween CRTC updates. Otherwise we'll
13804          * cause pipe underruns and other bad stuff.
13805          */
13806         do {
13807                 progress = false;
13808
13809                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13810                         bool vbl_wait = false;
13811                         unsigned int cmask = drm_crtc_mask(&crtc->base);
13812
13813                         pipe = crtc->pipe;
13814
13815                         if (updated & cmask || !new_crtc_state->base.active)
13816                                 continue;
13817
13818                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
13819                                                         entries,
13820                                                         INTEL_INFO(dev_priv)->num_pipes, i))
13821                                 continue;
13822
13823                         updated |= cmask;
13824                         entries[i] = new_crtc_state->wm.skl.ddb;
13825
13826                         /*
13827                          * If this is an already active pipe, it's DDB changed,
13828                          * and this isn't the last pipe that needs updating
13829                          * then we need to wait for a vblank to pass for the
13830                          * new ddb allocation to take effect.
13831                          */
13832                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
13833                                                  &old_crtc_state->wm.skl.ddb) &&
13834                             !new_crtc_state->base.active_changed &&
13835                             state->wm_results.dirty_pipes != updated)
13836                                 vbl_wait = true;
13837
13838                         intel_update_crtc(crtc, state, old_crtc_state,
13839                                           new_crtc_state);
13840
13841                         if (vbl_wait)
13842                                 intel_wait_for_vblank(dev_priv, pipe);
13843
13844                         progress = true;
13845                 }
13846         } while (progress);
13847
13848         /* If 2nd DBuf slice is no more required disable it */
13849         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
13850                 icl_dbuf_slices_update(dev_priv, required_slices);
13851 }
13852
13853 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
13854 {
13855         struct intel_atomic_state *state, *next;
13856         struct llist_node *freed;
13857
13858         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
13859         llist_for_each_entry_safe(state, next, freed, freed)
13860                 drm_atomic_state_put(&state->base);
13861 }
13862
13863 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
13864 {
13865         struct drm_i915_private *dev_priv =
13866                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
13867
13868         intel_atomic_helper_free_state(dev_priv);
13869 }
13870
13871 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
13872 {
13873         struct wait_queue_entry wait_fence, wait_reset;
13874         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
13875
13876         init_wait_entry(&wait_fence, 0);
13877         init_wait_entry(&wait_reset, 0);
13878         for (;;) {
13879                 prepare_to_wait(&intel_state->commit_ready.wait,
13880                                 &wait_fence, TASK_UNINTERRUPTIBLE);
13881                 prepare_to_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
13882                                               I915_RESET_MODESET),
13883                                 &wait_reset, TASK_UNINTERRUPTIBLE);
13884
13885
13886                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
13887                     test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
13888                         break;
13889
13890                 schedule();
13891         }
13892         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
13893         finish_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
13894                                   I915_RESET_MODESET),
13895                     &wait_reset);
13896 }
13897
13898 static void intel_atomic_cleanup_work(struct work_struct *work)
13899 {
13900         struct drm_atomic_state *state =
13901                 container_of(work, struct drm_atomic_state, commit_work);
13902         struct drm_i915_private *i915 = to_i915(state->dev);
13903
13904         drm_atomic_helper_cleanup_planes(&i915->drm, state);
13905         drm_atomic_helper_commit_cleanup_done(state);
13906         drm_atomic_state_put(state);
13907
13908         intel_atomic_helper_free_state(i915);
13909 }
13910
13911 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
13912 {
13913         struct drm_device *dev = state->base.dev;
13914         struct drm_i915_private *dev_priv = to_i915(dev);
13915         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
13916         struct intel_crtc *crtc;
13917         u64 put_domains[I915_MAX_PIPES] = {};
13918         intel_wakeref_t wakeref = 0;
13919         int i;
13920
13921         intel_atomic_commit_fence_wait(state);
13922
13923         drm_atomic_helper_wait_for_dependencies(&state->base);
13924
13925         if (state->modeset)
13926                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
13927
13928         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13929                 if (needs_modeset(new_crtc_state) ||
13930                     new_crtc_state->update_pipe) {
13931
13932                         put_domains[crtc->pipe] =
13933                                 modeset_get_crtc_power_domains(new_crtc_state);
13934                 }
13935
13936                 if (!needs_modeset(new_crtc_state))
13937                         continue;
13938
13939                 intel_pre_plane_update(old_crtc_state, new_crtc_state);
13940
13941                 if (old_crtc_state->base.active) {
13942                         intel_crtc_disable_planes(state, crtc);
13943
13944                         /*
13945                          * We need to disable pipe CRC before disabling the pipe,
13946                          * or we race against vblank off.
13947                          */
13948                         intel_crtc_disable_pipe_crc(crtc);
13949
13950                         dev_priv->display.crtc_disable(old_crtc_state, state);
13951                         crtc->active = false;
13952                         intel_fbc_disable(crtc);
13953                         intel_disable_shared_dpll(old_crtc_state);
13954
13955                         /*
13956                          * Underruns don't always raise
13957                          * interrupts, so check manually.
13958                          */
13959                         intel_check_cpu_fifo_underruns(dev_priv);
13960                         intel_check_pch_fifo_underruns(dev_priv);
13961
13962                         /* FIXME unify this for all platforms */
13963                         if (!new_crtc_state->base.active &&
13964                             !HAS_GMCH(dev_priv) &&
13965                             dev_priv->display.initial_watermarks)
13966                                 dev_priv->display.initial_watermarks(state,
13967                                                                      new_crtc_state);
13968                 }
13969         }
13970
13971         /* FIXME: Eventually get rid of our crtc->config pointer */
13972         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
13973                 crtc->config = new_crtc_state;
13974
13975         if (state->modeset) {
13976                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
13977
13978                 intel_set_cdclk_pre_plane_update(dev_priv,
13979                                                  &state->cdclk.actual,
13980                                                  &dev_priv->cdclk.actual,
13981                                                  state->cdclk.pipe);
13982
13983                 /*
13984                  * SKL workaround: bspec recommends we disable the SAGV when we
13985                  * have more then one pipe enabled
13986                  */
13987                 if (!intel_can_enable_sagv(state))
13988                         intel_disable_sagv(dev_priv);
13989
13990                 intel_modeset_verify_disabled(dev_priv, state);
13991         }
13992
13993         /* Complete the events for pipes that have now been disabled */
13994         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13995                 bool modeset = needs_modeset(new_crtc_state);
13996
13997                 /* Complete events for now disable pipes here. */
13998                 if (modeset && !new_crtc_state->base.active && new_crtc_state->base.event) {
13999                         spin_lock_irq(&dev->event_lock);
14000                         drm_crtc_send_vblank_event(&crtc->base, new_crtc_state->base.event);
14001                         spin_unlock_irq(&dev->event_lock);
14002
14003                         new_crtc_state->base.event = NULL;
14004                 }
14005         }
14006
14007         if (state->modeset)
14008                 intel_encoders_update_prepare(state);
14009
14010         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
14011         dev_priv->display.update_crtcs(state);
14012
14013         if (state->modeset) {
14014                 intel_encoders_update_complete(state);
14015
14016                 intel_set_cdclk_post_plane_update(dev_priv,
14017                                                   &state->cdclk.actual,
14018                                                   &dev_priv->cdclk.actual,
14019                                                   state->cdclk.pipe);
14020         }
14021
14022         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
14023          * already, but still need the state for the delayed optimization. To
14024          * fix this:
14025          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
14026          * - schedule that vblank worker _before_ calling hw_done
14027          * - at the start of commit_tail, cancel it _synchrously
14028          * - switch over to the vblank wait helper in the core after that since
14029          *   we don't need out special handling any more.
14030          */
14031         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
14032
14033         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14034                 if (new_crtc_state->base.active &&
14035                     !needs_modeset(new_crtc_state) &&
14036                     (new_crtc_state->base.color_mgmt_changed ||
14037                      new_crtc_state->update_pipe))
14038                         intel_color_load_luts(new_crtc_state);
14039         }
14040
14041         /*
14042          * Now that the vblank has passed, we can go ahead and program the
14043          * optimal watermarks on platforms that need two-step watermark
14044          * programming.
14045          *
14046          * TODO: Move this (and other cleanup) to an async worker eventually.
14047          */
14048         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14049                 if (dev_priv->display.optimize_watermarks)
14050                         dev_priv->display.optimize_watermarks(state,
14051                                                               new_crtc_state);
14052         }
14053
14054         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14055                 intel_post_plane_update(old_crtc_state);
14056
14057                 if (put_domains[i])
14058                         modeset_put_power_domains(dev_priv, put_domains[i]);
14059
14060                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
14061         }
14062
14063         if (state->modeset)
14064                 intel_verify_planes(state);
14065
14066         if (state->modeset && intel_can_enable_sagv(state))
14067                 intel_enable_sagv(dev_priv);
14068
14069         drm_atomic_helper_commit_hw_done(&state->base);
14070
14071         if (state->modeset) {
14072                 /* As one of the primary mmio accessors, KMS has a high
14073                  * likelihood of triggering bugs in unclaimed access. After we
14074                  * finish modesetting, see if an error has been flagged, and if
14075                  * so enable debugging for the next modeset - and hope we catch
14076                  * the culprit.
14077                  */
14078                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
14079                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
14080         }
14081         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
14082
14083         /*
14084          * Defer the cleanup of the old state to a separate worker to not
14085          * impede the current task (userspace for blocking modesets) that
14086          * are executed inline. For out-of-line asynchronous modesets/flips,
14087          * deferring to a new worker seems overkill, but we would place a
14088          * schedule point (cond_resched()) here anyway to keep latencies
14089          * down.
14090          */
14091         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
14092         queue_work(system_highpri_wq, &state->base.commit_work);
14093 }
14094
14095 static void intel_atomic_commit_work(struct work_struct *work)
14096 {
14097         struct intel_atomic_state *state =
14098                 container_of(work, struct intel_atomic_state, base.commit_work);
14099
14100         intel_atomic_commit_tail(state);
14101 }
14102
14103 static int __i915_sw_fence_call
14104 intel_atomic_commit_ready(struct i915_sw_fence *fence,
14105                           enum i915_sw_fence_notify notify)
14106 {
14107         struct intel_atomic_state *state =
14108                 container_of(fence, struct intel_atomic_state, commit_ready);
14109
14110         switch (notify) {
14111         case FENCE_COMPLETE:
14112                 /* we do blocking waits in the worker, nothing to do here */
14113                 break;
14114         case FENCE_FREE:
14115                 {
14116                         struct intel_atomic_helper *helper =
14117                                 &to_i915(state->base.dev)->atomic_helper;
14118
14119                         if (llist_add(&state->freed, &helper->free_list))
14120                                 schedule_work(&helper->free_work);
14121                         break;
14122                 }
14123         }
14124
14125         return NOTIFY_DONE;
14126 }
14127
14128 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
14129 {
14130         struct intel_plane_state *old_plane_state, *new_plane_state;
14131         struct intel_plane *plane;
14132         int i;
14133
14134         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
14135                                              new_plane_state, i)
14136                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->base.fb),
14137                                         to_intel_frontbuffer(new_plane_state->base.fb),
14138                                         plane->frontbuffer_bit);
14139 }
14140
14141 static int intel_atomic_commit(struct drm_device *dev,
14142                                struct drm_atomic_state *_state,
14143                                bool nonblock)
14144 {
14145         struct intel_atomic_state *state = to_intel_atomic_state(_state);
14146         struct drm_i915_private *dev_priv = to_i915(dev);
14147         int ret = 0;
14148
14149         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
14150
14151         drm_atomic_state_get(&state->base);
14152         i915_sw_fence_init(&state->commit_ready,
14153                            intel_atomic_commit_ready);
14154
14155         /*
14156          * The intel_legacy_cursor_update() fast path takes care
14157          * of avoiding the vblank waits for simple cursor
14158          * movement and flips. For cursor on/off and size changes,
14159          * we want to perform the vblank waits so that watermark
14160          * updates happen during the correct frames. Gen9+ have
14161          * double buffered watermarks and so shouldn't need this.
14162          *
14163          * Unset state->legacy_cursor_update before the call to
14164          * drm_atomic_helper_setup_commit() because otherwise
14165          * drm_atomic_helper_wait_for_flip_done() is a noop and
14166          * we get FIFO underruns because we didn't wait
14167          * for vblank.
14168          *
14169          * FIXME doing watermarks and fb cleanup from a vblank worker
14170          * (assuming we had any) would solve these problems.
14171          */
14172         if (INTEL_GEN(dev_priv) < 9 && state->base.legacy_cursor_update) {
14173                 struct intel_crtc_state *new_crtc_state;
14174                 struct intel_crtc *crtc;
14175                 int i;
14176
14177                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
14178                         if (new_crtc_state->wm.need_postvbl_update ||
14179                             new_crtc_state->update_wm_post)
14180                                 state->base.legacy_cursor_update = false;
14181         }
14182
14183         ret = intel_atomic_prepare_commit(state);
14184         if (ret) {
14185                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
14186                 i915_sw_fence_commit(&state->commit_ready);
14187                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
14188                 return ret;
14189         }
14190
14191         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
14192         if (!ret)
14193                 ret = drm_atomic_helper_swap_state(&state->base, true);
14194
14195         if (ret) {
14196                 i915_sw_fence_commit(&state->commit_ready);
14197
14198                 drm_atomic_helper_cleanup_planes(dev, &state->base);
14199                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
14200                 return ret;
14201         }
14202         dev_priv->wm.distrust_bios_wm = false;
14203         intel_shared_dpll_swap_state(state);
14204         intel_atomic_track_fbs(state);
14205
14206         if (state->modeset) {
14207                 memcpy(dev_priv->min_cdclk, state->min_cdclk,
14208                        sizeof(state->min_cdclk));
14209                 memcpy(dev_priv->min_voltage_level, state->min_voltage_level,
14210                        sizeof(state->min_voltage_level));
14211                 dev_priv->active_crtcs = state->active_crtcs;
14212                 dev_priv->cdclk.force_min_cdclk = state->cdclk.force_min_cdclk;
14213
14214                 intel_cdclk_swap_state(state);
14215         }
14216
14217         drm_atomic_state_get(&state->base);
14218         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
14219
14220         i915_sw_fence_commit(&state->commit_ready);
14221         if (nonblock && state->modeset) {
14222                 queue_work(dev_priv->modeset_wq, &state->base.commit_work);
14223         } else if (nonblock) {
14224                 queue_work(system_unbound_wq, &state->base.commit_work);
14225         } else {
14226                 if (state->modeset)
14227                         flush_workqueue(dev_priv->modeset_wq);
14228                 intel_atomic_commit_tail(state);
14229         }
14230
14231         return 0;
14232 }
14233
14234 struct wait_rps_boost {
14235         struct wait_queue_entry wait;
14236
14237         struct drm_crtc *crtc;
14238         struct i915_request *request;
14239 };
14240
14241 static int do_rps_boost(struct wait_queue_entry *_wait,
14242                         unsigned mode, int sync, void *key)
14243 {
14244         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
14245         struct i915_request *rq = wait->request;
14246
14247         /*
14248          * If we missed the vblank, but the request is already running it
14249          * is reasonable to assume that it will complete before the next
14250          * vblank without our intervention, so leave RPS alone.
14251          */
14252         if (!i915_request_started(rq))
14253                 gen6_rps_boost(rq);
14254         i915_request_put(rq);
14255
14256         drm_crtc_vblank_put(wait->crtc);
14257
14258         list_del(&wait->wait.entry);
14259         kfree(wait);
14260         return 1;
14261 }
14262
14263 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
14264                                        struct dma_fence *fence)
14265 {
14266         struct wait_rps_boost *wait;
14267
14268         if (!dma_fence_is_i915(fence))
14269                 return;
14270
14271         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
14272                 return;
14273
14274         if (drm_crtc_vblank_get(crtc))
14275                 return;
14276
14277         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
14278         if (!wait) {
14279                 drm_crtc_vblank_put(crtc);
14280                 return;
14281         }
14282
14283         wait->request = to_request(dma_fence_get(fence));
14284         wait->crtc = crtc;
14285
14286         wait->wait.func = do_rps_boost;
14287         wait->wait.flags = 0;
14288
14289         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
14290 }
14291
14292 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
14293 {
14294         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
14295         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
14296         struct drm_framebuffer *fb = plane_state->base.fb;
14297         struct i915_vma *vma;
14298
14299         if (plane->id == PLANE_CURSOR &&
14300             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
14301                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14302                 const int align = intel_cursor_alignment(dev_priv);
14303                 int err;
14304
14305                 err = i915_gem_object_attach_phys(obj, align);
14306                 if (err)
14307                         return err;
14308         }
14309
14310         vma = intel_pin_and_fence_fb_obj(fb,
14311                                          &plane_state->view,
14312                                          intel_plane_uses_fence(plane_state),
14313                                          &plane_state->flags);
14314         if (IS_ERR(vma))
14315                 return PTR_ERR(vma);
14316
14317         plane_state->vma = vma;
14318
14319         return 0;
14320 }
14321
14322 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
14323 {
14324         struct i915_vma *vma;
14325
14326         vma = fetch_and_zero(&old_plane_state->vma);
14327         if (vma)
14328                 intel_unpin_fb_vma(vma, old_plane_state->flags);
14329 }
14330
14331 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
14332 {
14333         struct i915_sched_attr attr = {
14334                 .priority = I915_PRIORITY_DISPLAY,
14335         };
14336
14337         i915_gem_object_wait_priority(obj, 0, &attr);
14338 }
14339
14340 /**
14341  * intel_prepare_plane_fb - Prepare fb for usage on plane
14342  * @plane: drm plane to prepare for
14343  * @new_state: the plane state being prepared
14344  *
14345  * Prepares a framebuffer for usage on a display plane.  Generally this
14346  * involves pinning the underlying object and updating the frontbuffer tracking
14347  * bits.  Some older platforms need special physical address handling for
14348  * cursor planes.
14349  *
14350  * Must be called with struct_mutex held.
14351  *
14352  * Returns 0 on success, negative error code on failure.
14353  */
14354 int
14355 intel_prepare_plane_fb(struct drm_plane *plane,
14356                        struct drm_plane_state *new_state)
14357 {
14358         struct intel_atomic_state *intel_state =
14359                 to_intel_atomic_state(new_state->state);
14360         struct drm_i915_private *dev_priv = to_i915(plane->dev);
14361         struct drm_framebuffer *fb = new_state->fb;
14362         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14363         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
14364         int ret;
14365
14366         if (old_obj) {
14367                 struct intel_crtc_state *crtc_state =
14368                         intel_atomic_get_new_crtc_state(intel_state,
14369                                                         to_intel_crtc(plane->state->crtc));
14370
14371                 /* Big Hammer, we also need to ensure that any pending
14372                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
14373                  * current scanout is retired before unpinning the old
14374                  * framebuffer. Note that we rely on userspace rendering
14375                  * into the buffer attached to the pipe they are waiting
14376                  * on. If not, userspace generates a GPU hang with IPEHR
14377                  * point to the MI_WAIT_FOR_EVENT.
14378                  *
14379                  * This should only fail upon a hung GPU, in which case we
14380                  * can safely continue.
14381                  */
14382                 if (needs_modeset(crtc_state)) {
14383                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
14384                                                               old_obj->base.resv, NULL,
14385                                                               false, 0,
14386                                                               GFP_KERNEL);
14387                         if (ret < 0)
14388                                 return ret;
14389                 }
14390         }
14391
14392         if (new_state->fence) { /* explicit fencing */
14393                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
14394                                                     new_state->fence,
14395                                                     I915_FENCE_TIMEOUT,
14396                                                     GFP_KERNEL);
14397                 if (ret < 0)
14398                         return ret;
14399         }
14400
14401         if (!obj)
14402                 return 0;
14403
14404         ret = i915_gem_object_pin_pages(obj);
14405         if (ret)
14406                 return ret;
14407
14408         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
14409         if (ret) {
14410                 i915_gem_object_unpin_pages(obj);
14411                 return ret;
14412         }
14413
14414         ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
14415
14416         mutex_unlock(&dev_priv->drm.struct_mutex);
14417         i915_gem_object_unpin_pages(obj);
14418         if (ret)
14419                 return ret;
14420
14421         fb_obj_bump_render_priority(obj);
14422         intel_frontbuffer_flush(obj->frontbuffer, ORIGIN_DIRTYFB);
14423
14424         if (!new_state->fence) { /* implicit fencing */
14425                 struct dma_fence *fence;
14426
14427                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
14428                                                       obj->base.resv, NULL,
14429                                                       false, I915_FENCE_TIMEOUT,
14430                                                       GFP_KERNEL);
14431                 if (ret < 0)
14432                         return ret;
14433
14434                 fence = dma_resv_get_excl_rcu(obj->base.resv);
14435                 if (fence) {
14436                         add_rps_boost_after_vblank(new_state->crtc, fence);
14437                         dma_fence_put(fence);
14438                 }
14439         } else {
14440                 add_rps_boost_after_vblank(new_state->crtc, new_state->fence);
14441         }
14442
14443         /*
14444          * We declare pageflips to be interactive and so merit a small bias
14445          * towards upclocking to deliver the frame on time. By only changing
14446          * the RPS thresholds to sample more regularly and aim for higher
14447          * clocks we can hopefully deliver low power workloads (like kodi)
14448          * that are not quite steady state without resorting to forcing
14449          * maximum clocks following a vblank miss (see do_rps_boost()).
14450          */
14451         if (!intel_state->rps_interactive) {
14452                 intel_rps_mark_interactive(dev_priv, true);
14453                 intel_state->rps_interactive = true;
14454         }
14455
14456         return 0;
14457 }
14458
14459 /**
14460  * intel_cleanup_plane_fb - Cleans up an fb after plane use
14461  * @plane: drm plane to clean up for
14462  * @old_state: the state from the previous modeset
14463  *
14464  * Cleans up a framebuffer that has just been removed from a plane.
14465  *
14466  * Must be called with struct_mutex held.
14467  */
14468 void
14469 intel_cleanup_plane_fb(struct drm_plane *plane,
14470                        struct drm_plane_state *old_state)
14471 {
14472         struct intel_atomic_state *intel_state =
14473                 to_intel_atomic_state(old_state->state);
14474         struct drm_i915_private *dev_priv = to_i915(plane->dev);
14475
14476         if (intel_state->rps_interactive) {
14477                 intel_rps_mark_interactive(dev_priv, false);
14478                 intel_state->rps_interactive = false;
14479         }
14480
14481         /* Should only be called after a successful intel_prepare_plane_fb()! */
14482         mutex_lock(&dev_priv->drm.struct_mutex);
14483         intel_plane_unpin_fb(to_intel_plane_state(old_state));
14484         mutex_unlock(&dev_priv->drm.struct_mutex);
14485 }
14486
14487 int
14488 skl_max_scale(const struct intel_crtc_state *crtc_state,
14489               u32 pixel_format)
14490 {
14491         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
14492         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14493         int max_scale, mult;
14494         int crtc_clock, max_dotclk, tmpclk1, tmpclk2;
14495
14496         if (!crtc_state->base.enable)
14497                 return DRM_PLANE_HELPER_NO_SCALING;
14498
14499         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
14500         max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk;
14501
14502         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
14503                 max_dotclk *= 2;
14504
14505         if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock))
14506                 return DRM_PLANE_HELPER_NO_SCALING;
14507
14508         /*
14509          * skl max scale is lower of:
14510          *    close to 3 but not 3, -1 is for that purpose
14511          *            or
14512          *    cdclk/crtc_clock
14513          */
14514         mult = is_planar_yuv_format(pixel_format) ? 2 : 3;
14515         tmpclk1 = (1 << 16) * mult - 1;
14516         tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock);
14517         max_scale = min(tmpclk1, tmpclk2);
14518
14519         return max_scale;
14520 }
14521
14522 static void intel_begin_crtc_commit(struct intel_atomic_state *state,
14523                                     struct intel_crtc *crtc)
14524 {
14525         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14526         struct intel_crtc_state *old_crtc_state =
14527                 intel_atomic_get_old_crtc_state(state, crtc);
14528         struct intel_crtc_state *new_crtc_state =
14529                 intel_atomic_get_new_crtc_state(state, crtc);
14530         bool modeset = needs_modeset(new_crtc_state);
14531
14532         /* Perform vblank evasion around commit operation */
14533         intel_pipe_update_start(new_crtc_state);
14534
14535         if (modeset)
14536                 goto out;
14537
14538         if (new_crtc_state->base.color_mgmt_changed ||
14539             new_crtc_state->update_pipe)
14540                 intel_color_commit(new_crtc_state);
14541
14542         if (new_crtc_state->update_pipe)
14543                 intel_update_pipe_config(old_crtc_state, new_crtc_state);
14544         else if (INTEL_GEN(dev_priv) >= 9)
14545                 skl_detach_scalers(new_crtc_state);
14546
14547         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
14548                 bdw_set_pipemisc(new_crtc_state);
14549
14550 out:
14551         if (dev_priv->display.atomic_update_watermarks)
14552                 dev_priv->display.atomic_update_watermarks(state,
14553                                                            new_crtc_state);
14554 }
14555
14556 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
14557                                   struct intel_crtc_state *crtc_state)
14558 {
14559         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14560
14561         if (!IS_GEN(dev_priv, 2))
14562                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
14563
14564         if (crtc_state->has_pch_encoder) {
14565                 enum pipe pch_transcoder =
14566                         intel_crtc_pch_transcoder(crtc);
14567
14568                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
14569         }
14570 }
14571
14572 static void intel_finish_crtc_commit(struct intel_atomic_state *state,
14573                                      struct intel_crtc *crtc)
14574 {
14575         struct intel_crtc_state *old_crtc_state =
14576                 intel_atomic_get_old_crtc_state(state, crtc);
14577         struct intel_crtc_state *new_crtc_state =
14578                 intel_atomic_get_new_crtc_state(state, crtc);
14579
14580         intel_pipe_update_end(new_crtc_state);
14581
14582         if (new_crtc_state->update_pipe &&
14583             !needs_modeset(new_crtc_state) &&
14584             old_crtc_state->base.mode.private_flags & I915_MODE_FLAG_INHERITED)
14585                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14586 }
14587
14588 /**
14589  * intel_plane_destroy - destroy a plane
14590  * @plane: plane to destroy
14591  *
14592  * Common destruction function for all types of planes (primary, cursor,
14593  * sprite).
14594  */
14595 void intel_plane_destroy(struct drm_plane *plane)
14596 {
14597         drm_plane_cleanup(plane);
14598         kfree(to_intel_plane(plane));
14599 }
14600
14601 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
14602                                             u32 format, u64 modifier)
14603 {
14604         switch (modifier) {
14605         case DRM_FORMAT_MOD_LINEAR:
14606         case I915_FORMAT_MOD_X_TILED:
14607                 break;
14608         default:
14609                 return false;
14610         }
14611
14612         switch (format) {
14613         case DRM_FORMAT_C8:
14614         case DRM_FORMAT_RGB565:
14615         case DRM_FORMAT_XRGB1555:
14616         case DRM_FORMAT_XRGB8888:
14617                 return modifier == DRM_FORMAT_MOD_LINEAR ||
14618                         modifier == I915_FORMAT_MOD_X_TILED;
14619         default:
14620                 return false;
14621         }
14622 }
14623
14624 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
14625                                             u32 format, u64 modifier)
14626 {
14627         switch (modifier) {
14628         case DRM_FORMAT_MOD_LINEAR:
14629         case I915_FORMAT_MOD_X_TILED:
14630                 break;
14631         default:
14632                 return false;
14633         }
14634
14635         switch (format) {
14636         case DRM_FORMAT_C8:
14637         case DRM_FORMAT_RGB565:
14638         case DRM_FORMAT_XRGB8888:
14639         case DRM_FORMAT_XBGR8888:
14640         case DRM_FORMAT_XRGB2101010:
14641         case DRM_FORMAT_XBGR2101010:
14642                 return modifier == DRM_FORMAT_MOD_LINEAR ||
14643                         modifier == I915_FORMAT_MOD_X_TILED;
14644         default:
14645                 return false;
14646         }
14647 }
14648
14649 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
14650                                               u32 format, u64 modifier)
14651 {
14652         return modifier == DRM_FORMAT_MOD_LINEAR &&
14653                 format == DRM_FORMAT_ARGB8888;
14654 }
14655
14656 static const struct drm_plane_funcs i965_plane_funcs = {
14657         .update_plane = drm_atomic_helper_update_plane,
14658         .disable_plane = drm_atomic_helper_disable_plane,
14659         .destroy = intel_plane_destroy,
14660         .atomic_duplicate_state = intel_plane_duplicate_state,
14661         .atomic_destroy_state = intel_plane_destroy_state,
14662         .format_mod_supported = i965_plane_format_mod_supported,
14663 };
14664
14665 static const struct drm_plane_funcs i8xx_plane_funcs = {
14666         .update_plane = drm_atomic_helper_update_plane,
14667         .disable_plane = drm_atomic_helper_disable_plane,
14668         .destroy = intel_plane_destroy,
14669         .atomic_duplicate_state = intel_plane_duplicate_state,
14670         .atomic_destroy_state = intel_plane_destroy_state,
14671         .format_mod_supported = i8xx_plane_format_mod_supported,
14672 };
14673
14674 static int
14675 intel_legacy_cursor_update(struct drm_plane *plane,
14676                            struct drm_crtc *crtc,
14677                            struct drm_framebuffer *fb,
14678                            int crtc_x, int crtc_y,
14679                            unsigned int crtc_w, unsigned int crtc_h,
14680                            u32 src_x, u32 src_y,
14681                            u32 src_w, u32 src_h,
14682                            struct drm_modeset_acquire_ctx *ctx)
14683 {
14684         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
14685         struct drm_plane_state *old_plane_state, *new_plane_state;
14686         struct intel_plane *intel_plane = to_intel_plane(plane);
14687         struct intel_crtc_state *crtc_state =
14688                 to_intel_crtc_state(crtc->state);
14689         struct intel_crtc_state *new_crtc_state;
14690         int ret;
14691
14692         /*
14693          * When crtc is inactive or there is a modeset pending,
14694          * wait for it to complete in the slowpath
14695          */
14696         if (!crtc_state->base.active || needs_modeset(crtc_state) ||
14697             crtc_state->update_pipe)
14698                 goto slow;
14699
14700         old_plane_state = plane->state;
14701         /*
14702          * Don't do an async update if there is an outstanding commit modifying
14703          * the plane.  This prevents our async update's changes from getting
14704          * overridden by a previous synchronous update's state.
14705          */
14706         if (old_plane_state->commit &&
14707             !try_wait_for_completion(&old_plane_state->commit->hw_done))
14708                 goto slow;
14709
14710         /*
14711          * If any parameters change that may affect watermarks,
14712          * take the slowpath. Only changing fb or position should be
14713          * in the fastpath.
14714          */
14715         if (old_plane_state->crtc != crtc ||
14716             old_plane_state->src_w != src_w ||
14717             old_plane_state->src_h != src_h ||
14718             old_plane_state->crtc_w != crtc_w ||
14719             old_plane_state->crtc_h != crtc_h ||
14720             !old_plane_state->fb != !fb)
14721                 goto slow;
14722
14723         new_plane_state = intel_plane_duplicate_state(plane);
14724         if (!new_plane_state)
14725                 return -ENOMEM;
14726
14727         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(crtc));
14728         if (!new_crtc_state) {
14729                 ret = -ENOMEM;
14730                 goto out_free;
14731         }
14732
14733         drm_atomic_set_fb_for_plane(new_plane_state, fb);
14734
14735         new_plane_state->src_x = src_x;
14736         new_plane_state->src_y = src_y;
14737         new_plane_state->src_w = src_w;
14738         new_plane_state->src_h = src_h;
14739         new_plane_state->crtc_x = crtc_x;
14740         new_plane_state->crtc_y = crtc_y;
14741         new_plane_state->crtc_w = crtc_w;
14742         new_plane_state->crtc_h = crtc_h;
14743
14744         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
14745                                                   to_intel_plane_state(old_plane_state),
14746                                                   to_intel_plane_state(new_plane_state));
14747         if (ret)
14748                 goto out_free;
14749
14750         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
14751         if (ret)
14752                 goto out_free;
14753
14754         ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
14755         if (ret)
14756                 goto out_unlock;
14757
14758         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_FLIP);
14759         intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->fb),
14760                                 to_intel_frontbuffer(fb),
14761                                 intel_plane->frontbuffer_bit);
14762
14763         /* Swap plane state */
14764         plane->state = new_plane_state;
14765
14766         /*
14767          * We cannot swap crtc_state as it may be in use by an atomic commit or
14768          * page flip that's running simultaneously. If we swap crtc_state and
14769          * destroy the old state, we will cause a use-after-free there.
14770          *
14771          * Only update active_planes, which is needed for our internal
14772          * bookkeeping. Either value will do the right thing when updating
14773          * planes atomically. If the cursor was part of the atomic update then
14774          * we would have taken the slowpath.
14775          */
14776         crtc_state->active_planes = new_crtc_state->active_planes;
14777
14778         if (plane->state->visible)
14779                 intel_update_plane(intel_plane, crtc_state,
14780                                    to_intel_plane_state(plane->state));
14781         else
14782                 intel_disable_plane(intel_plane, crtc_state);
14783
14784         intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
14785
14786 out_unlock:
14787         mutex_unlock(&dev_priv->drm.struct_mutex);
14788 out_free:
14789         if (new_crtc_state)
14790                 intel_crtc_destroy_state(crtc, &new_crtc_state->base);
14791         if (ret)
14792                 intel_plane_destroy_state(plane, new_plane_state);
14793         else
14794                 intel_plane_destroy_state(plane, old_plane_state);
14795         return ret;
14796
14797 slow:
14798         return drm_atomic_helper_update_plane(plane, crtc, fb,
14799                                               crtc_x, crtc_y, crtc_w, crtc_h,
14800                                               src_x, src_y, src_w, src_h, ctx);
14801 }
14802
14803 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
14804         .update_plane = intel_legacy_cursor_update,
14805         .disable_plane = drm_atomic_helper_disable_plane,
14806         .destroy = intel_plane_destroy,
14807         .atomic_duplicate_state = intel_plane_duplicate_state,
14808         .atomic_destroy_state = intel_plane_destroy_state,
14809         .format_mod_supported = intel_cursor_format_mod_supported,
14810 };
14811
14812 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
14813                                enum i9xx_plane_id i9xx_plane)
14814 {
14815         if (!HAS_FBC(dev_priv))
14816                 return false;
14817
14818         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
14819                 return i9xx_plane == PLANE_A; /* tied to pipe A */
14820         else if (IS_IVYBRIDGE(dev_priv))
14821                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
14822                         i9xx_plane == PLANE_C;
14823         else if (INTEL_GEN(dev_priv) >= 4)
14824                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
14825         else
14826                 return i9xx_plane == PLANE_A;
14827 }
14828
14829 static struct intel_plane *
14830 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
14831 {
14832         struct intel_plane *plane;
14833         const struct drm_plane_funcs *plane_funcs;
14834         unsigned int supported_rotations;
14835         unsigned int possible_crtcs;
14836         const u64 *modifiers;
14837         const u32 *formats;
14838         int num_formats;
14839         int ret;
14840
14841         if (INTEL_GEN(dev_priv) >= 9)
14842                 return skl_universal_plane_create(dev_priv, pipe,
14843                                                   PLANE_PRIMARY);
14844
14845         plane = intel_plane_alloc();
14846         if (IS_ERR(plane))
14847                 return plane;
14848
14849         plane->pipe = pipe;
14850         /*
14851          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
14852          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
14853          */
14854         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
14855                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
14856         else
14857                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
14858         plane->id = PLANE_PRIMARY;
14859         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
14860
14861         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
14862         if (plane->has_fbc) {
14863                 struct intel_fbc *fbc = &dev_priv->fbc;
14864
14865                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
14866         }
14867
14868         if (INTEL_GEN(dev_priv) >= 4) {
14869                 formats = i965_primary_formats;
14870                 num_formats = ARRAY_SIZE(i965_primary_formats);
14871                 modifiers = i9xx_format_modifiers;
14872
14873                 plane->max_stride = i9xx_plane_max_stride;
14874                 plane->update_plane = i9xx_update_plane;
14875                 plane->disable_plane = i9xx_disable_plane;
14876                 plane->get_hw_state = i9xx_plane_get_hw_state;
14877                 plane->check_plane = i9xx_plane_check;
14878
14879                 plane_funcs = &i965_plane_funcs;
14880         } else {
14881                 formats = i8xx_primary_formats;
14882                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
14883                 modifiers = i9xx_format_modifiers;
14884
14885                 plane->max_stride = i9xx_plane_max_stride;
14886                 plane->update_plane = i9xx_update_plane;
14887                 plane->disable_plane = i9xx_disable_plane;
14888                 plane->get_hw_state = i9xx_plane_get_hw_state;
14889                 plane->check_plane = i9xx_plane_check;
14890
14891                 plane_funcs = &i8xx_plane_funcs;
14892         }
14893
14894         possible_crtcs = BIT(pipe);
14895
14896         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
14897                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
14898                                                possible_crtcs, plane_funcs,
14899                                                formats, num_formats, modifiers,
14900                                                DRM_PLANE_TYPE_PRIMARY,
14901                                                "primary %c", pipe_name(pipe));
14902         else
14903                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
14904                                                possible_crtcs, plane_funcs,
14905                                                formats, num_formats, modifiers,
14906                                                DRM_PLANE_TYPE_PRIMARY,
14907                                                "plane %c",
14908                                                plane_name(plane->i9xx_plane));
14909         if (ret)
14910                 goto fail;
14911
14912         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
14913                 supported_rotations =
14914                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
14915                         DRM_MODE_REFLECT_X;
14916         } else if (INTEL_GEN(dev_priv) >= 4) {
14917                 supported_rotations =
14918                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
14919         } else {
14920                 supported_rotations = DRM_MODE_ROTATE_0;
14921         }
14922
14923         if (INTEL_GEN(dev_priv) >= 4)
14924                 drm_plane_create_rotation_property(&plane->base,
14925                                                    DRM_MODE_ROTATE_0,
14926                                                    supported_rotations);
14927
14928         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
14929
14930         return plane;
14931
14932 fail:
14933         intel_plane_free(plane);
14934
14935         return ERR_PTR(ret);
14936 }
14937
14938 static struct intel_plane *
14939 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
14940                           enum pipe pipe)
14941 {
14942         unsigned int possible_crtcs;
14943         struct intel_plane *cursor;
14944         int ret;
14945
14946         cursor = intel_plane_alloc();
14947         if (IS_ERR(cursor))
14948                 return cursor;
14949
14950         cursor->pipe = pipe;
14951         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
14952         cursor->id = PLANE_CURSOR;
14953         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
14954
14955         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
14956                 cursor->max_stride = i845_cursor_max_stride;
14957                 cursor->update_plane = i845_update_cursor;
14958                 cursor->disable_plane = i845_disable_cursor;
14959                 cursor->get_hw_state = i845_cursor_get_hw_state;
14960                 cursor->check_plane = i845_check_cursor;
14961         } else {
14962                 cursor->max_stride = i9xx_cursor_max_stride;
14963                 cursor->update_plane = i9xx_update_cursor;
14964                 cursor->disable_plane = i9xx_disable_cursor;
14965                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
14966                 cursor->check_plane = i9xx_check_cursor;
14967         }
14968
14969         cursor->cursor.base = ~0;
14970         cursor->cursor.cntl = ~0;
14971
14972         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
14973                 cursor->cursor.size = ~0;
14974
14975         possible_crtcs = BIT(pipe);
14976
14977         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
14978                                        possible_crtcs, &intel_cursor_plane_funcs,
14979                                        intel_cursor_formats,
14980                                        ARRAY_SIZE(intel_cursor_formats),
14981                                        cursor_format_modifiers,
14982                                        DRM_PLANE_TYPE_CURSOR,
14983                                        "cursor %c", pipe_name(pipe));
14984         if (ret)
14985                 goto fail;
14986
14987         if (INTEL_GEN(dev_priv) >= 4)
14988                 drm_plane_create_rotation_property(&cursor->base,
14989                                                    DRM_MODE_ROTATE_0,
14990                                                    DRM_MODE_ROTATE_0 |
14991                                                    DRM_MODE_ROTATE_180);
14992
14993         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
14994
14995         return cursor;
14996
14997 fail:
14998         intel_plane_free(cursor);
14999
15000         return ERR_PTR(ret);
15001 }
15002
15003 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
15004                                     struct intel_crtc_state *crtc_state)
15005 {
15006         struct intel_crtc_scaler_state *scaler_state =
15007                 &crtc_state->scaler_state;
15008         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
15009         int i;
15010
15011         crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[crtc->pipe];
15012         if (!crtc->num_scalers)
15013                 return;
15014
15015         for (i = 0; i < crtc->num_scalers; i++) {
15016                 struct intel_scaler *scaler = &scaler_state->scalers[i];
15017
15018                 scaler->in_use = 0;
15019                 scaler->mode = 0;
15020         }
15021
15022         scaler_state->scaler_id = -1;
15023 }
15024
15025 #define INTEL_CRTC_FUNCS \
15026         .gamma_set = drm_atomic_helper_legacy_gamma_set, \
15027         .set_config = drm_atomic_helper_set_config, \
15028         .destroy = intel_crtc_destroy, \
15029         .page_flip = drm_atomic_helper_page_flip, \
15030         .atomic_duplicate_state = intel_crtc_duplicate_state, \
15031         .atomic_destroy_state = intel_crtc_destroy_state, \
15032         .set_crc_source = intel_crtc_set_crc_source, \
15033         .verify_crc_source = intel_crtc_verify_crc_source, \
15034         .get_crc_sources = intel_crtc_get_crc_sources
15035
15036 static const struct drm_crtc_funcs bdw_crtc_funcs = {
15037         INTEL_CRTC_FUNCS,
15038
15039         .get_vblank_counter = g4x_get_vblank_counter,
15040         .enable_vblank = bdw_enable_vblank,
15041         .disable_vblank = bdw_disable_vblank,
15042 };
15043
15044 static const struct drm_crtc_funcs ilk_crtc_funcs = {
15045         INTEL_CRTC_FUNCS,
15046
15047         .get_vblank_counter = g4x_get_vblank_counter,
15048         .enable_vblank = ilk_enable_vblank,
15049         .disable_vblank = ilk_disable_vblank,
15050 };
15051
15052 static const struct drm_crtc_funcs g4x_crtc_funcs = {
15053         INTEL_CRTC_FUNCS,
15054
15055         .get_vblank_counter = g4x_get_vblank_counter,
15056         .enable_vblank = i965_enable_vblank,
15057         .disable_vblank = i965_disable_vblank,
15058 };
15059
15060 static const struct drm_crtc_funcs i965_crtc_funcs = {
15061         INTEL_CRTC_FUNCS,
15062
15063         .get_vblank_counter = i915_get_vblank_counter,
15064         .enable_vblank = i965_enable_vblank,
15065         .disable_vblank = i965_disable_vblank,
15066 };
15067
15068 static const struct drm_crtc_funcs i945gm_crtc_funcs = {
15069         INTEL_CRTC_FUNCS,
15070
15071         .get_vblank_counter = i915_get_vblank_counter,
15072         .enable_vblank = i945gm_enable_vblank,
15073         .disable_vblank = i945gm_disable_vblank,
15074 };
15075
15076 static const struct drm_crtc_funcs i915_crtc_funcs = {
15077         INTEL_CRTC_FUNCS,
15078
15079         .get_vblank_counter = i915_get_vblank_counter,
15080         .enable_vblank = i8xx_enable_vblank,
15081         .disable_vblank = i8xx_disable_vblank,
15082 };
15083
15084 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
15085         INTEL_CRTC_FUNCS,
15086
15087         /* no hw vblank counter */
15088         .enable_vblank = i8xx_enable_vblank,
15089         .disable_vblank = i8xx_disable_vblank,
15090 };
15091
15092 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
15093 {
15094         const struct drm_crtc_funcs *funcs;
15095         struct intel_crtc *intel_crtc;
15096         struct intel_crtc_state *crtc_state = NULL;
15097         struct intel_plane *primary = NULL;
15098         struct intel_plane *cursor = NULL;
15099         int sprite, ret;
15100
15101         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
15102         if (!intel_crtc)
15103                 return -ENOMEM;
15104
15105         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
15106         if (!crtc_state) {
15107                 ret = -ENOMEM;
15108                 goto fail;
15109         }
15110         __drm_atomic_helper_crtc_reset(&intel_crtc->base, &crtc_state->base);
15111         intel_crtc->config = crtc_state;
15112
15113         primary = intel_primary_plane_create(dev_priv, pipe);
15114         if (IS_ERR(primary)) {
15115                 ret = PTR_ERR(primary);
15116                 goto fail;
15117         }
15118         intel_crtc->plane_ids_mask |= BIT(primary->id);
15119
15120         for_each_sprite(dev_priv, pipe, sprite) {
15121                 struct intel_plane *plane;
15122
15123                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
15124                 if (IS_ERR(plane)) {
15125                         ret = PTR_ERR(plane);
15126                         goto fail;
15127                 }
15128                 intel_crtc->plane_ids_mask |= BIT(plane->id);
15129         }
15130
15131         cursor = intel_cursor_plane_create(dev_priv, pipe);
15132         if (IS_ERR(cursor)) {
15133                 ret = PTR_ERR(cursor);
15134                 goto fail;
15135         }
15136         intel_crtc->plane_ids_mask |= BIT(cursor->id);
15137
15138         if (HAS_GMCH(dev_priv)) {
15139                 if (IS_CHERRYVIEW(dev_priv) ||
15140                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
15141                         funcs = &g4x_crtc_funcs;
15142                 else if (IS_GEN(dev_priv, 4))
15143                         funcs = &i965_crtc_funcs;
15144                 else if (IS_I945GM(dev_priv))
15145                         funcs = &i945gm_crtc_funcs;
15146                 else if (IS_GEN(dev_priv, 3))
15147                         funcs = &i915_crtc_funcs;
15148                 else
15149                         funcs = &i8xx_crtc_funcs;
15150         } else {
15151                 if (INTEL_GEN(dev_priv) >= 8)
15152                         funcs = &bdw_crtc_funcs;
15153                 else
15154                         funcs = &ilk_crtc_funcs;
15155         }
15156
15157         ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base,
15158                                         &primary->base, &cursor->base,
15159                                         funcs, "pipe %c", pipe_name(pipe));
15160         if (ret)
15161                 goto fail;
15162
15163         intel_crtc->pipe = pipe;
15164
15165         /* initialize shared scalers */
15166         intel_crtc_init_scalers(intel_crtc, crtc_state);
15167
15168         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
15169                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
15170         dev_priv->pipe_to_crtc_mapping[pipe] = intel_crtc;
15171
15172         if (INTEL_GEN(dev_priv) < 9) {
15173                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
15174
15175                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
15176                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
15177                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = intel_crtc;
15178         }
15179
15180         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
15181
15182         intel_color_init(intel_crtc);
15183
15184         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
15185
15186         return 0;
15187
15188 fail:
15189         /*
15190          * drm_mode_config_cleanup() will free up any
15191          * crtcs/planes already initialized.
15192          */
15193         kfree(crtc_state);
15194         kfree(intel_crtc);
15195
15196         return ret;
15197 }
15198
15199 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
15200                                       struct drm_file *file)
15201 {
15202         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
15203         struct drm_crtc *drmmode_crtc;
15204         struct intel_crtc *crtc;
15205
15206         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
15207         if (!drmmode_crtc)
15208                 return -ENOENT;
15209
15210         crtc = to_intel_crtc(drmmode_crtc);
15211         pipe_from_crtc_id->pipe = crtc->pipe;
15212
15213         return 0;
15214 }
15215
15216 static int intel_encoder_clones(struct intel_encoder *encoder)
15217 {
15218         struct drm_device *dev = encoder->base.dev;
15219         struct intel_encoder *source_encoder;
15220         int index_mask = 0;
15221         int entry = 0;
15222
15223         for_each_intel_encoder(dev, source_encoder) {
15224                 if (encoders_cloneable(encoder, source_encoder))
15225                         index_mask |= (1 << entry);
15226
15227                 entry++;
15228         }
15229
15230         return index_mask;
15231 }
15232
15233 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
15234 {
15235         if (!IS_MOBILE(dev_priv))
15236                 return false;
15237
15238         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
15239                 return false;
15240
15241         if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
15242                 return false;
15243
15244         return true;
15245 }
15246
15247 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
15248 {
15249         if (INTEL_GEN(dev_priv) >= 9)
15250                 return false;
15251
15252         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
15253                 return false;
15254
15255         if (HAS_PCH_LPT_H(dev_priv) &&
15256             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
15257                 return false;
15258
15259         /* DDI E can't be used if DDI A requires 4 lanes */
15260         if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
15261                 return false;
15262
15263         if (!dev_priv->vbt.int_crt_support)
15264                 return false;
15265
15266         return true;
15267 }
15268
15269 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
15270 {
15271         int pps_num;
15272         int pps_idx;
15273
15274         if (HAS_DDI(dev_priv))
15275                 return;
15276         /*
15277          * This w/a is needed at least on CPT/PPT, but to be sure apply it
15278          * everywhere where registers can be write protected.
15279          */
15280         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15281                 pps_num = 2;
15282         else
15283                 pps_num = 1;
15284
15285         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
15286                 u32 val = I915_READ(PP_CONTROL(pps_idx));
15287
15288                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
15289                 I915_WRITE(PP_CONTROL(pps_idx), val);
15290         }
15291 }
15292
15293 static void intel_pps_init(struct drm_i915_private *dev_priv)
15294 {
15295         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
15296                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
15297         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15298                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
15299         else
15300                 dev_priv->pps_mmio_base = PPS_BASE;
15301
15302         intel_pps_unlock_regs_wa(dev_priv);
15303 }
15304
15305 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
15306 {
15307         struct intel_encoder *encoder;
15308         bool dpd_is_edp = false;
15309
15310         intel_pps_init(dev_priv);
15311
15312         if (!HAS_DISPLAY(dev_priv))
15313                 return;
15314
15315         if (INTEL_GEN(dev_priv) >= 12) {
15316                 /* TODO: initialize TC ports as well */
15317                 intel_ddi_init(dev_priv, PORT_A);
15318                 intel_ddi_init(dev_priv, PORT_B);
15319                 icl_dsi_init(dev_priv);
15320         } else if (IS_ELKHARTLAKE(dev_priv)) {
15321                 intel_ddi_init(dev_priv, PORT_A);
15322                 intel_ddi_init(dev_priv, PORT_B);
15323                 intel_ddi_init(dev_priv, PORT_C);
15324                 intel_ddi_init(dev_priv, PORT_D);
15325                 icl_dsi_init(dev_priv);
15326         } else if (IS_GEN(dev_priv, 11)) {
15327                 intel_ddi_init(dev_priv, PORT_A);
15328                 intel_ddi_init(dev_priv, PORT_B);
15329                 intel_ddi_init(dev_priv, PORT_C);
15330                 intel_ddi_init(dev_priv, PORT_D);
15331                 intel_ddi_init(dev_priv, PORT_E);
15332                 /*
15333                  * On some ICL SKUs port F is not present. No strap bits for
15334                  * this, so rely on VBT.
15335                  * Work around broken VBTs on SKUs known to have no port F.
15336                  */
15337                 if (IS_ICL_WITH_PORT_F(dev_priv) &&
15338                     intel_bios_is_port_present(dev_priv, PORT_F))
15339                         intel_ddi_init(dev_priv, PORT_F);
15340
15341                 icl_dsi_init(dev_priv);
15342         } else if (IS_GEN9_LP(dev_priv)) {
15343                 /*
15344                  * FIXME: Broxton doesn't support port detection via the
15345                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
15346                  * detect the ports.
15347                  */
15348                 intel_ddi_init(dev_priv, PORT_A);
15349                 intel_ddi_init(dev_priv, PORT_B);
15350                 intel_ddi_init(dev_priv, PORT_C);
15351
15352                 vlv_dsi_init(dev_priv);
15353         } else if (HAS_DDI(dev_priv)) {
15354                 int found;
15355
15356                 if (intel_ddi_crt_present(dev_priv))
15357                         intel_crt_init(dev_priv);
15358
15359                 /*
15360                  * Haswell uses DDI functions to detect digital outputs.
15361                  * On SKL pre-D0 the strap isn't connected, so we assume
15362                  * it's there.
15363                  */
15364                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
15365                 /* WaIgnoreDDIAStrap: skl */
15366                 if (found || IS_GEN9_BC(dev_priv))
15367                         intel_ddi_init(dev_priv, PORT_A);
15368
15369                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
15370                  * register */
15371                 found = I915_READ(SFUSE_STRAP);
15372
15373                 if (found & SFUSE_STRAP_DDIB_DETECTED)
15374                         intel_ddi_init(dev_priv, PORT_B);
15375                 if (found & SFUSE_STRAP_DDIC_DETECTED)
15376                         intel_ddi_init(dev_priv, PORT_C);
15377                 if (found & SFUSE_STRAP_DDID_DETECTED)
15378                         intel_ddi_init(dev_priv, PORT_D);
15379                 if (found & SFUSE_STRAP_DDIF_DETECTED)
15380                         intel_ddi_init(dev_priv, PORT_F);
15381                 /*
15382                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
15383                  */
15384                 if (IS_GEN9_BC(dev_priv) &&
15385                     intel_bios_is_port_present(dev_priv, PORT_E))
15386                         intel_ddi_init(dev_priv, PORT_E);
15387
15388         } else if (HAS_PCH_SPLIT(dev_priv)) {
15389                 int found;
15390
15391                 /*
15392                  * intel_edp_init_connector() depends on this completing first,
15393                  * to prevent the registration of both eDP and LVDS and the
15394                  * incorrect sharing of the PPS.
15395                  */
15396                 intel_lvds_init(dev_priv);
15397                 intel_crt_init(dev_priv);
15398
15399                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
15400
15401                 if (ilk_has_edp_a(dev_priv))
15402                         intel_dp_init(dev_priv, DP_A, PORT_A);
15403
15404                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
15405                         /* PCH SDVOB multiplex with HDMIB */
15406                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
15407                         if (!found)
15408                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
15409                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
15410                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
15411                 }
15412
15413                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
15414                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
15415
15416                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
15417                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
15418
15419                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
15420                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
15421
15422                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
15423                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
15424         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15425                 bool has_edp, has_port;
15426
15427                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
15428                         intel_crt_init(dev_priv);
15429
15430                 /*
15431                  * The DP_DETECTED bit is the latched state of the DDC
15432                  * SDA pin at boot. However since eDP doesn't require DDC
15433                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
15434                  * eDP ports may have been muxed to an alternate function.
15435                  * Thus we can't rely on the DP_DETECTED bit alone to detect
15436                  * eDP ports. Consult the VBT as well as DP_DETECTED to
15437                  * detect eDP ports.
15438                  *
15439                  * Sadly the straps seem to be missing sometimes even for HDMI
15440                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
15441                  * and VBT for the presence of the port. Additionally we can't
15442                  * trust the port type the VBT declares as we've seen at least
15443                  * HDMI ports that the VBT claim are DP or eDP.
15444                  */
15445                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
15446                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
15447                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
15448                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
15449                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
15450                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
15451
15452                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
15453                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
15454                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
15455                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
15456                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
15457                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
15458
15459                 if (IS_CHERRYVIEW(dev_priv)) {
15460                         /*
15461                          * eDP not supported on port D,
15462                          * so no need to worry about it
15463                          */
15464                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
15465                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
15466                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
15467                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
15468                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
15469                 }
15470
15471                 vlv_dsi_init(dev_priv);
15472         } else if (IS_PINEVIEW(dev_priv)) {
15473                 intel_lvds_init(dev_priv);
15474                 intel_crt_init(dev_priv);
15475         } else if (IS_GEN_RANGE(dev_priv, 3, 4)) {
15476                 bool found = false;
15477
15478                 if (IS_MOBILE(dev_priv))
15479                         intel_lvds_init(dev_priv);
15480
15481                 intel_crt_init(dev_priv);
15482
15483                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
15484                         DRM_DEBUG_KMS("probing SDVOB\n");
15485                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
15486                         if (!found && IS_G4X(dev_priv)) {
15487                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
15488                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
15489                         }
15490
15491                         if (!found && IS_G4X(dev_priv))
15492                                 intel_dp_init(dev_priv, DP_B, PORT_B);
15493                 }
15494
15495                 /* Before G4X SDVOC doesn't have its own detect register */
15496
15497                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
15498                         DRM_DEBUG_KMS("probing SDVOC\n");
15499                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
15500                 }
15501
15502                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
15503
15504                         if (IS_G4X(dev_priv)) {
15505                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
15506                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
15507                         }
15508                         if (IS_G4X(dev_priv))
15509                                 intel_dp_init(dev_priv, DP_C, PORT_C);
15510                 }
15511
15512                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
15513                         intel_dp_init(dev_priv, DP_D, PORT_D);
15514
15515                 if (SUPPORTS_TV(dev_priv))
15516                         intel_tv_init(dev_priv);
15517         } else if (IS_GEN(dev_priv, 2)) {
15518                 if (IS_I85X(dev_priv))
15519                         intel_lvds_init(dev_priv);
15520
15521                 intel_crt_init(dev_priv);
15522                 intel_dvo_init(dev_priv);
15523         }
15524
15525         intel_psr_init(dev_priv);
15526
15527         for_each_intel_encoder(&dev_priv->drm, encoder) {
15528                 encoder->base.possible_crtcs = encoder->crtc_mask;
15529                 encoder->base.possible_clones =
15530                         intel_encoder_clones(encoder);
15531         }
15532
15533         intel_init_pch_refclk(dev_priv);
15534
15535         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
15536 }
15537
15538 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
15539 {
15540         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
15541
15542         drm_framebuffer_cleanup(fb);
15543         intel_frontbuffer_put(intel_fb->frontbuffer);
15544
15545         kfree(intel_fb);
15546 }
15547
15548 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
15549                                                 struct drm_file *file,
15550                                                 unsigned int *handle)
15551 {
15552         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15553
15554         if (obj->userptr.mm) {
15555                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
15556                 return -EINVAL;
15557         }
15558
15559         return drm_gem_handle_create(file, &obj->base, handle);
15560 }
15561
15562 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
15563                                         struct drm_file *file,
15564                                         unsigned flags, unsigned color,
15565                                         struct drm_clip_rect *clips,
15566                                         unsigned num_clips)
15567 {
15568         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15569
15570         i915_gem_object_flush_if_display(obj);
15571         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
15572
15573         return 0;
15574 }
15575
15576 static const struct drm_framebuffer_funcs intel_fb_funcs = {
15577         .destroy = intel_user_framebuffer_destroy,
15578         .create_handle = intel_user_framebuffer_create_handle,
15579         .dirty = intel_user_framebuffer_dirty,
15580 };
15581
15582 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
15583                                   struct drm_i915_gem_object *obj,
15584                                   struct drm_mode_fb_cmd2 *mode_cmd)
15585 {
15586         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
15587         struct drm_framebuffer *fb = &intel_fb->base;
15588         u32 max_stride;
15589         unsigned int tiling, stride;
15590         int ret = -EINVAL;
15591         int i;
15592
15593         intel_fb->frontbuffer = intel_frontbuffer_get(obj);
15594         if (!intel_fb->frontbuffer)
15595                 return -ENOMEM;
15596
15597         i915_gem_object_lock(obj);
15598         tiling = i915_gem_object_get_tiling(obj);
15599         stride = i915_gem_object_get_stride(obj);
15600         i915_gem_object_unlock(obj);
15601
15602         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
15603                 /*
15604                  * If there's a fence, enforce that
15605                  * the fb modifier and tiling mode match.
15606                  */
15607                 if (tiling != I915_TILING_NONE &&
15608                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
15609                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
15610                         goto err;
15611                 }
15612         } else {
15613                 if (tiling == I915_TILING_X) {
15614                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
15615                 } else if (tiling == I915_TILING_Y) {
15616                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
15617                         goto err;
15618                 }
15619         }
15620
15621         if (!drm_any_plane_has_format(&dev_priv->drm,
15622                                       mode_cmd->pixel_format,
15623                                       mode_cmd->modifier[0])) {
15624                 struct drm_format_name_buf format_name;
15625
15626                 DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n",
15627                               drm_get_format_name(mode_cmd->pixel_format,
15628                                                   &format_name),
15629                               mode_cmd->modifier[0]);
15630                 goto err;
15631         }
15632
15633         /*
15634          * gen2/3 display engine uses the fence if present,
15635          * so the tiling mode must match the fb modifier exactly.
15636          */
15637         if (INTEL_GEN(dev_priv) < 4 &&
15638             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
15639                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
15640                 goto err;
15641         }
15642
15643         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
15644                                          mode_cmd->modifier[0]);
15645         if (mode_cmd->pitches[0] > max_stride) {
15646                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
15647                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
15648                               "tiled" : "linear",
15649                               mode_cmd->pitches[0], max_stride);
15650                 goto err;
15651         }
15652
15653         /*
15654          * If there's a fence, enforce that
15655          * the fb pitch and fence stride match.
15656          */
15657         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
15658                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
15659                               mode_cmd->pitches[0], stride);
15660                 goto err;
15661         }
15662
15663         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
15664         if (mode_cmd->offsets[0] != 0)
15665                 goto err;
15666
15667         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
15668
15669         for (i = 0; i < fb->format->num_planes; i++) {
15670                 u32 stride_alignment;
15671
15672                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
15673                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
15674                         goto err;
15675                 }
15676
15677                 stride_alignment = intel_fb_stride_alignment(fb, i);
15678
15679                 /*
15680                  * Display WA #0531: skl,bxt,kbl,glk
15681                  *
15682                  * Render decompression and plane width > 3840
15683                  * combined with horizontal panning requires the
15684                  * plane stride to be a multiple of 4. We'll just
15685                  * require the entire fb to accommodate that to avoid
15686                  * potential runtime errors at plane configuration time.
15687                  */
15688                 if (IS_GEN(dev_priv, 9) && i == 0 && fb->width > 3840 &&
15689                     is_ccs_modifier(fb->modifier))
15690                         stride_alignment *= 4;
15691
15692                 if (fb->pitches[i] & (stride_alignment - 1)) {
15693                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
15694                                       i, fb->pitches[i], stride_alignment);
15695                         goto err;
15696                 }
15697
15698                 fb->obj[i] = &obj->base;
15699         }
15700
15701         ret = intel_fill_fb_info(dev_priv, fb);
15702         if (ret)
15703                 goto err;
15704
15705         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
15706         if (ret) {
15707                 DRM_ERROR("framebuffer init failed %d\n", ret);
15708                 goto err;
15709         }
15710
15711         return 0;
15712
15713 err:
15714         intel_frontbuffer_put(intel_fb->frontbuffer);
15715         return ret;
15716 }
15717
15718 static struct drm_framebuffer *
15719 intel_user_framebuffer_create(struct drm_device *dev,
15720                               struct drm_file *filp,
15721                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
15722 {
15723         struct drm_framebuffer *fb;
15724         struct drm_i915_gem_object *obj;
15725         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
15726
15727         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
15728         if (!obj)
15729                 return ERR_PTR(-ENOENT);
15730
15731         fb = intel_framebuffer_create(obj, &mode_cmd);
15732         i915_gem_object_put(obj);
15733
15734         return fb;
15735 }
15736
15737 static void intel_atomic_state_free(struct drm_atomic_state *state)
15738 {
15739         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
15740
15741         drm_atomic_state_default_release(state);
15742
15743         i915_sw_fence_fini(&intel_state->commit_ready);
15744
15745         kfree(state);
15746 }
15747
15748 static enum drm_mode_status
15749 intel_mode_valid(struct drm_device *dev,
15750                  const struct drm_display_mode *mode)
15751 {
15752         struct drm_i915_private *dev_priv = to_i915(dev);
15753         int hdisplay_max, htotal_max;
15754         int vdisplay_max, vtotal_max;
15755
15756         /*
15757          * Can't reject DBLSCAN here because Xorg ddxen can add piles
15758          * of DBLSCAN modes to the output's mode list when they detect
15759          * the scaling mode property on the connector. And they don't
15760          * ask the kernel to validate those modes in any way until
15761          * modeset time at which point the client gets a protocol error.
15762          * So in order to not upset those clients we silently ignore the
15763          * DBLSCAN flag on such connectors. For other connectors we will
15764          * reject modes with the DBLSCAN flag in encoder->compute_config().
15765          * And we always reject DBLSCAN modes in connector->mode_valid()
15766          * as we never want such modes on the connector's mode list.
15767          */
15768
15769         if (mode->vscan > 1)
15770                 return MODE_NO_VSCAN;
15771
15772         if (mode->flags & DRM_MODE_FLAG_HSKEW)
15773                 return MODE_H_ILLEGAL;
15774
15775         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
15776                            DRM_MODE_FLAG_NCSYNC |
15777                            DRM_MODE_FLAG_PCSYNC))
15778                 return MODE_HSYNC;
15779
15780         if (mode->flags & (DRM_MODE_FLAG_BCAST |
15781                            DRM_MODE_FLAG_PIXMUX |
15782                            DRM_MODE_FLAG_CLKDIV2))
15783                 return MODE_BAD;
15784
15785         if (INTEL_GEN(dev_priv) >= 9 ||
15786             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
15787                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
15788                 vdisplay_max = 4096;
15789                 htotal_max = 8192;
15790                 vtotal_max = 8192;
15791         } else if (INTEL_GEN(dev_priv) >= 3) {
15792                 hdisplay_max = 4096;
15793                 vdisplay_max = 4096;
15794                 htotal_max = 8192;
15795                 vtotal_max = 8192;
15796         } else {
15797                 hdisplay_max = 2048;
15798                 vdisplay_max = 2048;
15799                 htotal_max = 4096;
15800                 vtotal_max = 4096;
15801         }
15802
15803         if (mode->hdisplay > hdisplay_max ||
15804             mode->hsync_start > htotal_max ||
15805             mode->hsync_end > htotal_max ||
15806             mode->htotal > htotal_max)
15807                 return MODE_H_ILLEGAL;
15808
15809         if (mode->vdisplay > vdisplay_max ||
15810             mode->vsync_start > vtotal_max ||
15811             mode->vsync_end > vtotal_max ||
15812             mode->vtotal > vtotal_max)
15813                 return MODE_V_ILLEGAL;
15814
15815         return MODE_OK;
15816 }
15817
15818 static const struct drm_mode_config_funcs intel_mode_funcs = {
15819         .fb_create = intel_user_framebuffer_create,
15820         .get_format_info = intel_get_format_info,
15821         .output_poll_changed = intel_fbdev_output_poll_changed,
15822         .mode_valid = intel_mode_valid,
15823         .atomic_check = intel_atomic_check,
15824         .atomic_commit = intel_atomic_commit,
15825         .atomic_state_alloc = intel_atomic_state_alloc,
15826         .atomic_state_clear = intel_atomic_state_clear,
15827         .atomic_state_free = intel_atomic_state_free,
15828 };
15829
15830 /**
15831  * intel_init_display_hooks - initialize the display modesetting hooks
15832  * @dev_priv: device private
15833  */
15834 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
15835 {
15836         intel_init_cdclk_hooks(dev_priv);
15837
15838         if (INTEL_GEN(dev_priv) >= 9) {
15839                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
15840                 dev_priv->display.get_initial_plane_config =
15841                         skylake_get_initial_plane_config;
15842                 dev_priv->display.crtc_compute_clock =
15843                         haswell_crtc_compute_clock;
15844                 dev_priv->display.crtc_enable = haswell_crtc_enable;
15845                 dev_priv->display.crtc_disable = haswell_crtc_disable;
15846         } else if (HAS_DDI(dev_priv)) {
15847                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
15848                 dev_priv->display.get_initial_plane_config =
15849                         i9xx_get_initial_plane_config;
15850                 dev_priv->display.crtc_compute_clock =
15851                         haswell_crtc_compute_clock;
15852                 dev_priv->display.crtc_enable = haswell_crtc_enable;
15853                 dev_priv->display.crtc_disable = haswell_crtc_disable;
15854         } else if (HAS_PCH_SPLIT(dev_priv)) {
15855                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
15856                 dev_priv->display.get_initial_plane_config =
15857                         i9xx_get_initial_plane_config;
15858                 dev_priv->display.crtc_compute_clock =
15859                         ironlake_crtc_compute_clock;
15860                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
15861                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
15862         } else if (IS_CHERRYVIEW(dev_priv)) {
15863                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15864                 dev_priv->display.get_initial_plane_config =
15865                         i9xx_get_initial_plane_config;
15866                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
15867                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
15868                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15869         } else if (IS_VALLEYVIEW(dev_priv)) {
15870                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15871                 dev_priv->display.get_initial_plane_config =
15872                         i9xx_get_initial_plane_config;
15873                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
15874                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
15875                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15876         } else if (IS_G4X(dev_priv)) {
15877                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15878                 dev_priv->display.get_initial_plane_config =
15879                         i9xx_get_initial_plane_config;
15880                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
15881                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15882                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15883         } else if (IS_PINEVIEW(dev_priv)) {
15884                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15885                 dev_priv->display.get_initial_plane_config =
15886                         i9xx_get_initial_plane_config;
15887                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
15888                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15889                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15890         } else if (!IS_GEN(dev_priv, 2)) {
15891                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15892                 dev_priv->display.get_initial_plane_config =
15893                         i9xx_get_initial_plane_config;
15894                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
15895                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15896                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15897         } else {
15898                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15899                 dev_priv->display.get_initial_plane_config =
15900                         i9xx_get_initial_plane_config;
15901                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
15902                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15903                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15904         }
15905
15906         if (IS_GEN(dev_priv, 5)) {
15907                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
15908         } else if (IS_GEN(dev_priv, 6)) {
15909                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
15910         } else if (IS_IVYBRIDGE(dev_priv)) {
15911                 /* FIXME: detect B0+ stepping and use auto training */
15912                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
15913         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
15914                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
15915         }
15916
15917         if (INTEL_GEN(dev_priv) >= 9)
15918                 dev_priv->display.update_crtcs = skl_update_crtcs;
15919         else
15920                 dev_priv->display.update_crtcs = intel_update_crtcs;
15921 }
15922
15923 static i915_reg_t i915_vgacntrl_reg(struct drm_i915_private *dev_priv)
15924 {
15925         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15926                 return VLV_VGACNTRL;
15927         else if (INTEL_GEN(dev_priv) >= 5)
15928                 return CPU_VGACNTRL;
15929         else
15930                 return VGACNTRL;
15931 }
15932
15933 /* Disable the VGA plane that we never use */
15934 static void i915_disable_vga(struct drm_i915_private *dev_priv)
15935 {
15936         struct pci_dev *pdev = dev_priv->drm.pdev;
15937         u8 sr1;
15938         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
15939
15940         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
15941         vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
15942         outb(SR01, VGA_SR_INDEX);
15943         sr1 = inb(VGA_SR_DATA);
15944         outb(sr1 | 1<<5, VGA_SR_DATA);
15945         vga_put(pdev, VGA_RSRC_LEGACY_IO);
15946         udelay(300);
15947
15948         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
15949         POSTING_READ(vga_reg);
15950 }
15951
15952 void intel_modeset_init_hw(struct drm_device *dev)
15953 {
15954         struct drm_i915_private *dev_priv = to_i915(dev);
15955
15956         intel_update_cdclk(dev_priv);
15957         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
15958         dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw;
15959 }
15960
15961 /*
15962  * Calculate what we think the watermarks should be for the state we've read
15963  * out of the hardware and then immediately program those watermarks so that
15964  * we ensure the hardware settings match our internal state.
15965  *
15966  * We can calculate what we think WM's should be by creating a duplicate of the
15967  * current state (which was constructed during hardware readout) and running it
15968  * through the atomic check code to calculate new watermark values in the
15969  * state object.
15970  */
15971 static void sanitize_watermarks(struct drm_device *dev)
15972 {
15973         struct drm_i915_private *dev_priv = to_i915(dev);
15974         struct drm_atomic_state *state;
15975         struct intel_atomic_state *intel_state;
15976         struct intel_crtc *crtc;
15977         struct intel_crtc_state *crtc_state;
15978         struct drm_modeset_acquire_ctx ctx;
15979         int ret;
15980         int i;
15981
15982         /* Only supported on platforms that use atomic watermark design */
15983         if (!dev_priv->display.optimize_watermarks)
15984                 return;
15985
15986         /*
15987          * We need to hold connection_mutex before calling duplicate_state so
15988          * that the connector loop is protected.
15989          */
15990         drm_modeset_acquire_init(&ctx, 0);
15991 retry:
15992         ret = drm_modeset_lock_all_ctx(dev, &ctx);
15993         if (ret == -EDEADLK) {
15994                 drm_modeset_backoff(&ctx);
15995                 goto retry;
15996         } else if (WARN_ON(ret)) {
15997                 goto fail;
15998         }
15999
16000         state = drm_atomic_helper_duplicate_state(dev, &ctx);
16001         if (WARN_ON(IS_ERR(state)))
16002                 goto fail;
16003
16004         intel_state = to_intel_atomic_state(state);
16005
16006         /*
16007          * Hardware readout is the only time we don't want to calculate
16008          * intermediate watermarks (since we don't trust the current
16009          * watermarks).
16010          */
16011         if (!HAS_GMCH(dev_priv))
16012                 intel_state->skip_intermediate_wm = true;
16013
16014         ret = intel_atomic_check(dev, state);
16015         if (ret) {
16016                 /*
16017                  * If we fail here, it means that the hardware appears to be
16018                  * programmed in a way that shouldn't be possible, given our
16019                  * understanding of watermark requirements.  This might mean a
16020                  * mistake in the hardware readout code or a mistake in the
16021                  * watermark calculations for a given platform.  Raise a WARN
16022                  * so that this is noticeable.
16023                  *
16024                  * If this actually happens, we'll have to just leave the
16025                  * BIOS-programmed watermarks untouched and hope for the best.
16026                  */
16027                 WARN(true, "Could not determine valid watermarks for inherited state\n");
16028                 goto put_state;
16029         }
16030
16031         /* Write calculated watermark values back */
16032         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
16033                 crtc_state->wm.need_postvbl_update = true;
16034                 dev_priv->display.optimize_watermarks(intel_state, crtc_state);
16035
16036                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
16037         }
16038
16039 put_state:
16040         drm_atomic_state_put(state);
16041 fail:
16042         drm_modeset_drop_locks(&ctx);
16043         drm_modeset_acquire_fini(&ctx);
16044 }
16045
16046 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
16047 {
16048         if (IS_GEN(dev_priv, 5)) {
16049                 u32 fdi_pll_clk =
16050                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
16051
16052                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
16053         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
16054                 dev_priv->fdi_pll_freq = 270000;
16055         } else {
16056                 return;
16057         }
16058
16059         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
16060 }
16061
16062 static int intel_initial_commit(struct drm_device *dev)
16063 {
16064         struct drm_atomic_state *state = NULL;
16065         struct drm_modeset_acquire_ctx ctx;
16066         struct drm_crtc *crtc;
16067         struct drm_crtc_state *crtc_state;
16068         int ret = 0;
16069
16070         state = drm_atomic_state_alloc(dev);
16071         if (!state)
16072                 return -ENOMEM;
16073
16074         drm_modeset_acquire_init(&ctx, 0);
16075
16076 retry:
16077         state->acquire_ctx = &ctx;
16078
16079         drm_for_each_crtc(crtc, dev) {
16080                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
16081                 if (IS_ERR(crtc_state)) {
16082                         ret = PTR_ERR(crtc_state);
16083                         goto out;
16084                 }
16085
16086                 if (crtc_state->active) {
16087                         ret = drm_atomic_add_affected_planes(state, crtc);
16088                         if (ret)
16089                                 goto out;
16090
16091                         /*
16092                          * FIXME hack to force a LUT update to avoid the
16093                          * plane update forcing the pipe gamma on without
16094                          * having a proper LUT loaded. Remove once we
16095                          * have readout for pipe gamma enable.
16096                          */
16097                         crtc_state->color_mgmt_changed = true;
16098                 }
16099         }
16100
16101         ret = drm_atomic_commit(state);
16102
16103 out:
16104         if (ret == -EDEADLK) {
16105                 drm_atomic_state_clear(state);
16106                 drm_modeset_backoff(&ctx);
16107                 goto retry;
16108         }
16109
16110         drm_atomic_state_put(state);
16111
16112         drm_modeset_drop_locks(&ctx);
16113         drm_modeset_acquire_fini(&ctx);
16114
16115         return ret;
16116 }
16117
16118 int intel_modeset_init(struct drm_device *dev)
16119 {
16120         struct drm_i915_private *dev_priv = to_i915(dev);
16121         enum pipe pipe;
16122         struct intel_crtc *crtc;
16123         int ret;
16124
16125         dev_priv->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
16126
16127         drm_mode_config_init(dev);
16128
16129         ret = intel_bw_init(dev_priv);
16130         if (ret)
16131                 return ret;
16132
16133         dev->mode_config.min_width = 0;
16134         dev->mode_config.min_height = 0;
16135
16136         dev->mode_config.preferred_depth = 24;
16137         dev->mode_config.prefer_shadow = 1;
16138
16139         dev->mode_config.allow_fb_modifiers = true;
16140
16141         dev->mode_config.funcs = &intel_mode_funcs;
16142
16143         init_llist_head(&dev_priv->atomic_helper.free_list);
16144         INIT_WORK(&dev_priv->atomic_helper.free_work,
16145                   intel_atomic_helper_free_state_worker);
16146
16147         intel_init_quirks(dev_priv);
16148
16149         intel_fbc_init(dev_priv);
16150
16151         intel_init_pm(dev_priv);
16152
16153         /*
16154          * There may be no VBT; and if the BIOS enabled SSC we can
16155          * just keep using it to avoid unnecessary flicker.  Whereas if the
16156          * BIOS isn't using it, don't assume it will work even if the VBT
16157          * indicates as much.
16158          */
16159         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
16160                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
16161                                             DREF_SSC1_ENABLE);
16162
16163                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
16164                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
16165                                      bios_lvds_use_ssc ? "en" : "dis",
16166                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
16167                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
16168                 }
16169         }
16170
16171         /*
16172          * Maximum framebuffer dimensions, chosen to match
16173          * the maximum render engine surface size on gen4+.
16174          */
16175         if (INTEL_GEN(dev_priv) >= 7) {
16176                 dev->mode_config.max_width = 16384;
16177                 dev->mode_config.max_height = 16384;
16178         } else if (INTEL_GEN(dev_priv) >= 4) {
16179                 dev->mode_config.max_width = 8192;
16180                 dev->mode_config.max_height = 8192;
16181         } else if (IS_GEN(dev_priv, 3)) {
16182                 dev->mode_config.max_width = 4096;
16183                 dev->mode_config.max_height = 4096;
16184         } else {
16185                 dev->mode_config.max_width = 2048;
16186                 dev->mode_config.max_height = 2048;
16187         }
16188
16189         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
16190                 dev->mode_config.cursor_width = IS_I845G(dev_priv) ? 64 : 512;
16191                 dev->mode_config.cursor_height = 1023;
16192         } else if (IS_GEN(dev_priv, 2)) {
16193                 dev->mode_config.cursor_width = 64;
16194                 dev->mode_config.cursor_height = 64;
16195         } else {
16196                 dev->mode_config.cursor_width = 256;
16197                 dev->mode_config.cursor_height = 256;
16198         }
16199
16200         DRM_DEBUG_KMS("%d display pipe%s available.\n",
16201                       INTEL_INFO(dev_priv)->num_pipes,
16202                       INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : "");
16203
16204         for_each_pipe(dev_priv, pipe) {
16205                 ret = intel_crtc_init(dev_priv, pipe);
16206                 if (ret) {
16207                         drm_mode_config_cleanup(dev);
16208                         return ret;
16209                 }
16210         }
16211
16212         intel_shared_dpll_init(dev);
16213         intel_update_fdi_pll_freq(dev_priv);
16214
16215         intel_update_czclk(dev_priv);
16216         intel_modeset_init_hw(dev);
16217
16218         intel_hdcp_component_init(dev_priv);
16219
16220         if (dev_priv->max_cdclk_freq == 0)
16221                 intel_update_max_cdclk(dev_priv);
16222
16223         /* Just disable it once at startup */
16224         i915_disable_vga(dev_priv);
16225         intel_setup_outputs(dev_priv);
16226
16227         drm_modeset_lock_all(dev);
16228         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
16229         drm_modeset_unlock_all(dev);
16230
16231         for_each_intel_crtc(dev, crtc) {
16232                 struct intel_initial_plane_config plane_config = {};
16233
16234                 if (!crtc->active)
16235                         continue;
16236
16237                 /*
16238                  * Note that reserving the BIOS fb up front prevents us
16239                  * from stuffing other stolen allocations like the ring
16240                  * on top.  This prevents some ugliness at boot time, and
16241                  * can even allow for smooth boot transitions if the BIOS
16242                  * fb is large enough for the active pipe configuration.
16243                  */
16244                 dev_priv->display.get_initial_plane_config(crtc,
16245                                                            &plane_config);
16246
16247                 /*
16248                  * If the fb is shared between multiple heads, we'll
16249                  * just get the first one.
16250                  */
16251                 intel_find_initial_plane_obj(crtc, &plane_config);
16252         }
16253
16254         /*
16255          * Make sure hardware watermarks really match the state we read out.
16256          * Note that we need to do this after reconstructing the BIOS fb's
16257          * since the watermark calculation done here will use pstate->fb.
16258          */
16259         if (!HAS_GMCH(dev_priv))
16260                 sanitize_watermarks(dev);
16261
16262         /*
16263          * Force all active planes to recompute their states. So that on
16264          * mode_setcrtc after probe, all the intel_plane_state variables
16265          * are already calculated and there is no assert_plane warnings
16266          * during bootup.
16267          */
16268         ret = intel_initial_commit(dev);
16269         if (ret)
16270                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
16271
16272         return 0;
16273 }
16274
16275 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
16276 {
16277         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16278         /* 640x480@60Hz, ~25175 kHz */
16279         struct dpll clock = {
16280                 .m1 = 18,
16281                 .m2 = 7,
16282                 .p1 = 13,
16283                 .p2 = 4,
16284                 .n = 2,
16285         };
16286         u32 dpll, fp;
16287         int i;
16288
16289         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
16290
16291         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
16292                       pipe_name(pipe), clock.vco, clock.dot);
16293
16294         fp = i9xx_dpll_compute_fp(&clock);
16295         dpll = DPLL_DVO_2X_MODE |
16296                 DPLL_VGA_MODE_DIS |
16297                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
16298                 PLL_P2_DIVIDE_BY_4 |
16299                 PLL_REF_INPUT_DREFCLK |
16300                 DPLL_VCO_ENABLE;
16301
16302         I915_WRITE(FP0(pipe), fp);
16303         I915_WRITE(FP1(pipe), fp);
16304
16305         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
16306         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
16307         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
16308         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
16309         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
16310         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
16311         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
16312
16313         /*
16314          * Apparently we need to have VGA mode enabled prior to changing
16315          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
16316          * dividers, even though the register value does change.
16317          */
16318         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
16319         I915_WRITE(DPLL(pipe), dpll);
16320
16321         /* Wait for the clocks to stabilize. */
16322         POSTING_READ(DPLL(pipe));
16323         udelay(150);
16324
16325         /* The pixel multiplier can only be updated once the
16326          * DPLL is enabled and the clocks are stable.
16327          *
16328          * So write it again.
16329          */
16330         I915_WRITE(DPLL(pipe), dpll);
16331
16332         /* We do this three times for luck */
16333         for (i = 0; i < 3 ; i++) {
16334                 I915_WRITE(DPLL(pipe), dpll);
16335                 POSTING_READ(DPLL(pipe));
16336                 udelay(150); /* wait for warmup */
16337         }
16338
16339         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
16340         POSTING_READ(PIPECONF(pipe));
16341
16342         intel_wait_for_pipe_scanline_moving(crtc);
16343 }
16344
16345 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
16346 {
16347         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16348
16349         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
16350                       pipe_name(pipe));
16351
16352         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
16353         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
16354         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
16355         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
16356         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
16357
16358         I915_WRITE(PIPECONF(pipe), 0);
16359         POSTING_READ(PIPECONF(pipe));
16360
16361         intel_wait_for_pipe_scanline_stopped(crtc);
16362
16363         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
16364         POSTING_READ(DPLL(pipe));
16365 }
16366
16367 static void
16368 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
16369 {
16370         struct intel_crtc *crtc;
16371
16372         if (INTEL_GEN(dev_priv) >= 4)
16373                 return;
16374
16375         for_each_intel_crtc(&dev_priv->drm, crtc) {
16376                 struct intel_plane *plane =
16377                         to_intel_plane(crtc->base.primary);
16378                 struct intel_crtc *plane_crtc;
16379                 enum pipe pipe;
16380
16381                 if (!plane->get_hw_state(plane, &pipe))
16382                         continue;
16383
16384                 if (pipe == crtc->pipe)
16385                         continue;
16386
16387                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
16388                               plane->base.base.id, plane->base.name);
16389
16390                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16391                 intel_plane_disable_noatomic(plane_crtc, plane);
16392         }
16393 }
16394
16395 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
16396 {
16397         struct drm_device *dev = crtc->base.dev;
16398         struct intel_encoder *encoder;
16399
16400         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
16401                 return true;
16402
16403         return false;
16404 }
16405
16406 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
16407 {
16408         struct drm_device *dev = encoder->base.dev;
16409         struct intel_connector *connector;
16410
16411         for_each_connector_on_encoder(dev, &encoder->base, connector)
16412                 return connector;
16413
16414         return NULL;
16415 }
16416
16417 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
16418                               enum pipe pch_transcoder)
16419 {
16420         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
16421                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
16422 }
16423
16424 static void intel_sanitize_crtc(struct intel_crtc *crtc,
16425                                 struct drm_modeset_acquire_ctx *ctx)
16426 {
16427         struct drm_device *dev = crtc->base.dev;
16428         struct drm_i915_private *dev_priv = to_i915(dev);
16429         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
16430         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
16431
16432         /* Clear any frame start delays used for debugging left by the BIOS */
16433         if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) {
16434                 i915_reg_t reg = PIPECONF(cpu_transcoder);
16435
16436                 I915_WRITE(reg,
16437                            I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
16438         }
16439
16440         if (crtc_state->base.active) {
16441                 struct intel_plane *plane;
16442
16443                 /* Disable everything but the primary plane */
16444                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
16445                         const struct intel_plane_state *plane_state =
16446                                 to_intel_plane_state(plane->base.state);
16447
16448                         if (plane_state->base.visible &&
16449                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
16450                                 intel_plane_disable_noatomic(crtc, plane);
16451                 }
16452
16453                 /*
16454                  * Disable any background color set by the BIOS, but enable the
16455                  * gamma and CSC to match how we program our planes.
16456                  */
16457                 if (INTEL_GEN(dev_priv) >= 9)
16458                         I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe),
16459                                    SKL_BOTTOM_COLOR_GAMMA_ENABLE |
16460                                    SKL_BOTTOM_COLOR_CSC_ENABLE);
16461         }
16462
16463         /* Adjust the state of the output pipe according to whether we
16464          * have active connectors/encoders. */
16465         if (crtc_state->base.active && !intel_crtc_has_encoders(crtc))
16466                 intel_crtc_disable_noatomic(&crtc->base, ctx);
16467
16468         if (crtc_state->base.active || HAS_GMCH(dev_priv)) {
16469                 /*
16470                  * We start out with underrun reporting disabled to avoid races.
16471                  * For correct bookkeeping mark this on active crtcs.
16472                  *
16473                  * Also on gmch platforms we dont have any hardware bits to
16474                  * disable the underrun reporting. Which means we need to start
16475                  * out with underrun reporting disabled also on inactive pipes,
16476                  * since otherwise we'll complain about the garbage we read when
16477                  * e.g. coming up after runtime pm.
16478                  *
16479                  * No protection against concurrent access is required - at
16480                  * worst a fifo underrun happens which also sets this to false.
16481                  */
16482                 crtc->cpu_fifo_underrun_disabled = true;
16483                 /*
16484                  * We track the PCH trancoder underrun reporting state
16485                  * within the crtc. With crtc for pipe A housing the underrun
16486                  * reporting state for PCH transcoder A, crtc for pipe B housing
16487                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
16488                  * and marking underrun reporting as disabled for the non-existing
16489                  * PCH transcoders B and C would prevent enabling the south
16490                  * error interrupt (see cpt_can_enable_serr_int()).
16491                  */
16492                 if (has_pch_trancoder(dev_priv, crtc->pipe))
16493                         crtc->pch_fifo_underrun_disabled = true;
16494         }
16495 }
16496
16497 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
16498 {
16499         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
16500
16501         /*
16502          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
16503          * the hardware when a high res displays plugged in. DPLL P
16504          * divider is zero, and the pipe timings are bonkers. We'll
16505          * try to disable everything in that case.
16506          *
16507          * FIXME would be nice to be able to sanitize this state
16508          * without several WARNs, but for now let's take the easy
16509          * road.
16510          */
16511         return IS_GEN(dev_priv, 6) &&
16512                 crtc_state->base.active &&
16513                 crtc_state->shared_dpll &&
16514                 crtc_state->port_clock == 0;
16515 }
16516
16517 static void intel_sanitize_encoder(struct intel_encoder *encoder)
16518 {
16519         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
16520         struct intel_connector *connector;
16521         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
16522         struct intel_crtc_state *crtc_state = crtc ?
16523                 to_intel_crtc_state(crtc->base.state) : NULL;
16524
16525         /* We need to check both for a crtc link (meaning that the
16526          * encoder is active and trying to read from a pipe) and the
16527          * pipe itself being active. */
16528         bool has_active_crtc = crtc_state &&
16529                 crtc_state->base.active;
16530
16531         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
16532                 DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n",
16533                               pipe_name(crtc->pipe));
16534                 has_active_crtc = false;
16535         }
16536
16537         connector = intel_encoder_find_connector(encoder);
16538         if (connector && !has_active_crtc) {
16539                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
16540                               encoder->base.base.id,
16541                               encoder->base.name);
16542
16543                 /* Connector is active, but has no active pipe. This is
16544                  * fallout from our resume register restoring. Disable
16545                  * the encoder manually again. */
16546                 if (crtc_state) {
16547                         struct drm_encoder *best_encoder;
16548
16549                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
16550                                       encoder->base.base.id,
16551                                       encoder->base.name);
16552
16553                         /* avoid oopsing in case the hooks consult best_encoder */
16554                         best_encoder = connector->base.state->best_encoder;
16555                         connector->base.state->best_encoder = &encoder->base;
16556
16557                         if (encoder->disable)
16558                                 encoder->disable(encoder, crtc_state,
16559                                                  connector->base.state);
16560                         if (encoder->post_disable)
16561                                 encoder->post_disable(encoder, crtc_state,
16562                                                       connector->base.state);
16563
16564                         connector->base.state->best_encoder = best_encoder;
16565                 }
16566                 encoder->base.crtc = NULL;
16567
16568                 /* Inconsistent output/port/pipe state happens presumably due to
16569                  * a bug in one of the get_hw_state functions. Or someplace else
16570                  * in our code, like the register restore mess on resume. Clamp
16571                  * things to off as a safer default. */
16572
16573                 connector->base.dpms = DRM_MODE_DPMS_OFF;
16574                 connector->base.encoder = NULL;
16575         }
16576
16577         /* notify opregion of the sanitized encoder state */
16578         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
16579
16580         if (INTEL_GEN(dev_priv) >= 11)
16581                 icl_sanitize_encoder_pll_mapping(encoder);
16582 }
16583
16584 void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
16585 {
16586         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
16587
16588         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
16589                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
16590                 i915_disable_vga(dev_priv);
16591         }
16592 }
16593
16594 void i915_redisable_vga(struct drm_i915_private *dev_priv)
16595 {
16596         intel_wakeref_t wakeref;
16597
16598         /*
16599          * This function can be called both from intel_modeset_setup_hw_state or
16600          * at a very early point in our resume sequence, where the power well
16601          * structures are not yet restored. Since this function is at a very
16602          * paranoid "someone might have enabled VGA while we were not looking"
16603          * level, just check if the power well is enabled instead of trying to
16604          * follow the "don't touch the power well if we don't need it" policy
16605          * the rest of the driver uses.
16606          */
16607         wakeref = intel_display_power_get_if_enabled(dev_priv,
16608                                                      POWER_DOMAIN_VGA);
16609         if (!wakeref)
16610                 return;
16611
16612         i915_redisable_vga_power_on(dev_priv);
16613
16614         intel_display_power_put(dev_priv, POWER_DOMAIN_VGA, wakeref);
16615 }
16616
16617 /* FIXME read out full plane state for all planes */
16618 static void readout_plane_state(struct drm_i915_private *dev_priv)
16619 {
16620         struct intel_plane *plane;
16621         struct intel_crtc *crtc;
16622
16623         for_each_intel_plane(&dev_priv->drm, plane) {
16624                 struct intel_plane_state *plane_state =
16625                         to_intel_plane_state(plane->base.state);
16626                 struct intel_crtc_state *crtc_state;
16627                 enum pipe pipe = PIPE_A;
16628                 bool visible;
16629
16630                 visible = plane->get_hw_state(plane, &pipe);
16631
16632                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16633                 crtc_state = to_intel_crtc_state(crtc->base.state);
16634
16635                 intel_set_plane_visible(crtc_state, plane_state, visible);
16636
16637                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
16638                               plane->base.base.id, plane->base.name,
16639                               enableddisabled(visible), pipe_name(pipe));
16640         }
16641
16642         for_each_intel_crtc(&dev_priv->drm, crtc) {
16643                 struct intel_crtc_state *crtc_state =
16644                         to_intel_crtc_state(crtc->base.state);
16645
16646                 fixup_active_planes(crtc_state);
16647         }
16648 }
16649
16650 static void intel_modeset_readout_hw_state(struct drm_device *dev)
16651 {
16652         struct drm_i915_private *dev_priv = to_i915(dev);
16653         enum pipe pipe;
16654         struct intel_crtc *crtc;
16655         struct intel_encoder *encoder;
16656         struct intel_connector *connector;
16657         struct drm_connector_list_iter conn_iter;
16658         int i;
16659
16660         dev_priv->active_crtcs = 0;
16661
16662         for_each_intel_crtc(dev, crtc) {
16663                 struct intel_crtc_state *crtc_state =
16664                         to_intel_crtc_state(crtc->base.state);
16665
16666                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->base);
16667                 memset(crtc_state, 0, sizeof(*crtc_state));
16668                 __drm_atomic_helper_crtc_reset(&crtc->base, &crtc_state->base);
16669
16670                 crtc_state->base.active = crtc_state->base.enable =
16671                         dev_priv->display.get_pipe_config(crtc, crtc_state);
16672
16673                 crtc->base.enabled = crtc_state->base.enable;
16674                 crtc->active = crtc_state->base.active;
16675
16676                 if (crtc_state->base.active)
16677                         dev_priv->active_crtcs |= 1 << crtc->pipe;
16678
16679                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
16680                               crtc->base.base.id, crtc->base.name,
16681                               enableddisabled(crtc_state->base.active));
16682         }
16683
16684         readout_plane_state(dev_priv);
16685
16686         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
16687                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
16688
16689                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
16690                                                         &pll->state.hw_state);
16691
16692                 if (IS_ELKHARTLAKE(dev_priv) && pll->on &&
16693                     pll->info->id == DPLL_ID_EHL_DPLL4) {
16694                         pll->wakeref = intel_display_power_get(dev_priv,
16695                                                                POWER_DOMAIN_DPLL_DC_OFF);
16696                 }
16697
16698                 pll->state.crtc_mask = 0;
16699                 for_each_intel_crtc(dev, crtc) {
16700                         struct intel_crtc_state *crtc_state =
16701                                 to_intel_crtc_state(crtc->base.state);
16702
16703                         if (crtc_state->base.active &&
16704                             crtc_state->shared_dpll == pll)
16705                                 pll->state.crtc_mask |= 1 << crtc->pipe;
16706                 }
16707                 pll->active_mask = pll->state.crtc_mask;
16708
16709                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
16710                               pll->info->name, pll->state.crtc_mask, pll->on);
16711         }
16712
16713         for_each_intel_encoder(dev, encoder) {
16714                 pipe = 0;
16715
16716                 if (encoder->get_hw_state(encoder, &pipe)) {
16717                         struct intel_crtc_state *crtc_state;
16718
16719                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16720                         crtc_state = to_intel_crtc_state(crtc->base.state);
16721
16722                         encoder->base.crtc = &crtc->base;
16723                         encoder->get_config(encoder, crtc_state);
16724                 } else {
16725                         encoder->base.crtc = NULL;
16726                 }
16727
16728                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
16729                               encoder->base.base.id, encoder->base.name,
16730                               enableddisabled(encoder->base.crtc),
16731                               pipe_name(pipe));
16732         }
16733
16734         drm_connector_list_iter_begin(dev, &conn_iter);
16735         for_each_intel_connector_iter(connector, &conn_iter) {
16736                 if (connector->get_hw_state(connector)) {
16737                         connector->base.dpms = DRM_MODE_DPMS_ON;
16738
16739                         encoder = connector->encoder;
16740                         connector->base.encoder = &encoder->base;
16741
16742                         if (encoder->base.crtc &&
16743                             encoder->base.crtc->state->active) {
16744                                 /*
16745                                  * This has to be done during hardware readout
16746                                  * because anything calling .crtc_disable may
16747                                  * rely on the connector_mask being accurate.
16748                                  */
16749                                 encoder->base.crtc->state->connector_mask |=
16750                                         drm_connector_mask(&connector->base);
16751                                 encoder->base.crtc->state->encoder_mask |=
16752                                         drm_encoder_mask(&encoder->base);
16753                         }
16754
16755                 } else {
16756                         connector->base.dpms = DRM_MODE_DPMS_OFF;
16757                         connector->base.encoder = NULL;
16758                 }
16759                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
16760                               connector->base.base.id, connector->base.name,
16761                               enableddisabled(connector->base.encoder));
16762         }
16763         drm_connector_list_iter_end(&conn_iter);
16764
16765         for_each_intel_crtc(dev, crtc) {
16766                 struct intel_bw_state *bw_state =
16767                         to_intel_bw_state(dev_priv->bw_obj.state);
16768                 struct intel_crtc_state *crtc_state =
16769                         to_intel_crtc_state(crtc->base.state);
16770                 struct intel_plane *plane;
16771                 int min_cdclk = 0;
16772
16773                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
16774                 if (crtc_state->base.active) {
16775                         intel_mode_from_pipe_config(&crtc->base.mode, crtc_state);
16776                         crtc->base.mode.hdisplay = crtc_state->pipe_src_w;
16777                         crtc->base.mode.vdisplay = crtc_state->pipe_src_h;
16778                         intel_mode_from_pipe_config(&crtc_state->base.adjusted_mode, crtc_state);
16779                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
16780
16781                         /*
16782                          * The initial mode needs to be set in order to keep
16783                          * the atomic core happy. It wants a valid mode if the
16784                          * crtc's enabled, so we do the above call.
16785                          *
16786                          * But we don't set all the derived state fully, hence
16787                          * set a flag to indicate that a full recalculation is
16788                          * needed on the next commit.
16789                          */
16790                         crtc_state->base.mode.private_flags = I915_MODE_FLAG_INHERITED;
16791
16792                         intel_crtc_compute_pixel_rate(crtc_state);
16793
16794                         if (dev_priv->display.modeset_calc_cdclk) {
16795                                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
16796                                 if (WARN_ON(min_cdclk < 0))
16797                                         min_cdclk = 0;
16798                         }
16799
16800                         drm_calc_timestamping_constants(&crtc->base,
16801                                                         &crtc_state->base.adjusted_mode);
16802                         update_scanline_offset(crtc_state);
16803                 }
16804
16805                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
16806                 dev_priv->min_voltage_level[crtc->pipe] =
16807                         crtc_state->min_voltage_level;
16808
16809                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
16810                         const struct intel_plane_state *plane_state =
16811                                 to_intel_plane_state(plane->base.state);
16812
16813                         /*
16814                          * FIXME don't have the fb yet, so can't
16815                          * use intel_plane_data_rate() :(
16816                          */
16817                         if (plane_state->base.visible)
16818                                 crtc_state->data_rate[plane->id] =
16819                                         4 * crtc_state->pixel_rate;
16820                 }
16821
16822                 intel_bw_crtc_update(bw_state, crtc_state);
16823
16824                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
16825         }
16826 }
16827
16828 static void
16829 get_encoder_power_domains(struct drm_i915_private *dev_priv)
16830 {
16831         struct intel_encoder *encoder;
16832
16833         for_each_intel_encoder(&dev_priv->drm, encoder) {
16834                 struct intel_crtc_state *crtc_state;
16835
16836                 if (!encoder->get_power_domains)
16837                         continue;
16838
16839                 /*
16840                  * MST-primary and inactive encoders don't have a crtc state
16841                  * and neither of these require any power domain references.
16842                  */
16843                 if (!encoder->base.crtc)
16844                         continue;
16845
16846                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
16847                 encoder->get_power_domains(encoder, crtc_state);
16848         }
16849 }
16850
16851 static void intel_early_display_was(struct drm_i915_private *dev_priv)
16852 {
16853         /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
16854         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
16855                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
16856                            DARBF_GATING_DIS);
16857
16858         if (IS_HASWELL(dev_priv)) {
16859                 /*
16860                  * WaRsPkgCStateDisplayPMReq:hsw
16861                  * System hang if this isn't done before disabling all planes!
16862                  */
16863                 I915_WRITE(CHICKEN_PAR1_1,
16864                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
16865         }
16866 }
16867
16868 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
16869                                        enum port port, i915_reg_t hdmi_reg)
16870 {
16871         u32 val = I915_READ(hdmi_reg);
16872
16873         if (val & SDVO_ENABLE ||
16874             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
16875                 return;
16876
16877         DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n",
16878                       port_name(port));
16879
16880         val &= ~SDVO_PIPE_SEL_MASK;
16881         val |= SDVO_PIPE_SEL(PIPE_A);
16882
16883         I915_WRITE(hdmi_reg, val);
16884 }
16885
16886 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
16887                                      enum port port, i915_reg_t dp_reg)
16888 {
16889         u32 val = I915_READ(dp_reg);
16890
16891         if (val & DP_PORT_EN ||
16892             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
16893                 return;
16894
16895         DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n",
16896                       port_name(port));
16897
16898         val &= ~DP_PIPE_SEL_MASK;
16899         val |= DP_PIPE_SEL(PIPE_A);
16900
16901         I915_WRITE(dp_reg, val);
16902 }
16903
16904 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
16905 {
16906         /*
16907          * The BIOS may select transcoder B on some of the PCH
16908          * ports even it doesn't enable the port. This would trip
16909          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
16910          * Sanitize the transcoder select bits to prevent that. We
16911          * assume that the BIOS never actually enabled the port,
16912          * because if it did we'd actually have to toggle the port
16913          * on and back off to make the transcoder A select stick
16914          * (see. intel_dp_link_down(), intel_disable_hdmi(),
16915          * intel_disable_sdvo()).
16916          */
16917         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
16918         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
16919         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
16920
16921         /* PCH SDVOB multiplex with HDMIB */
16922         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
16923         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
16924         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
16925 }
16926
16927 /* Scan out the current hw modeset state,
16928  * and sanitizes it to the current state
16929  */
16930 static void
16931 intel_modeset_setup_hw_state(struct drm_device *dev,
16932                              struct drm_modeset_acquire_ctx *ctx)
16933 {
16934         struct drm_i915_private *dev_priv = to_i915(dev);
16935         struct intel_crtc_state *crtc_state;
16936         struct intel_encoder *encoder;
16937         struct intel_crtc *crtc;
16938         intel_wakeref_t wakeref;
16939         int i;
16940
16941         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
16942
16943         intel_early_display_was(dev_priv);
16944         intel_modeset_readout_hw_state(dev);
16945
16946         /* HW state is read out, now we need to sanitize this mess. */
16947
16948         /* Sanitize the TypeC port mode upfront, encoders depend on this */
16949         for_each_intel_encoder(dev, encoder) {
16950                 enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
16951
16952                 /* We need to sanitize only the MST primary port. */
16953                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
16954                     intel_phy_is_tc(dev_priv, phy))
16955                         intel_tc_port_sanitize(enc_to_dig_port(&encoder->base));
16956         }
16957
16958         get_encoder_power_domains(dev_priv);
16959
16960         if (HAS_PCH_IBX(dev_priv))
16961                 ibx_sanitize_pch_ports(dev_priv);
16962
16963         /*
16964          * intel_sanitize_plane_mapping() may need to do vblank
16965          * waits, so we need vblank interrupts restored beforehand.
16966          */
16967         for_each_intel_crtc(&dev_priv->drm, crtc) {
16968                 crtc_state = to_intel_crtc_state(crtc->base.state);
16969
16970                 drm_crtc_vblank_reset(&crtc->base);
16971
16972                 if (crtc_state->base.active)
16973                         intel_crtc_vblank_on(crtc_state);
16974         }
16975
16976         intel_sanitize_plane_mapping(dev_priv);
16977
16978         for_each_intel_encoder(dev, encoder)
16979                 intel_sanitize_encoder(encoder);
16980
16981         for_each_intel_crtc(&dev_priv->drm, crtc) {
16982                 crtc_state = to_intel_crtc_state(crtc->base.state);
16983                 intel_sanitize_crtc(crtc, ctx);
16984                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
16985         }
16986
16987         intel_modeset_update_connector_atomic_state(dev);
16988
16989         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
16990                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
16991
16992                 if (!pll->on || pll->active_mask)
16993                         continue;
16994
16995                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
16996                               pll->info->name);
16997
16998                 pll->info->funcs->disable(dev_priv, pll);
16999                 pll->on = false;
17000         }
17001
17002         if (IS_G4X(dev_priv)) {
17003                 g4x_wm_get_hw_state(dev_priv);
17004                 g4x_wm_sanitize(dev_priv);
17005         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
17006                 vlv_wm_get_hw_state(dev_priv);
17007                 vlv_wm_sanitize(dev_priv);
17008         } else if (INTEL_GEN(dev_priv) >= 9) {
17009                 skl_wm_get_hw_state(dev_priv);
17010         } else if (HAS_PCH_SPLIT(dev_priv)) {
17011                 ilk_wm_get_hw_state(dev_priv);
17012         }
17013
17014         for_each_intel_crtc(dev, crtc) {
17015                 u64 put_domains;
17016
17017                 crtc_state = to_intel_crtc_state(crtc->base.state);
17018                 put_domains = modeset_get_crtc_power_domains(crtc_state);
17019                 if (WARN_ON(put_domains))
17020                         modeset_put_power_domains(dev_priv, put_domains);
17021         }
17022
17023         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
17024
17025         intel_fbc_init_pipe_state(dev_priv);
17026 }
17027
17028 void intel_display_resume(struct drm_device *dev)
17029 {
17030         struct drm_i915_private *dev_priv = to_i915(dev);
17031         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
17032         struct drm_modeset_acquire_ctx ctx;
17033         int ret;
17034
17035         dev_priv->modeset_restore_state = NULL;
17036         if (state)
17037                 state->acquire_ctx = &ctx;
17038
17039         drm_modeset_acquire_init(&ctx, 0);
17040
17041         while (1) {
17042                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
17043                 if (ret != -EDEADLK)
17044                         break;
17045
17046                 drm_modeset_backoff(&ctx);
17047         }
17048
17049         if (!ret)
17050                 ret = __intel_display_resume(dev, state, &ctx);
17051
17052         intel_enable_ipc(dev_priv);
17053         drm_modeset_drop_locks(&ctx);
17054         drm_modeset_acquire_fini(&ctx);
17055
17056         if (ret)
17057                 DRM_ERROR("Restoring old state failed with %i\n", ret);
17058         if (state)
17059                 drm_atomic_state_put(state);
17060 }
17061
17062 static void intel_hpd_poll_fini(struct drm_device *dev)
17063 {
17064         struct intel_connector *connector;
17065         struct drm_connector_list_iter conn_iter;
17066
17067         /* Kill all the work that may have been queued by hpd. */
17068         drm_connector_list_iter_begin(dev, &conn_iter);
17069         for_each_intel_connector_iter(connector, &conn_iter) {
17070                 if (connector->modeset_retry_work.func)
17071                         cancel_work_sync(&connector->modeset_retry_work);
17072                 if (connector->hdcp.shim) {
17073                         cancel_delayed_work_sync(&connector->hdcp.check_work);
17074                         cancel_work_sync(&connector->hdcp.prop_work);
17075                 }
17076         }
17077         drm_connector_list_iter_end(&conn_iter);
17078 }
17079
17080 void intel_modeset_driver_remove(struct drm_device *dev)
17081 {
17082         struct drm_i915_private *dev_priv = to_i915(dev);
17083
17084         flush_workqueue(dev_priv->modeset_wq);
17085
17086         flush_work(&dev_priv->atomic_helper.free_work);
17087         WARN_ON(!llist_empty(&dev_priv->atomic_helper.free_list));
17088
17089         /*
17090          * Interrupts and polling as the first thing to avoid creating havoc.
17091          * Too much stuff here (turning of connectors, ...) would
17092          * experience fancy races otherwise.
17093          */
17094         intel_irq_uninstall(dev_priv);
17095
17096         /*
17097          * Due to the hpd irq storm handling the hotplug work can re-arm the
17098          * poll handlers. Hence disable polling after hpd handling is shut down.
17099          */
17100         intel_hpd_poll_fini(dev);
17101
17102         /* poll work can call into fbdev, hence clean that up afterwards */
17103         intel_fbdev_fini(dev_priv);
17104
17105         intel_unregister_dsm_handler();
17106
17107         intel_fbc_global_disable(dev_priv);
17108
17109         /* flush any delayed tasks or pending work */
17110         flush_scheduled_work();
17111
17112         intel_hdcp_component_fini(dev_priv);
17113
17114         drm_mode_config_cleanup(dev);
17115
17116         intel_overlay_cleanup(dev_priv);
17117
17118         intel_gmbus_teardown(dev_priv);
17119
17120         destroy_workqueue(dev_priv->modeset_wq);
17121
17122         intel_fbc_cleanup_cfb(dev_priv);
17123 }
17124
17125 /*
17126  * set vga decode state - true == enable VGA decode
17127  */
17128 int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state)
17129 {
17130         unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
17131         u16 gmch_ctrl;
17132
17133         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
17134                 DRM_ERROR("failed to read control word\n");
17135                 return -EIO;
17136         }
17137
17138         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
17139                 return 0;
17140
17141         if (state)
17142                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
17143         else
17144                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
17145
17146         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
17147                 DRM_ERROR("failed to write control word\n");
17148                 return -EIO;
17149         }
17150
17151         return 0;
17152 }
17153
17154 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
17155
17156 struct intel_display_error_state {
17157
17158         u32 power_well_driver;
17159
17160         struct intel_cursor_error_state {
17161                 u32 control;
17162                 u32 position;
17163                 u32 base;
17164                 u32 size;
17165         } cursor[I915_MAX_PIPES];
17166
17167         struct intel_pipe_error_state {
17168                 bool power_domain_on;
17169                 u32 source;
17170                 u32 stat;
17171         } pipe[I915_MAX_PIPES];
17172
17173         struct intel_plane_error_state {
17174                 u32 control;
17175                 u32 stride;
17176                 u32 size;
17177                 u32 pos;
17178                 u32 addr;
17179                 u32 surface;
17180                 u32 tile_offset;
17181         } plane[I915_MAX_PIPES];
17182
17183         struct intel_transcoder_error_state {
17184                 bool available;
17185                 bool power_domain_on;
17186                 enum transcoder cpu_transcoder;
17187
17188                 u32 conf;
17189
17190                 u32 htotal;
17191                 u32 hblank;
17192                 u32 hsync;
17193                 u32 vtotal;
17194                 u32 vblank;
17195                 u32 vsync;
17196         } transcoder[5];
17197 };
17198
17199 struct intel_display_error_state *
17200 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
17201 {
17202         struct intel_display_error_state *error;
17203         int transcoders[] = {
17204                 TRANSCODER_A,
17205                 TRANSCODER_B,
17206                 TRANSCODER_C,
17207                 TRANSCODER_D,
17208                 TRANSCODER_EDP,
17209         };
17210         int i;
17211
17212         BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
17213
17214         if (!HAS_DISPLAY(dev_priv))
17215                 return NULL;
17216
17217         error = kzalloc(sizeof(*error), GFP_ATOMIC);
17218         if (error == NULL)
17219                 return NULL;
17220
17221         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
17222                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
17223
17224         for_each_pipe(dev_priv, i) {
17225                 error->pipe[i].power_domain_on =
17226                         __intel_display_power_is_enabled(dev_priv,
17227                                                          POWER_DOMAIN_PIPE(i));
17228                 if (!error->pipe[i].power_domain_on)
17229                         continue;
17230
17231                 error->cursor[i].control = I915_READ(CURCNTR(i));
17232                 error->cursor[i].position = I915_READ(CURPOS(i));
17233                 error->cursor[i].base = I915_READ(CURBASE(i));
17234
17235                 error->plane[i].control = I915_READ(DSPCNTR(i));
17236                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
17237                 if (INTEL_GEN(dev_priv) <= 3) {
17238                         error->plane[i].size = I915_READ(DSPSIZE(i));
17239                         error->plane[i].pos = I915_READ(DSPPOS(i));
17240                 }
17241                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
17242                         error->plane[i].addr = I915_READ(DSPADDR(i));
17243                 if (INTEL_GEN(dev_priv) >= 4) {
17244                         error->plane[i].surface = I915_READ(DSPSURF(i));
17245                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
17246                 }
17247
17248                 error->pipe[i].source = I915_READ(PIPESRC(i));
17249
17250                 if (HAS_GMCH(dev_priv))
17251                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
17252         }
17253
17254         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
17255                 enum transcoder cpu_transcoder = transcoders[i];
17256
17257                 if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder])
17258                         continue;
17259
17260                 error->transcoder[i].available = true;
17261                 error->transcoder[i].power_domain_on =
17262                         __intel_display_power_is_enabled(dev_priv,
17263                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
17264                 if (!error->transcoder[i].power_domain_on)
17265                         continue;
17266
17267                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
17268
17269                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
17270                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
17271                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
17272                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
17273                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
17274                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
17275                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
17276         }
17277
17278         return error;
17279 }
17280
17281 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
17282
17283 void
17284 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
17285                                 struct intel_display_error_state *error)
17286 {
17287         struct drm_i915_private *dev_priv = m->i915;
17288         int i;
17289
17290         if (!error)
17291                 return;
17292
17293         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes);
17294         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
17295                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
17296                            error->power_well_driver);
17297         for_each_pipe(dev_priv, i) {
17298                 err_printf(m, "Pipe [%d]:\n", i);
17299                 err_printf(m, "  Power: %s\n",
17300                            onoff(error->pipe[i].power_domain_on));
17301                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
17302                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
17303
17304                 err_printf(m, "Plane [%d]:\n", i);
17305                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
17306                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
17307                 if (INTEL_GEN(dev_priv) <= 3) {
17308                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
17309                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
17310                 }
17311                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
17312                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
17313                 if (INTEL_GEN(dev_priv) >= 4) {
17314                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
17315                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
17316                 }
17317
17318                 err_printf(m, "Cursor [%d]:\n", i);
17319                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
17320                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
17321                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
17322         }
17323
17324         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
17325                 if (!error->transcoder[i].available)
17326                         continue;
17327
17328                 err_printf(m, "CPU transcoder: %s\n",
17329                            transcoder_name(error->transcoder[i].cpu_transcoder));
17330                 err_printf(m, "  Power: %s\n",
17331                            onoff(error->transcoder[i].power_domain_on));
17332                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
17333                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
17334                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
17335                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
17336                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
17337                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
17338                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
17339         }
17340 }
17341
17342 #endif