drivers/gpu/drm/i915/display/skl_watermark.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2022 Intel Corporation
   4  */
   5
   6 #include <drm/drm_blend.h>
   7
   8 #include "i915_drv.h"
   9 #include "i915_fixed.h"
  10 #include "i915_reg.h"
  11 #include "i9xx_wm.h"
  12 #include "intel_atomic.h"
  13 #include "intel_atomic_plane.h"
  14 #include "intel_bw.h"
  15 #include "intel_crtc.h"
  16 #include "intel_de.h"
  17 #include "intel_display.h"
  18 #include "intel_display_power.h"
  19 #include "intel_display_types.h"
  20 #include "intel_fb.h"
  21 #include "intel_pcode.h"
  22 #include "intel_wm.h"
  23 #include "skl_watermark.h"
  24 #include "skl_watermark_regs.h"
  25
  26 static void skl_sagv_disable(struct drm_i915_private *i915);
  27
  28 /* Stores plane specific WM parameters */
  29 struct skl_wm_params {
  30         bool x_tiled, y_tiled;
  31         bool rc_surface;
  32         bool is_planar;
  33         u32 width;
  34         u8 cpp;
  35         u32 plane_pixel_rate;
  36         u32 y_min_scanlines;
  37         u32 plane_bytes_per_line;
  38         uint_fixed_16_16_t plane_blocks_per_line;
  39         uint_fixed_16_16_t y_tile_minimum;
  40         u32 linetime_us;
  41         u32 dbuf_block_size;
  42 };
  43
  44 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915)
  45 {
  46         u8 enabled_slices = 0;
  47         enum dbuf_slice slice;
  48
  49         for_each_dbuf_slice(i915, slice) {
  50                 if (intel_de_read(i915, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
  51                         enabled_slices |= BIT(slice);
  52         }
  53
  54         return enabled_slices;
  55 }
  56
  57 /*
  58  * FIXME: We still don't have the proper code detect if we need to apply the WA,
  59  * so assume we'll always need it in order to avoid underruns.
  60  */
  61 static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
  62 {
  63         return DISPLAY_VER(i915) == 9;
  64 }
  65
  66 static bool
  67 intel_has_sagv(struct drm_i915_private *i915)
  68 {
  69         return HAS_SAGV(i915) &&
  70                 i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
  71 }
  72
  73 static u32
  74 intel_sagv_block_time(struct drm_i915_private *i915)
  75 {
  76         if (DISPLAY_VER(i915) >= 14) {
  77                 u32 val;
  78
  79                 val = intel_de_read(i915, MTL_LATENCY_SAGV);
  80
  81                 return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
  82         } else if (DISPLAY_VER(i915) >= 12) {
  83                 u32 val = 0;
  84                 int ret;
  85
  86                 ret = snb_pcode_read(&i915->uncore,
  87                                      GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
  88                                      &val, NULL);
  89                 if (ret) {
  90                         drm_dbg_kms(&i915->drm, "Couldn't read SAGV block time!\n");
  91                         return 0;
  92                 }
  93
  94                 return val;
  95         } else if (DISPLAY_VER(i915) == 11) {
  96                 return 10;
  97         } else if (HAS_SAGV(i915)) {
  98                 return 30;
  99         } else {
 100                 return 0;
 101         }
 102 }
 103
 104 static void intel_sagv_init(struct drm_i915_private *i915)
 105 {
 106         if (!HAS_SAGV(i915))
 107                 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 108
 109         /*
 110          * Probe to see if we have working SAGV control.
 111          * For icl+ this was already determined by intel_bw_init_hw().
 112          */
 113         if (DISPLAY_VER(i915) < 11)
 114                 skl_sagv_disable(i915);
 115
 116         drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN);
 117
 118         i915->display.sagv.block_time_us = intel_sagv_block_time(i915);
 119
 120         drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
 121                     str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us);
 122
 123         /* avoid overflow when adding with wm0 latency/etc. */
 124         if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX,
 125                      "Excessive SAGV block time %u, ignoring\n",
 126                      i915->display.sagv.block_time_us))
 127                 i915->display.sagv.block_time_us = 0;
 128
 129         if (!intel_has_sagv(i915))
 130                 i915->display.sagv.block_time_us = 0;
 131 }
 132
 133 /*
 134  * SAGV dynamically adjusts the system agent voltage and clock frequencies
 135  * depending on power and performance requirements. The display engine access
 136  * to system memory is blocked during the adjustment time. Because of the
 137  * blocking time, having this enabled can cause full system hangs and/or pipe
 138  * underruns if we don't meet all of the following requirements:
 139  *
 140  *  - <= 1 pipe enabled
 141  *  - All planes can enable watermarks for latencies >= SAGV engine block time
 142  *  - We're not using an interlaced display configuration
 143  */
 144 static void skl_sagv_enable(struct drm_i915_private *i915)
 145 {
 146         int ret;
 147
 148         if (!intel_has_sagv(i915))
 149                 return;
 150
 151         if (i915->display.sagv.status == I915_SAGV_ENABLED)
 152                 return;
 153
 154         drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
 155         ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
 156                               GEN9_SAGV_ENABLE);
 157
 158         /* We don't need to wait for SAGV when enabling */
 159
 160         /*
 161          * Some skl systems, pre-release machines in particular,
 162          * don't actually have SAGV.
 163          */
 164         if (IS_SKYLAKE(i915) && ret == -ENXIO) {
 165                 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
 166                 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 167                 return;
 168         } else if (ret < 0) {
 169                 drm_err(&i915->drm, "Failed to enable SAGV\n");
 170                 return;
 171         }
 172
 173         i915->display.sagv.status = I915_SAGV_ENABLED;
 174 }
 175
 176 static void skl_sagv_disable(struct drm_i915_private *i915)
 177 {
 178         int ret;
 179
 180         if (!intel_has_sagv(i915))
 181                 return;
 182
 183         if (i915->display.sagv.status == I915_SAGV_DISABLED)
 184                 return;
 185
 186         drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
 187         /* bspec says to keep retrying for at least 1 ms */
 188         ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
 189                                 GEN9_SAGV_DISABLE,
 190                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
 191                                 1);
 192         /*
 193          * Some skl systems, pre-release machines in particular,
 194          * don't actually have SAGV.
 195          */
 196         if (IS_SKYLAKE(i915) && ret == -ENXIO) {
 197                 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
 198                 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 199                 return;
 200         } else if (ret < 0) {
 201                 drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
 202                 return;
 203         }
 204
 205         i915->display.sagv.status = I915_SAGV_DISABLED;
 206 }
 207
 208 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
 209 {
 210         struct drm_i915_private *i915 = to_i915(state->base.dev);
 211         const struct intel_bw_state *new_bw_state =
 212                 intel_atomic_get_new_bw_state(state);
 213
 214         if (!new_bw_state)
 215                 return;
 216
 217         if (!intel_can_enable_sagv(i915, new_bw_state))
 218                 skl_sagv_disable(i915);
 219 }
 220
 221 static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
 222 {
 223         struct drm_i915_private *i915 = to_i915(state->base.dev);
 224         const struct intel_bw_state *new_bw_state =
 225                 intel_atomic_get_new_bw_state(state);
 226
 227         if (!new_bw_state)
 228                 return;
 229
 230         if (intel_can_enable_sagv(i915, new_bw_state))
 231                 skl_sagv_enable(i915);
 232 }
 233
 234 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
 235 {
 236         struct drm_i915_private *i915 = to_i915(state->base.dev);
 237         const struct intel_bw_state *old_bw_state =
 238                 intel_atomic_get_old_bw_state(state);
 239         const struct intel_bw_state *new_bw_state =
 240                 intel_atomic_get_new_bw_state(state);
 241         u16 old_mask, new_mask;
 242
 243         if (!new_bw_state)
 244                 return;
 245
 246         old_mask = old_bw_state->qgv_points_mask;
 247         new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
 248
 249         if (old_mask == new_mask)
 250                 return;
 251
 252         WARN_ON(!new_bw_state->base.changed);
 253
 254         drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
 255                     old_mask, new_mask);
 256
 257         /*
 258          * Restrict required qgv points before updating the configuration.
 259          * According to BSpec we can't mask and unmask qgv points at the same
 260          * time. Also masking should be done before updating the configuration
 261          * and unmasking afterwards.
 262          */
 263         icl_pcode_restrict_qgv_points(i915, new_mask);
 264 }
 265
 266 static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
 267 {
 268         struct drm_i915_private *i915 = to_i915(state->base.dev);
 269         const struct intel_bw_state *old_bw_state =
 270                 intel_atomic_get_old_bw_state(state);
 271         const struct intel_bw_state *new_bw_state =
 272                 intel_atomic_get_new_bw_state(state);
 273         u16 old_mask, new_mask;
 274
 275         if (!new_bw_state)
 276                 return;
 277
 278         old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
 279         new_mask = new_bw_state->qgv_points_mask;
 280
 281         if (old_mask == new_mask)
 282                 return;
 283
 284         WARN_ON(!new_bw_state->base.changed);
 285
 286         drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
 287                     old_mask, new_mask);
 288
 289         /*
 290          * Allow required qgv points after updating the configuration.
 291          * According to BSpec we can't mask and unmask qgv points at the same
 292          * time. Also masking should be done before updating the configuration
 293          * and unmasking afterwards.
 294          */
 295         icl_pcode_restrict_qgv_points(i915, new_mask);
 296 }
 297
 298 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
 299 {
 300         struct drm_i915_private *i915 = to_i915(state->base.dev);
 301
 302         /*
 303          * Just return if we can't control SAGV or don't have it.
 304          * This is different from situation when we have SAGV but just can't
 305          * afford it due to DBuf limitation - in case if SAGV is completely
 306          * disabled in a BIOS, we are not even allowed to send a PCode request,
 307          * as it will throw an error. So have to check it here.
 308          */
 309         if (!intel_has_sagv(i915))
 310                 return;
 311
 312         if (DISPLAY_VER(i915) >= 11)
 313                 icl_sagv_pre_plane_update(state);
 314         else
 315                 skl_sagv_pre_plane_update(state);
 316 }
 317
 318 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
 319 {
 320         struct drm_i915_private *i915 = to_i915(state->base.dev);
 321
 322         /*
 323          * Just return if we can't control SAGV or don't have it.
 324          * This is different from situation when we have SAGV but just can't
 325          * afford it due to DBuf limitation - in case if SAGV is completely
 326          * disabled in a BIOS, we are not even allowed to send a PCode request,
 327          * as it will throw an error. So have to check it here.
 328          */
 329         if (!intel_has_sagv(i915))
 330                 return;
 331
 332         if (DISPLAY_VER(i915) >= 11)
 333                 icl_sagv_post_plane_update(state);
 334         else
 335                 skl_sagv_post_plane_update(state);
 336 }
 337
 338 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 339 {
 340         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 341         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 342         enum plane_id plane_id;
 343         int max_level = INT_MAX;
 344
 345         if (!intel_has_sagv(i915))
 346                 return false;
 347
 348         if (!crtc_state->hw.active)
 349                 return true;
 350
 351         if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
 352                 return false;
 353
 354         for_each_plane_id_on_crtc(crtc, plane_id) {
 355                 const struct skl_plane_wm *wm =
 356                         &crtc_state->wm.skl.optimal.planes[plane_id];
 357                 int level;
 358
 359                 /* Skip this plane if it's not enabled */
 360                 if (!wm->wm[0].enable)
 361                         continue;
 362
 363                 /* Find the highest enabled wm level for this plane */
 364                 for (level = i915->display.wm.num_levels - 1;
 365                      !wm->wm[level].enable; --level)
 366                      { }
 367
 368                 /* Highest common enabled wm level for all planes */
 369                 max_level = min(level, max_level);
 370         }
 371
 372         /* No enabled planes? */
 373         if (max_level == INT_MAX)
 374                 return true;
 375
 376         for_each_plane_id_on_crtc(crtc, plane_id) {
 377                 const struct skl_plane_wm *wm =
 378                         &crtc_state->wm.skl.optimal.planes[plane_id];
 379
 380                 /*
 381                  * All enabled planes must have enabled a common wm level that
 382                  * can tolerate memory latencies higher than sagv_block_time_us
 383                  */
 384                 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
 385                         return false;
 386         }
 387
 388         return true;
 389 }
 390
 391 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 392 {
 393         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 394         enum plane_id plane_id;
 395
 396         if (!crtc_state->hw.active)
 397                 return true;
 398
 399         for_each_plane_id_on_crtc(crtc, plane_id) {
 400                 const struct skl_plane_wm *wm =
 401                         &crtc_state->wm.skl.optimal.planes[plane_id];
 402
 403                 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
 404                         return false;
 405         }
 406
 407         return true;
 408 }
 409
 410 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 411 {
 412         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 413         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 414
 415         if (!i915->params.enable_sagv)
 416                 return false;
 417
 418         if (DISPLAY_VER(i915) >= 12)
 419                 return tgl_crtc_can_enable_sagv(crtc_state);
 420         else
 421                 return skl_crtc_can_enable_sagv(crtc_state);
 422 }
 423
 424 bool intel_can_enable_sagv(struct drm_i915_private *i915,
 425                            const struct intel_bw_state *bw_state)
 426 {
 427         if (DISPLAY_VER(i915) < 11 &&
 428             bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
 429                 return false;
 430
 431         return bw_state->pipe_sagv_reject == 0;
 432 }
 433
 434 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
 435 {
 436         struct drm_i915_private *i915 = to_i915(state->base.dev);
 437         int ret;
 438         struct intel_crtc *crtc;
 439         struct intel_crtc_state *new_crtc_state;
 440         struct intel_bw_state *new_bw_state = NULL;
 441         const struct intel_bw_state *old_bw_state = NULL;
 442         int i;
 443
 444         for_each_new_intel_crtc_in_state(state, crtc,
 445                                          new_crtc_state, i) {
 446                 new_bw_state = intel_atomic_get_bw_state(state);
 447                 if (IS_ERR(new_bw_state))
 448                         return PTR_ERR(new_bw_state);
 449
 450                 old_bw_state = intel_atomic_get_old_bw_state(state);
 451
 452                 if (intel_crtc_can_enable_sagv(new_crtc_state))
 453                         new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
 454                 else
 455                         new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
 456         }
 457
 458         if (!new_bw_state)
 459                 return 0;
 460
 461         new_bw_state->active_pipes =
 462                 intel_calc_active_pipes(state, old_bw_state->active_pipes);
 463
 464         if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
 465                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
 466                 if (ret)
 467                         return ret;
 468         }
 469
 470         if (intel_can_enable_sagv(i915, new_bw_state) !=
 471             intel_can_enable_sagv(i915, old_bw_state)) {
 472                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
 473                 if (ret)
 474                         return ret;
 475         } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
 476                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
 477                 if (ret)
 478                         return ret;
 479         }
 480
 481         for_each_new_intel_crtc_in_state(state, crtc,
 482                                          new_crtc_state, i) {
 483                 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
 484
 485                 /*
 486                  * We store use_sagv_wm in the crtc state rather than relying on
 487                  * that bw state since we have no convenient way to get at the
 488                  * latter from the plane commit hooks (especially in the legacy
 489                  * cursor case)
 490                  */
 491                 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(i915) &&
 492                         DISPLAY_VER(i915) >= 12 &&
 493                         intel_can_enable_sagv(i915, new_bw_state);
 494         }
 495
 496         return 0;
 497 }
 498
 499 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
 500                               u16 start, u16 end)
 501 {
 502         entry->start = start;
 503         entry->end = end;
 504
 505         return end;
 506 }
 507
 508 static int intel_dbuf_slice_size(struct drm_i915_private *i915)
 509 {
 510         return INTEL_INFO(i915)->display.dbuf.size /
 511                 hweight8(INTEL_INFO(i915)->display.dbuf.slice_mask);
 512 }
 513
 514 static void
 515 skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
 516                          struct skl_ddb_entry *ddb)
 517 {
 518         int slice_size = intel_dbuf_slice_size(i915);
 519
 520         if (!slice_mask) {
 521                 ddb->start = 0;
 522                 ddb->end = 0;
 523                 return;
 524         }
 525
 526         ddb->start = (ffs(slice_mask) - 1) * slice_size;
 527         ddb->end = fls(slice_mask) * slice_size;
 528
 529         WARN_ON(ddb->start >= ddb->end);
 530         WARN_ON(ddb->end > INTEL_INFO(i915)->display.dbuf.size);
 531 }
 532
 533 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
 534 {
 535         struct skl_ddb_entry ddb;
 536
 537         if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
 538                 slice_mask = BIT(DBUF_S1);
 539         else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
 540                 slice_mask = BIT(DBUF_S3);
 541
 542         skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
 543
 544         return ddb.start;
 545 }
 546
 547 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
 548                             const struct skl_ddb_entry *entry)
 549 {
 550         int slice_size = intel_dbuf_slice_size(i915);
 551         enum dbuf_slice start_slice, end_slice;
 552         u8 slice_mask = 0;
 553
 554         if (!skl_ddb_entry_size(entry))
 555                 return 0;
 556
 557         start_slice = entry->start / slice_size;
 558         end_slice = (entry->end - 1) / slice_size;
 559
 560         /*
 561          * Per plane DDB entry can in a really worst case be on multiple slices
 562          * but single entry is anyway contigious.
 563          */
 564         while (start_slice <= end_slice) {
 565                 slice_mask |= BIT(start_slice);
 566                 start_slice++;
 567         }
 568
 569         return slice_mask;
 570 }
 571
 572 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
 573 {
 574         const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
 575         int hdisplay, vdisplay;
 576
 577         if (!crtc_state->hw.active)
 578                 return 0;
 579
 580         /*
 581          * Watermark/ddb requirement highly depends upon width of the
 582          * framebuffer, So instead of allocating DDB equally among pipes
 583          * distribute DDB based on resolution/width of the display.
 584          */
 585         drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
 586
 587         return hdisplay;
 588 }
 589
 590 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
 591                                     enum pipe for_pipe,
 592                                     unsigned int *weight_start,
 593                                     unsigned int *weight_end,
 594                                     unsigned int *weight_total)
 595 {
 596         struct drm_i915_private *i915 =
 597                 to_i915(dbuf_state->base.state->base.dev);
 598         enum pipe pipe;
 599
 600         *weight_start = 0;
 601         *weight_end = 0;
 602         *weight_total = 0;
 603
 604         for_each_pipe(i915, pipe) {
 605                 int weight = dbuf_state->weight[pipe];
 606
 607                 /*
 608                  * Do not account pipes using other slice sets
 609                  * luckily as of current BSpec slice sets do not partially
 610                  * intersect(pipes share either same one slice or same slice set
 611                  * i.e no partial intersection), so it is enough to check for
 612                  * equality for now.
 613                  */
 614                 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
 615                         continue;
 616
 617                 *weight_total += weight;
 618                 if (pipe < for_pipe) {
 619                         *weight_start += weight;
 620                         *weight_end += weight;
 621                 } else if (pipe == for_pipe) {
 622                         *weight_end += weight;
 623                 }
 624         }
 625 }
 626
 627 static int
 628 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
 629 {
 630         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 631         unsigned int weight_total, weight_start, weight_end;
 632         const struct intel_dbuf_state *old_dbuf_state =
 633                 intel_atomic_get_old_dbuf_state(state);
 634         struct intel_dbuf_state *new_dbuf_state =
 635                 intel_atomic_get_new_dbuf_state(state);
 636         struct intel_crtc_state *crtc_state;
 637         struct skl_ddb_entry ddb_slices;
 638         enum pipe pipe = crtc->pipe;
 639         unsigned int mbus_offset = 0;
 640         u32 ddb_range_size;
 641         u32 dbuf_slice_mask;
 642         u32 start, end;
 643         int ret;
 644
 645         if (new_dbuf_state->weight[pipe] == 0) {
 646                 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
 647                 goto out;
 648         }
 649
 650         dbuf_slice_mask = new_dbuf_state->slices[pipe];
 651
 652         skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices);
 653         mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask);
 654         ddb_range_size = skl_ddb_entry_size(&ddb_slices);
 655
 656         intel_crtc_dbuf_weights(new_dbuf_state, pipe,
 657                                 &weight_start, &weight_end, &weight_total);
 658
 659         start = ddb_range_size * weight_start / weight_total;
 660         end = ddb_range_size * weight_end / weight_total;
 661
 662         skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
 663                            ddb_slices.start - mbus_offset + start,
 664                            ddb_slices.start - mbus_offset + end);
 665
 666 out:
 667         if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
 668             skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
 669                                 &new_dbuf_state->ddb[pipe]))
 670                 return 0;
 671
 672         ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
 673         if (ret)
 674                 return ret;
 675
 676         crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
 677         if (IS_ERR(crtc_state))
 678                 return PTR_ERR(crtc_state);
 679
 680         /*
 681          * Used for checking overlaps, so we need absolute
 682          * offsets instead of MBUS relative offsets.
 683          */
 684         crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
 685         crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
 686
 687         drm_dbg_kms(&i915->drm,
 688                     "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
 689                     crtc->base.base.id, crtc->base.name,
 690                     old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
 691                     old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
 692                     new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
 693                     old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
 694
 695         return 0;
 696 }
 697
 698 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
 699                                  int width, const struct drm_format_info *format,
 700                                  u64 modifier, unsigned int rotation,
 701                                  u32 plane_pixel_rate, struct skl_wm_params *wp,
 702                                  int color_plane);
 703
 704 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
 705                                  struct intel_plane *plane,
 706                                  int level,
 707                                  unsigned int latency,
 708                                  const struct skl_wm_params *wp,
 709                                  const struct skl_wm_level *result_prev,
 710                                  struct skl_wm_level *result /* out */);
 711
 712 static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level,
 713                                    const struct skl_wm_params *wp)
 714 {
 715         unsigned int latency = i915->display.wm.skl_latency[level];
 716
 717         if (latency == 0)
 718                 return 0;
 719
 720         /*
 721          * WaIncreaseLatencyIPCEnabled: kbl,cfl
 722          * Display WA #1141: kbl,cfl
 723          */
 724         if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
 725             skl_watermark_ipc_enabled(i915))
 726                 latency += 4;
 727
 728         if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled)
 729                 latency += 15;
 730
 731         return latency;
 732 }
 733
 734 static unsigned int
 735 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
 736                       int num_active)
 737 {
 738         struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
 739         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
 740         struct skl_wm_level wm = {};
 741         int ret, min_ddb_alloc = 0;
 742         struct skl_wm_params wp;
 743         int level;
 744
 745         ret = skl_compute_wm_params(crtc_state, 256,
 746                                     drm_format_info(DRM_FORMAT_ARGB8888),
 747                                     DRM_FORMAT_MOD_LINEAR,
 748                                     DRM_MODE_ROTATE_0,
 749                                     crtc_state->pixel_rate, &wp, 0);
 750         drm_WARN_ON(&i915->drm, ret);
 751
 752         for (level = 0; level < i915->display.wm.num_levels; level++) {
 753                 unsigned int latency = skl_wm_latency(i915, level, &wp);
 754
 755                 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
 756                 if (wm.min_ddb_alloc == U16_MAX)
 757                         break;
 758
 759                 min_ddb_alloc = wm.min_ddb_alloc;
 760         }
 761
 762         return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
 763 }
 764
 765 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
 766 {
 767         skl_ddb_entry_init(entry,
 768                            REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
 769                            REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
 770         if (entry->end)
 771                 entry->end++;
 772 }
 773
 774 static void
 775 skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
 776                            const enum pipe pipe,
 777                            const enum plane_id plane_id,
 778                            struct skl_ddb_entry *ddb,
 779                            struct skl_ddb_entry *ddb_y)
 780 {
 781         u32 val;
 782
 783         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
 784         if (plane_id == PLANE_CURSOR) {
 785                 val = intel_de_read(i915, CUR_BUF_CFG(pipe));
 786                 skl_ddb_entry_init_from_hw(ddb, val);
 787                 return;
 788         }
 789
 790         val = intel_de_read(i915, PLANE_BUF_CFG(pipe, plane_id));
 791         skl_ddb_entry_init_from_hw(ddb, val);
 792
 793         if (DISPLAY_VER(i915) >= 11)
 794                 return;
 795
 796         val = intel_de_read(i915, PLANE_NV12_BUF_CFG(pipe, plane_id));
 797         skl_ddb_entry_init_from_hw(ddb_y, val);
 798 }
 799
 800 static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
 801                                       struct skl_ddb_entry *ddb,
 802                                       struct skl_ddb_entry *ddb_y)
 803 {
 804         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 805         enum intel_display_power_domain power_domain;
 806         enum pipe pipe = crtc->pipe;
 807         intel_wakeref_t wakeref;
 808         enum plane_id plane_id;
 809
 810         power_domain = POWER_DOMAIN_PIPE(pipe);
 811         wakeref = intel_display_power_get_if_enabled(i915, power_domain);
 812         if (!wakeref)
 813                 return;
 814
 815         for_each_plane_id_on_crtc(crtc, plane_id)
 816                 skl_ddb_get_hw_plane_state(i915, pipe,
 817                                            plane_id,
 818                                            &ddb[plane_id],
 819                                            &ddb_y[plane_id]);
 820
 821         intel_display_power_put(i915, power_domain, wakeref);
 822 }
 823
 824 struct dbuf_slice_conf_entry {
 825         u8 active_pipes;
 826         u8 dbuf_mask[I915_MAX_PIPES];
 827         bool join_mbus;
 828 };
 829
 830 /*
 831  * Table taken from Bspec 12716
 832  * Pipes do have some preferred DBuf slice affinity,
 833  * plus there are some hardcoded requirements on how
 834  * those should be distributed for multipipe scenarios.
 835  * For more DBuf slices algorithm can get even more messy
 836  * and less readable, so decided to use a table almost
 837  * as is from BSpec itself - that way it is at least easier
 838  * to compare, change and check.
 839  */
 840 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
 841 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
 842 {
 843         {
 844                 .active_pipes = BIT(PIPE_A),
 845                 .dbuf_mask = {
 846                         [PIPE_A] = BIT(DBUF_S1),
 847                 },
 848         },
 849         {
 850                 .active_pipes = BIT(PIPE_B),
 851                 .dbuf_mask = {
 852                         [PIPE_B] = BIT(DBUF_S1),
 853                 },
 854         },
 855         {
 856                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
 857                 .dbuf_mask = {
 858                         [PIPE_A] = BIT(DBUF_S1),
 859                         [PIPE_B] = BIT(DBUF_S2),
 860                 },
 861         },
 862         {
 863                 .active_pipes = BIT(PIPE_C),
 864                 .dbuf_mask = {
 865                         [PIPE_C] = BIT(DBUF_S2),
 866                 },
 867         },
 868         {
 869                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
 870                 .dbuf_mask = {
 871                         [PIPE_A] = BIT(DBUF_S1),
 872                         [PIPE_C] = BIT(DBUF_S2),
 873                 },
 874         },
 875         {
 876                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
 877                 .dbuf_mask = {
 878                         [PIPE_B] = BIT(DBUF_S1),
 879                         [PIPE_C] = BIT(DBUF_S2),
 880                 },
 881         },
 882         {
 883                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
 884                 .dbuf_mask = {
 885                         [PIPE_A] = BIT(DBUF_S1),
 886                         [PIPE_B] = BIT(DBUF_S1),
 887                         [PIPE_C] = BIT(DBUF_S2),
 888                 },
 889         },
 890         {}
 891 };
 892
 893 /*
 894  * Table taken from Bspec 49255
 895  * Pipes do have some preferred DBuf slice affinity,
 896  * plus there are some hardcoded requirements on how
 897  * those should be distributed for multipipe scenarios.
 898  * For more DBuf slices algorithm can get even more messy
 899  * and less readable, so decided to use a table almost
 900  * as is from BSpec itself - that way it is at least easier
 901  * to compare, change and check.
 902  */
 903 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
 904 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
 905 {
 906         {
 907                 .active_pipes = BIT(PIPE_A),
 908                 .dbuf_mask = {
 909                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
 910                 },
 911         },
 912         {
 913                 .active_pipes = BIT(PIPE_B),
 914                 .dbuf_mask = {
 915                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
 916                 },
 917         },
 918         {
 919                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
 920                 .dbuf_mask = {
 921                         [PIPE_A] = BIT(DBUF_S2),
 922                         [PIPE_B] = BIT(DBUF_S1),
 923                 },
 924         },
 925         {
 926                 .active_pipes = BIT(PIPE_C),
 927                 .dbuf_mask = {
 928                         [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
 929                 },
 930         },
 931         {
 932                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
 933                 .dbuf_mask = {
 934                         [PIPE_A] = BIT(DBUF_S1),
 935                         [PIPE_C] = BIT(DBUF_S2),
 936                 },
 937         },
 938         {
 939                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
 940                 .dbuf_mask = {
 941                         [PIPE_B] = BIT(DBUF_S1),
 942                         [PIPE_C] = BIT(DBUF_S2),
 943                 },
 944         },
 945         {
 946                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
 947                 .dbuf_mask = {
 948                         [PIPE_A] = BIT(DBUF_S1),
 949                         [PIPE_B] = BIT(DBUF_S1),
 950                         [PIPE_C] = BIT(DBUF_S2),
 951                 },
 952         },
 953         {
 954                 .active_pipes = BIT(PIPE_D),
 955                 .dbuf_mask = {
 956                         [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
 957                 },
 958         },
 959         {
 960                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
 961                 .dbuf_mask = {
 962                         [PIPE_A] = BIT(DBUF_S1),
 963                         [PIPE_D] = BIT(DBUF_S2),
 964                 },
 965         },
 966         {
 967                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
 968                 .dbuf_mask = {
 969                         [PIPE_B] = BIT(DBUF_S1),
 970                         [PIPE_D] = BIT(DBUF_S2),
 971                 },
 972         },
 973         {
 974                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
 975                 .dbuf_mask = {
 976                         [PIPE_A] = BIT(DBUF_S1),
 977                         [PIPE_B] = BIT(DBUF_S1),
 978                         [PIPE_D] = BIT(DBUF_S2),
 979                 },
 980         },
 981         {
 982                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
 983                 .dbuf_mask = {
 984                         [PIPE_C] = BIT(DBUF_S1),
 985                         [PIPE_D] = BIT(DBUF_S2),
 986                 },
 987         },
 988         {
 989                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
 990                 .dbuf_mask = {
 991                         [PIPE_A] = BIT(DBUF_S1),
 992                         [PIPE_C] = BIT(DBUF_S2),
 993                         [PIPE_D] = BIT(DBUF_S2),
 994                 },
 995         },
 996         {
 997                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
 998                 .dbuf_mask = {
 999                         [PIPE_B] = BIT(DBUF_S1),
1000                         [PIPE_C] = BIT(DBUF_S2),
1001                         [PIPE_D] = BIT(DBUF_S2),
1002                 },
1003         },
1004         {
1005                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1006                 .dbuf_mask = {
1007                         [PIPE_A] = BIT(DBUF_S1),
1008                         [PIPE_B] = BIT(DBUF_S1),
1009                         [PIPE_C] = BIT(DBUF_S2),
1010                         [PIPE_D] = BIT(DBUF_S2),
1011                 },
1012         },
1013         {}
1014 };
1015
1016 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
1017         {
1018                 .active_pipes = BIT(PIPE_A),
1019                 .dbuf_mask = {
1020                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1021                 },
1022         },
1023         {
1024                 .active_pipes = BIT(PIPE_B),
1025                 .dbuf_mask = {
1026                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1027                 },
1028         },
1029         {
1030                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1031                 .dbuf_mask = {
1032                         [PIPE_A] = BIT(DBUF_S1),
1033                         [PIPE_B] = BIT(DBUF_S2),
1034                 },
1035         },
1036         {
1037                 .active_pipes = BIT(PIPE_C),
1038                 .dbuf_mask = {
1039                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1040                 },
1041         },
1042         {
1043                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1044                 .dbuf_mask = {
1045                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1046                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1047                 },
1048         },
1049         {
1050                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1051                 .dbuf_mask = {
1052                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1053                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1054                 },
1055         },
1056         {
1057                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1058                 .dbuf_mask = {
1059                         [PIPE_A] = BIT(DBUF_S1),
1060                         [PIPE_B] = BIT(DBUF_S2),
1061                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1062                 },
1063         },
1064         {
1065                 .active_pipes = BIT(PIPE_D),
1066                 .dbuf_mask = {
1067                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1068                 },
1069         },
1070         {
1071                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1072                 .dbuf_mask = {
1073                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1074                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1075                 },
1076         },
1077         {
1078                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1079                 .dbuf_mask = {
1080                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1081                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1082                 },
1083         },
1084         {
1085                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1086                 .dbuf_mask = {
1087                         [PIPE_A] = BIT(DBUF_S1),
1088                         [PIPE_B] = BIT(DBUF_S2),
1089                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1090                 },
1091         },
1092         {
1093                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1094                 .dbuf_mask = {
1095                         [PIPE_C] = BIT(DBUF_S3),
1096                         [PIPE_D] = BIT(DBUF_S4),
1097                 },
1098         },
1099         {
1100                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1101                 .dbuf_mask = {
1102                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1103                         [PIPE_C] = BIT(DBUF_S3),
1104                         [PIPE_D] = BIT(DBUF_S4),
1105                 },
1106         },
1107         {
1108                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1109                 .dbuf_mask = {
1110                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1111                         [PIPE_C] = BIT(DBUF_S3),
1112                         [PIPE_D] = BIT(DBUF_S4),
1113                 },
1114         },
1115         {
1116                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1117                 .dbuf_mask = {
1118                         [PIPE_A] = BIT(DBUF_S1),
1119                         [PIPE_B] = BIT(DBUF_S2),
1120                         [PIPE_C] = BIT(DBUF_S3),
1121                         [PIPE_D] = BIT(DBUF_S4),
1122                 },
1123         },
1124         {}
1125 };
1126
1127 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1128         /*
1129          * Keep the join_mbus cases first so check_mbus_joined()
1130          * will prefer them over the !join_mbus cases.
1131          */
1132         {
1133                 .active_pipes = BIT(PIPE_A),
1134                 .dbuf_mask = {
1135                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1136                 },
1137                 .join_mbus = true,
1138         },
1139         {
1140                 .active_pipes = BIT(PIPE_B),
1141                 .dbuf_mask = {
1142                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1143                 },
1144                 .join_mbus = true,
1145         },
1146         {
1147                 .active_pipes = BIT(PIPE_A),
1148                 .dbuf_mask = {
1149                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1150                 },
1151                 .join_mbus = false,
1152         },
1153         {
1154                 .active_pipes = BIT(PIPE_B),
1155                 .dbuf_mask = {
1156                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1157                 },
1158                 .join_mbus = false,
1159         },
1160         {
1161                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1162                 .dbuf_mask = {
1163                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1164                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1165                 },
1166         },
1167         {
1168                 .active_pipes = BIT(PIPE_C),
1169                 .dbuf_mask = {
1170                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1171                 },
1172         },
1173         {
1174                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1175                 .dbuf_mask = {
1176                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1177                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1178                 },
1179         },
1180         {
1181                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1182                 .dbuf_mask = {
1183                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1184                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1185                 },
1186         },
1187         {
1188                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1189                 .dbuf_mask = {
1190                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1191                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1192                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1193                 },
1194         },
1195         {
1196                 .active_pipes = BIT(PIPE_D),
1197                 .dbuf_mask = {
1198                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1199                 },
1200         },
1201         {
1202                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1203                 .dbuf_mask = {
1204                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1205                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1206                 },
1207         },
1208         {
1209                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1210                 .dbuf_mask = {
1211                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1212                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1213                 },
1214         },
1215         {
1216                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1217                 .dbuf_mask = {
1218                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1219                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1220                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1221                 },
1222         },
1223         {
1224                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1225                 .dbuf_mask = {
1226                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1227                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1228                 },
1229         },
1230         {
1231                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1232                 .dbuf_mask = {
1233                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1234                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1235                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1236                 },
1237         },
1238         {
1239                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1240                 .dbuf_mask = {
1241                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1242                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1243                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1244                 },
1245         },
1246         {
1247                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1248                 .dbuf_mask = {
1249                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1250                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1251                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1252                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1253                 },
1254         },
1255         {}
1256
1257 };
1258
1259 static bool check_mbus_joined(u8 active_pipes,
1260                               const struct dbuf_slice_conf_entry *dbuf_slices)
1261 {
1262         int i;
1263
1264         for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1265                 if (dbuf_slices[i].active_pipes == active_pipes)
1266                         return dbuf_slices[i].join_mbus;
1267         }
1268         return false;
1269 }
1270
1271 static bool adlp_check_mbus_joined(u8 active_pipes)
1272 {
1273         return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
1274 }
1275
1276 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1277                               const struct dbuf_slice_conf_entry *dbuf_slices)
1278 {
1279         int i;
1280
1281         for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1282                 if (dbuf_slices[i].active_pipes == active_pipes &&
1283                     dbuf_slices[i].join_mbus == join_mbus)
1284                         return dbuf_slices[i].dbuf_mask[pipe];
1285         }
1286         return 0;
1287 }
1288
1289 /*
1290  * This function finds an entry with same enabled pipe configuration and
1291  * returns correspondent DBuf slice mask as stated in BSpec for particular
1292  * platform.
1293  */
1294 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1295 {
1296         /*
1297          * FIXME: For ICL this is still a bit unclear as prev BSpec revision
1298          * required calculating "pipe ratio" in order to determine
1299          * if one or two slices can be used for single pipe configurations
1300          * as additional constraint to the existing table.
1301          * However based on recent info, it should be not "pipe ratio"
1302          * but rather ratio between pixel_rate and cdclk with additional
1303          * constants, so for now we are using only table until this is
1304          * clarified. Also this is the reason why crtc_state param is
1305          * still here - we will need it once those additional constraints
1306          * pop up.
1307          */
1308         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1309                                    icl_allowed_dbufs);
1310 }
1311
1312 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1313 {
1314         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1315                                    tgl_allowed_dbufs);
1316 }
1317
1318 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1319 {
1320         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1321                                    adlp_allowed_dbufs);
1322 }
1323
1324 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1325 {
1326         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1327                                    dg2_allowed_dbufs);
1328 }
1329
1330 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1331 {
1332         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1333         enum pipe pipe = crtc->pipe;
1334
1335         if (IS_DG2(i915))
1336                 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1337         else if (DISPLAY_VER(i915) >= 13)
1338                 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1339         else if (DISPLAY_VER(i915) == 12)
1340                 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1341         else if (DISPLAY_VER(i915) == 11)
1342                 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1343         /*
1344          * For anything else just return one slice yet.
1345          * Should be extended for other platforms.
1346          */
1347         return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1348 }
1349
1350 static bool
1351 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1352                      struct intel_plane *plane)
1353 {
1354         struct drm_i915_private *i915 = to_i915(plane->base.dev);
1355
1356         return DISPLAY_VER(i915) >= 13 &&
1357                crtc_state->uapi.async_flip &&
1358                plane->async_flip;
1359 }
1360
1361 static u64
1362 skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1363 {
1364         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1365         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1366         enum plane_id plane_id;
1367         u64 data_rate = 0;
1368
1369         for_each_plane_id_on_crtc(crtc, plane_id) {
1370                 if (plane_id == PLANE_CURSOR)
1371                         continue;
1372
1373                 data_rate += crtc_state->rel_data_rate[plane_id];
1374
1375                 if (DISPLAY_VER(i915) < 11)
1376                         data_rate += crtc_state->rel_data_rate_y[plane_id];
1377         }
1378
1379         return data_rate;
1380 }
1381
1382 static const struct skl_wm_level *
1383 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1384                    enum plane_id plane_id,
1385                    int level)
1386 {
1387         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1388
1389         if (level == 0 && pipe_wm->use_sagv_wm)
1390                 return &wm->sagv.wm0;
1391
1392         return &wm->wm[level];
1393 }
1394
1395 static const struct skl_wm_level *
1396 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1397                    enum plane_id plane_id)
1398 {
1399         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1400
1401         if (pipe_wm->use_sagv_wm)
1402                 return &wm->sagv.trans_wm;
1403
1404         return &wm->trans_wm;
1405 }
1406
1407 /*
1408  * We only disable the watermarks for each plane if
1409  * they exceed the ddb allocation of said plane. This
1410  * is done so that we don't end up touching cursor
1411  * watermarks needlessly when some other plane reduces
1412  * our max possible watermark level.
1413  *
1414  * Bspec has this to say about the PLANE_WM enable bit:
1415  * "All the watermarks at this level for all enabled
1416  *  planes must be enabled before the level will be used."
1417  * So this is actually safe to do.
1418  */
1419 static void
1420 skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1421 {
1422         if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
1423                 memset(wm, 0, sizeof(*wm));
1424 }
1425
1426 static void
1427 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1428                         const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1429 {
1430         if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
1431             uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1432                 memset(wm, 0, sizeof(*wm));
1433                 memset(uv_wm, 0, sizeof(*uv_wm));
1434         }
1435 }
1436
1437 static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level,
1438                                 const struct skl_plane_wm *wm)
1439 {
1440         /*
1441          * Wa_1408961008:icl, ehl
1442          * Wa_14012656716:tgl, adl
1443          * Wa_14017887344:icl
1444          * Wa_14017868169:adl, tgl
1445          * Due to some power saving optimizations, different subsystems
1446          * like PSR, might still use even disabled wm level registers,
1447          * for "reference", so lets keep at least the values sane.
1448          * Considering amount of WA requiring us to do similar things, was
1449          * decided to simply do it for all of the platforms, as those wm
1450          * levels are disabled, this isn't going to do harm anyway.
1451          */
1452         return level > 0 && !wm->wm[level].enable;
1453 }
1454
1455 struct skl_plane_ddb_iter {
1456         u64 data_rate;
1457         u16 start, size;
1458 };
1459
1460 static void
1461 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1462                        struct skl_ddb_entry *ddb,
1463                        const struct skl_wm_level *wm,
1464                        u64 data_rate)
1465 {
1466         u16 size, extra = 0;
1467
1468         if (data_rate) {
1469                 extra = min_t(u16, iter->size,
1470                               DIV64_U64_ROUND_UP(iter->size * data_rate,
1471                                                  iter->data_rate));
1472                 iter->size -= extra;
1473                 iter->data_rate -= data_rate;
1474         }
1475
1476         /*
1477          * Keep ddb entry of all disabled planes explicitly zeroed
1478          * to avoid skl_ddb_add_affected_planes() adding them to
1479          * the state when other planes change their allocations.
1480          */
1481         size = wm->min_ddb_alloc + extra;
1482         if (size)
1483                 iter->start = skl_ddb_entry_init(ddb, iter->start,
1484                                                  iter->start + size);
1485 }
1486
1487 static int
1488 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1489                             struct intel_crtc *crtc)
1490 {
1491         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1492         struct intel_crtc_state *crtc_state =
1493                 intel_atomic_get_new_crtc_state(state, crtc);
1494         const struct intel_dbuf_state *dbuf_state =
1495                 intel_atomic_get_new_dbuf_state(state);
1496         const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1497         int num_active = hweight8(dbuf_state->active_pipes);
1498         struct skl_plane_ddb_iter iter;
1499         enum plane_id plane_id;
1500         u16 cursor_size;
1501         u32 blocks;
1502         int level;
1503
1504         /* Clear the partitioning for disabled planes. */
1505         memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1506         memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1507
1508         if (!crtc_state->hw.active)
1509                 return 0;
1510
1511         iter.start = alloc->start;
1512         iter.size = skl_ddb_entry_size(alloc);
1513         if (iter.size == 0)
1514                 return 0;
1515
1516         /* Allocate fixed number of blocks for cursor. */
1517         cursor_size = skl_cursor_allocation(crtc_state, num_active);
1518         iter.size -= cursor_size;
1519         skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1520                            alloc->end - cursor_size, alloc->end);
1521
1522         iter.data_rate = skl_total_relative_data_rate(crtc_state);
1523
1524         /*
1525          * Find the highest watermark level for which we can satisfy the block
1526          * requirement of active planes.
1527          */
1528         for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
1529                 blocks = 0;
1530                 for_each_plane_id_on_crtc(crtc, plane_id) {
1531                         const struct skl_plane_wm *wm =
1532                                 &crtc_state->wm.skl.optimal.planes[plane_id];
1533
1534                         if (plane_id == PLANE_CURSOR) {
1535                                 const struct skl_ddb_entry *ddb =
1536                                         &crtc_state->wm.skl.plane_ddb[plane_id];
1537
1538                                 if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1539                                         drm_WARN_ON(&i915->drm,
1540                                                     wm->wm[level].min_ddb_alloc != U16_MAX);
1541                                         blocks = U32_MAX;
1542                                         break;
1543                                 }
1544                                 continue;
1545                         }
1546
1547                         blocks += wm->wm[level].min_ddb_alloc;
1548                         blocks += wm->uv_wm[level].min_ddb_alloc;
1549                 }
1550
1551                 if (blocks <= iter.size) {
1552                         iter.size -= blocks;
1553                         break;
1554                 }
1555         }
1556
1557         if (level < 0) {
1558                 drm_dbg_kms(&i915->drm,
1559                             "Requested display configuration exceeds system DDB limitations");
1560                 drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
1561                             blocks, iter.size);
1562                 return -EINVAL;
1563         }
1564
1565         /* avoid the WARN later when we don't allocate any extra DDB */
1566         if (iter.data_rate == 0)
1567                 iter.size = 0;
1568
1569         /*
1570          * Grant each plane the blocks it requires at the highest achievable
1571          * watermark level, plus an extra share of the leftover blocks
1572          * proportional to its relative data rate.
1573          */
1574         for_each_plane_id_on_crtc(crtc, plane_id) {
1575                 struct skl_ddb_entry *ddb =
1576                         &crtc_state->wm.skl.plane_ddb[plane_id];
1577                 struct skl_ddb_entry *ddb_y =
1578                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
1579                 const struct skl_plane_wm *wm =
1580                         &crtc_state->wm.skl.optimal.planes[plane_id];
1581
1582                 if (plane_id == PLANE_CURSOR)
1583                         continue;
1584
1585                 if (DISPLAY_VER(i915) < 11 &&
1586                     crtc_state->nv12_planes & BIT(plane_id)) {
1587                         skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
1588                                                crtc_state->rel_data_rate_y[plane_id]);
1589                         skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
1590                                                crtc_state->rel_data_rate[plane_id]);
1591                 } else {
1592                         skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
1593                                                crtc_state->rel_data_rate[plane_id]);
1594                 }
1595         }
1596         drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
1597
1598         /*
1599          * When we calculated watermark values we didn't know how high
1600          * of a level we'd actually be able to hit, so we just marked
1601          * all levels as "enabled."  Go back now and disable the ones
1602          * that aren't actually possible.
1603          */
1604         for (level++; level < i915->display.wm.num_levels; level++) {
1605                 for_each_plane_id_on_crtc(crtc, plane_id) {
1606                         const struct skl_ddb_entry *ddb =
1607                                 &crtc_state->wm.skl.plane_ddb[plane_id];
1608                         const struct skl_ddb_entry *ddb_y =
1609                                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1610                         struct skl_plane_wm *wm =
1611                                 &crtc_state->wm.skl.optimal.planes[plane_id];
1612
1613                         if (DISPLAY_VER(i915) < 11 &&
1614                             crtc_state->nv12_planes & BIT(plane_id))
1615                                 skl_check_nv12_wm_level(&wm->wm[level],
1616                                                         &wm->uv_wm[level],
1617                                                         ddb_y, ddb);
1618                         else
1619                                 skl_check_wm_level(&wm->wm[level], ddb);
1620
1621                         if (skl_need_wm_copy_wa(i915, level, wm)) {
1622                                 wm->wm[level].blocks = wm->wm[level - 1].blocks;
1623                                 wm->wm[level].lines = wm->wm[level - 1].lines;
1624                                 wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1625                         }
1626                 }
1627         }
1628
1629         /*
1630          * Go back and disable the transition and SAGV watermarks
1631          * if it turns out we don't have enough DDB blocks for them.
1632          */
1633         for_each_plane_id_on_crtc(crtc, plane_id) {
1634                 const struct skl_ddb_entry *ddb =
1635                         &crtc_state->wm.skl.plane_ddb[plane_id];
1636                 const struct skl_ddb_entry *ddb_y =
1637                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
1638                 struct skl_plane_wm *wm =
1639                         &crtc_state->wm.skl.optimal.planes[plane_id];
1640
1641                 if (DISPLAY_VER(i915) < 11 &&
1642                     crtc_state->nv12_planes & BIT(plane_id)) {
1643                         skl_check_wm_level(&wm->trans_wm, ddb_y);
1644                 } else {
1645                         WARN_ON(skl_ddb_entry_size(ddb_y));
1646
1647                         skl_check_wm_level(&wm->trans_wm, ddb);
1648                 }
1649
1650                 skl_check_wm_level(&wm->sagv.wm0, ddb);
1651                 skl_check_wm_level(&wm->sagv.trans_wm, ddb);
1652         }
1653
1654         return 0;
1655 }
1656
1657 /*
1658  * The max latency should be 257 (max the punit can code is 255 and we add 2us
1659  * for the read latency) and cpp should always be <= 8, so that
1660  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
1661  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1662  */
1663 static uint_fixed_16_16_t
1664 skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
1665                u8 cpp, u32 latency, u32 dbuf_block_size)
1666 {
1667         u32 wm_intermediate_val;
1668         uint_fixed_16_16_t ret;
1669
1670         if (latency == 0)
1671                 return FP_16_16_MAX;
1672
1673         wm_intermediate_val = latency * pixel_rate * cpp;
1674         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
1675
1676         if (DISPLAY_VER(i915) >= 10)
1677                 ret = add_fixed16_u32(ret, 1);
1678
1679         return ret;
1680 }
1681
1682 static uint_fixed_16_16_t
1683 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1684                uint_fixed_16_16_t plane_blocks_per_line)
1685 {
1686         u32 wm_intermediate_val;
1687         uint_fixed_16_16_t ret;
1688
1689         if (latency == 0)
1690                 return FP_16_16_MAX;
1691
1692         wm_intermediate_val = latency * pixel_rate;
1693         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1694                                            pipe_htotal * 1000);
1695         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
1696         return ret;
1697 }
1698
1699 static uint_fixed_16_16_t
1700 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1701 {
1702         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1703         u32 pixel_rate;
1704         u32 crtc_htotal;
1705         uint_fixed_16_16_t linetime_us;
1706
1707         if (!crtc_state->hw.active)
1708                 return u32_to_fixed16(0);
1709
1710         pixel_rate = crtc_state->pixel_rate;
1711
1712         if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
1713                 return u32_to_fixed16(0);
1714
1715         crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1716         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
1717
1718         return linetime_us;
1719 }
1720
1721 static int
1722 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1723                       int width, const struct drm_format_info *format,
1724                       u64 modifier, unsigned int rotation,
1725                       u32 plane_pixel_rate, struct skl_wm_params *wp,
1726                       int color_plane)
1727 {
1728         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1729         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1730         u32 interm_pbpl;
1731
1732         /* only planar format has two planes */
1733         if (color_plane == 1 &&
1734             !intel_format_info_is_yuv_semiplanar(format, modifier)) {
1735                 drm_dbg_kms(&i915->drm,
1736                             "Non planar format have single plane\n");
1737                 return -EINVAL;
1738         }
1739
1740         wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1741         wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1742                 intel_fb_is_tiled_modifier(modifier);
1743         wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1744         wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
1745
1746         wp->width = width;
1747         if (color_plane == 1 && wp->is_planar)
1748                 wp->width /= 2;
1749
1750         wp->cpp = format->cpp[color_plane];
1751         wp->plane_pixel_rate = plane_pixel_rate;
1752
1753         if (DISPLAY_VER(i915) >= 11 &&
1754             modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
1755                 wp->dbuf_block_size = 256;
1756         else
1757                 wp->dbuf_block_size = 512;
1758
1759         if (drm_rotation_90_or_270(rotation)) {
1760                 switch (wp->cpp) {
1761                 case 1:
1762                         wp->y_min_scanlines = 16;
1763                         break;
1764                 case 2:
1765                         wp->y_min_scanlines = 8;
1766                         break;
1767                 case 4:
1768                         wp->y_min_scanlines = 4;
1769                         break;
1770                 default:
1771                         MISSING_CASE(wp->cpp);
1772                         return -EINVAL;
1773                 }
1774         } else {
1775                 wp->y_min_scanlines = 4;
1776         }
1777
1778         if (skl_needs_memory_bw_wa(i915))
1779                 wp->y_min_scanlines *= 2;
1780
1781         wp->plane_bytes_per_line = wp->width * wp->cpp;
1782         if (wp->y_tiled) {
1783                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1784                                            wp->y_min_scanlines,
1785                                            wp->dbuf_block_size);
1786
1787                 if (DISPLAY_VER(i915) >= 10)
1788                         interm_pbpl++;
1789
1790                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
1791                                                         wp->y_min_scanlines);
1792         } else {
1793                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1794                                            wp->dbuf_block_size);
1795
1796                 if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
1797                         interm_pbpl++;
1798
1799                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
1800         }
1801
1802         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
1803                                              wp->plane_blocks_per_line);
1804
1805         wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state));
1806
1807         return 0;
1808 }
1809
1810 static int
1811 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1812                             const struct intel_plane_state *plane_state,
1813                             struct skl_wm_params *wp, int color_plane)
1814 {
1815         const struct drm_framebuffer *fb = plane_state->hw.fb;
1816         int width;
1817
1818         /*
1819          * Src coordinates are already rotated by 270 degrees for
1820          * the 90/270 degree plane rotation cases (to match the
1821          * GTT mapping), hence no need to account for rotation here.
1822          */
1823         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1824
1825         return skl_compute_wm_params(crtc_state, width,
1826                                      fb->format, fb->modifier,
1827                                      plane_state->hw.rotation,
1828                                      intel_plane_pixel_rate(crtc_state, plane_state),
1829                                      wp, color_plane);
1830 }
1831
1832 static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
1833 {
1834         if (DISPLAY_VER(i915) >= 10)
1835                 return true;
1836
1837         /* The number of lines are ignored for the level 0 watermark. */
1838         return level > 0;
1839 }
1840
1841 static int skl_wm_max_lines(struct drm_i915_private *i915)
1842 {
1843         if (DISPLAY_VER(i915) >= 13)
1844                 return 255;
1845         else
1846                 return 31;
1847 }
1848
1849 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1850                                  struct intel_plane *plane,
1851                                  int level,
1852                                  unsigned int latency,
1853                                  const struct skl_wm_params *wp,
1854                                  const struct skl_wm_level *result_prev,
1855                                  struct skl_wm_level *result /* out */)
1856 {
1857         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1858         uint_fixed_16_16_t method1, method2;
1859         uint_fixed_16_16_t selected_result;
1860         u32 blocks, lines, min_ddb_alloc = 0;
1861
1862         if (latency == 0 ||
1863             (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1864                 /* reject it */
1865                 result->min_ddb_alloc = U16_MAX;
1866                 return;
1867         }
1868
1869         method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
1870                                  wp->cpp, latency, wp->dbuf_block_size);
1871         method2 = skl_wm_method2(wp->plane_pixel_rate,
1872                                  crtc_state->hw.pipe_mode.crtc_htotal,
1873                                  latency,
1874                                  wp->plane_blocks_per_line);
1875
1876         if (wp->y_tiled) {
1877                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
1878         } else {
1879                 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1880                      wp->dbuf_block_size < 1) &&
1881                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1882                         selected_result = method2;
1883                 } else if (latency >= wp->linetime_us) {
1884                         if (DISPLAY_VER(i915) == 9)
1885                                 selected_result = min_fixed16(method1, method2);
1886                         else
1887                                 selected_result = method2;
1888                 } else {
1889                         selected_result = method1;
1890                 }
1891         }
1892
1893         blocks = fixed16_to_u32_round_up(selected_result) + 1;
1894         /*
1895          * Lets have blocks at minimum equivalent to plane_blocks_per_line
1896          * as there will be at minimum one line for lines configuration. This
1897          * is a work around for FIFO underruns observed with resolutions like
1898          * 4k 60 Hz in single channel DRAM configurations.
1899          *
1900          * As per the Bspec 49325, if the ddb allocation can hold at least
1901          * one plane_blocks_per_line, we should have selected method2 in
1902          * the above logic. Assuming that modern versions have enough dbuf
1903          * and method2 guarantees blocks equivalent to at least 1 line,
1904          * select the blocks as plane_blocks_per_line.
1905          *
1906          * TODO: Revisit the logic when we have better understanding on DRAM
1907          * channels' impact on the level 0 memory latency and the relevant
1908          * wm calculations.
1909          */
1910         if (skl_wm_has_lines(i915, level))
1911                 blocks = max(blocks,
1912                              fixed16_to_u32_round_up(wp->plane_blocks_per_line));
1913         lines = div_round_up_fixed16(selected_result,
1914                                      wp->plane_blocks_per_line);
1915
1916         if (DISPLAY_VER(i915) == 9) {
1917                 /* Display WA #1125: skl,bxt,kbl */
1918                 if (level == 0 && wp->rc_surface)
1919                         blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1920
1921                 /* Display WA #1126: skl,bxt,kbl */
1922                 if (level >= 1 && level <= 7) {
1923                         if (wp->y_tiled) {
1924                                 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1925                                 lines += wp->y_min_scanlines;
1926                         } else {
1927                                 blocks++;
1928                         }
1929
1930                         /*
1931                          * Make sure result blocks for higher latency levels are
1932                          * at least as high as level below the current level.
1933                          * Assumption in DDB algorithm optimization for special
1934                          * cases. Also covers Display WA #1125 for RC.
1935                          */
1936                         if (result_prev->blocks > blocks)
1937                                 blocks = result_prev->blocks;
1938                 }
1939         }
1940
1941         if (DISPLAY_VER(i915) >= 11) {
1942                 if (wp->y_tiled) {
1943                         int extra_lines;
1944
1945                         if (lines % wp->y_min_scanlines == 0)
1946                                 extra_lines = wp->y_min_scanlines;
1947                         else
1948                                 extra_lines = wp->y_min_scanlines * 2 -
1949                                         lines % wp->y_min_scanlines;
1950
1951                         min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
1952                                                                  wp->plane_blocks_per_line);
1953                 } else {
1954                         min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
1955                 }
1956         }
1957
1958         if (!skl_wm_has_lines(i915, level))
1959                 lines = 0;
1960
1961         if (lines > skl_wm_max_lines(i915)) {
1962                 /* reject it */
1963                 result->min_ddb_alloc = U16_MAX;
1964                 return;
1965         }
1966
1967         /*
1968          * If lines is valid, assume we can use this watermark level
1969          * for now.  We'll come back and disable it after we calculate the
1970          * DDB allocation if it turns out we don't actually have enough
1971          * blocks to satisfy it.
1972          */
1973         result->blocks = blocks;
1974         result->lines = lines;
1975         /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
1976         result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
1977         result->enable = true;
1978
1979         if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
1980                 result->can_sagv = latency >= i915->display.sagv.block_time_us;
1981 }
1982
1983 static void
1984 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
1985                       struct intel_plane *plane,
1986                       const struct skl_wm_params *wm_params,
1987                       struct skl_wm_level *levels)
1988 {
1989         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1990         struct skl_wm_level *result_prev = &levels[0];
1991         int level;
1992
1993         for (level = 0; level < i915->display.wm.num_levels; level++) {
1994                 struct skl_wm_level *result = &levels[level];
1995                 unsigned int latency = skl_wm_latency(i915, level, wm_params);
1996
1997                 skl_compute_plane_wm(crtc_state, plane, level, latency,
1998                                      wm_params, result_prev, result);
1999
2000                 result_prev = result;
2001         }
2002 }
2003
2004 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
2005                                 struct intel_plane *plane,
2006                                 const struct skl_wm_params *wm_params,
2007                                 struct skl_plane_wm *plane_wm)
2008 {
2009         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2010         struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
2011         struct skl_wm_level *levels = plane_wm->wm;
2012         unsigned int latency = 0;
2013
2014         if (i915->display.sagv.block_time_us)
2015                 latency = i915->display.sagv.block_time_us +
2016                         skl_wm_latency(i915, 0, wm_params);
2017
2018         skl_compute_plane_wm(crtc_state, plane, 0, latency,
2019                              wm_params, &levels[0],
2020                              sagv_wm);
2021 }
2022
2023 static void skl_compute_transition_wm(struct drm_i915_private *i915,
2024                                       struct skl_wm_level *trans_wm,
2025                                       const struct skl_wm_level *wm0,
2026                                       const struct skl_wm_params *wp)
2027 {
2028         u16 trans_min, trans_amount, trans_y_tile_min;
2029         u16 wm0_blocks, trans_offset, blocks;
2030
2031         /* Transition WM don't make any sense if ipc is disabled */
2032         if (!skl_watermark_ipc_enabled(i915))
2033                 return;
2034
2035         /*
2036          * WaDisableTWM:skl,kbl,cfl,bxt
2037          * Transition WM are not recommended by HW team for GEN9
2038          */
2039         if (DISPLAY_VER(i915) == 9)
2040                 return;
2041
2042         if (DISPLAY_VER(i915) >= 11)
2043                 trans_min = 4;
2044         else
2045                 trans_min = 14;
2046
2047         /* Display WA #1140: glk,cnl */
2048         if (DISPLAY_VER(i915) == 10)
2049                 trans_amount = 0;
2050         else
2051                 trans_amount = 10; /* This is configurable amount */
2052
2053         trans_offset = trans_min + trans_amount;
2054
2055         /*
2056          * The spec asks for Selected Result Blocks for wm0 (the real value),
2057          * not Result Blocks (the integer value). Pay attention to the capital
2058          * letters. The value wm_l0->blocks is actually Result Blocks, but
2059          * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2060          * and since we later will have to get the ceiling of the sum in the
2061          * transition watermarks calculation, we can just pretend Selected
2062          * Result Blocks is Result Blocks minus 1 and it should work for the
2063          * current platforms.
2064          */
2065         wm0_blocks = wm0->blocks - 1;
2066
2067         if (wp->y_tiled) {
2068                 trans_y_tile_min =
2069                         (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
2070                 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2071         } else {
2072                 blocks = wm0_blocks + trans_offset;
2073         }
2074         blocks++;
2075
2076         /*
2077          * Just assume we can enable the transition watermark.  After
2078          * computing the DDB we'll come back and disable it if that
2079          * assumption turns out to be false.
2080          */
2081         trans_wm->blocks = blocks;
2082         trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2083         trans_wm->enable = true;
2084 }
2085
2086 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2087                                      const struct intel_plane_state *plane_state,
2088                                      struct intel_plane *plane, int color_plane)
2089 {
2090         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2091         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2092         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2093         struct skl_wm_params wm_params;
2094         int ret;
2095
2096         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2097                                           &wm_params, color_plane);
2098         if (ret)
2099                 return ret;
2100
2101         skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
2102
2103         skl_compute_transition_wm(i915, &wm->trans_wm,
2104                                   &wm->wm[0], &wm_params);
2105
2106         if (DISPLAY_VER(i915) >= 12) {
2107                 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
2108
2109                 skl_compute_transition_wm(i915, &wm->sagv.trans_wm,
2110                                           &wm->sagv.wm0, &wm_params);
2111         }
2112
2113         return 0;
2114 }
2115
2116 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2117                                  const struct intel_plane_state *plane_state,
2118                                  struct intel_plane *plane)
2119 {
2120         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2121         struct skl_wm_params wm_params;
2122         int ret;
2123
2124         wm->is_planar = true;
2125
2126         /* uv plane watermarks must also be validated for NV12/Planar */
2127         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2128                                           &wm_params, 1);
2129         if (ret)
2130                 return ret;
2131
2132         skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
2133
2134         return 0;
2135 }
2136
2137 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2138                               const struct intel_plane_state *plane_state)
2139 {
2140         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2141         enum plane_id plane_id = plane->id;
2142         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2143         const struct drm_framebuffer *fb = plane_state->hw.fb;
2144         int ret;
2145
2146         memset(wm, 0, sizeof(*wm));
2147
2148         if (!intel_wm_plane_visible(crtc_state, plane_state))
2149                 return 0;
2150
2151         ret = skl_build_plane_wm_single(crtc_state, plane_state,
2152                                         plane, 0);
2153         if (ret)
2154                 return ret;
2155
2156         if (fb->format->is_yuv && fb->format->num_planes > 1) {
2157                 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2158                                             plane);
2159                 if (ret)
2160                         return ret;
2161         }
2162
2163         return 0;
2164 }
2165
2166 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2167                               const struct intel_plane_state *plane_state)
2168 {
2169         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2170         struct drm_i915_private *i915 = to_i915(plane->base.dev);
2171         enum plane_id plane_id = plane->id;
2172         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2173         int ret;
2174
2175         /* Watermarks calculated in master */
2176         if (plane_state->planar_slave)
2177                 return 0;
2178
2179         memset(wm, 0, sizeof(*wm));
2180
2181         if (plane_state->planar_linked_plane) {
2182                 const struct drm_framebuffer *fb = plane_state->hw.fb;
2183
2184                 drm_WARN_ON(&i915->drm,
2185                             !intel_wm_plane_visible(crtc_state, plane_state));
2186                 drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
2187                             fb->format->num_planes == 1);
2188
2189                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2190                                                 plane_state->planar_linked_plane, 0);
2191                 if (ret)
2192                         return ret;
2193
2194                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2195                                                 plane, 1);
2196                 if (ret)
2197                         return ret;
2198         } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2199                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2200                                                 plane, 0);
2201                 if (ret)
2202                         return ret;
2203         }
2204
2205         return 0;
2206 }
2207
2208 static bool
2209 skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
2210                         int wm0_lines, int latency)
2211 {
2212         const struct drm_display_mode *adjusted_mode =
2213                 &crtc_state->hw.adjusted_mode;
2214
2215         /* FIXME missing scaler and DSC pre-fill time */
2216         return crtc_state->framestart_delay +
2217                 intel_usecs_to_scanlines(adjusted_mode, latency) +
2218                 wm0_lines >
2219                 adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
2220 }
2221
2222 static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2223 {
2224         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2225         enum plane_id plane_id;
2226         int wm0_lines = 0;
2227
2228         for_each_plane_id_on_crtc(crtc, plane_id) {
2229                 const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2230
2231                 /* FIXME what about !skl_wm_has_lines() platforms? */
2232                 wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2233         }
2234
2235         return wm0_lines;
2236 }
2237
2238 static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2239                                        int wm0_lines)
2240 {
2241         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2242         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2243         int level;
2244
2245         for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
2246                 int latency;
2247
2248                 /* FIXME should we care about the latency w/a's? */
2249                 latency = skl_wm_latency(i915, level, NULL);
2250                 if (latency == 0)
2251                         continue;
2252
2253                 /* FIXME is it correct to use 0 latency for wm0 here? */
2254                 if (level == 0)
2255                         latency = 0;
2256
2257                 if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
2258                         return level;
2259         }
2260
2261         return -EINVAL;
2262 }
2263
2264 static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2265 {
2266         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2267         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2268         int wm0_lines, level;
2269
2270         if (!crtc_state->hw.active)
2271                 return 0;
2272
2273         wm0_lines = skl_max_wm0_lines(crtc_state);
2274
2275         level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
2276         if (level < 0)
2277                 return level;
2278
2279         /*
2280          * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2281          * based on whether we're limited by the vblank duration.
2282          */
2283         crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1;
2284
2285         for (level++; level < i915->display.wm.num_levels; level++) {
2286                 enum plane_id plane_id;
2287
2288                 for_each_plane_id_on_crtc(crtc, plane_id) {
2289                         struct skl_plane_wm *wm =
2290                                 &crtc_state->wm.skl.optimal.planes[plane_id];
2291
2292                         /*
2293                          * FIXME just clear enable or flag the entire
2294                          * thing as bad via min_ddb_alloc=U16_MAX?
2295                          */
2296                         wm->wm[level].enable = false;
2297                         wm->uv_wm[level].enable = false;
2298                 }
2299         }
2300
2301         if (DISPLAY_VER(i915) >= 12 &&
2302             i915->display.sagv.block_time_us &&
2303             skl_is_vblank_too_short(crtc_state, wm0_lines,
2304                                     i915->display.sagv.block_time_us)) {
2305                 enum plane_id plane_id;
2306
2307                 for_each_plane_id_on_crtc(crtc, plane_id) {
2308                         struct skl_plane_wm *wm =
2309                                 &crtc_state->wm.skl.optimal.planes[plane_id];
2310
2311                         wm->sagv.wm0.enable = false;
2312                         wm->sagv.trans_wm.enable = false;
2313                 }
2314         }
2315
2316         return 0;
2317 }
2318
2319 static int skl_build_pipe_wm(struct intel_atomic_state *state,
2320                              struct intel_crtc *crtc)
2321 {
2322         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2323         struct intel_crtc_state *crtc_state =
2324                 intel_atomic_get_new_crtc_state(state, crtc);
2325         const struct intel_plane_state *plane_state;
2326         struct intel_plane *plane;
2327         int ret, i;
2328
2329         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2330                 /*
2331                  * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2332                  * instead but we don't populate that correctly for NV12 Y
2333                  * planes so for now hack this.
2334                  */
2335                 if (plane->pipe != crtc->pipe)
2336                         continue;
2337
2338                 if (DISPLAY_VER(i915) >= 11)
2339                         ret = icl_build_plane_wm(crtc_state, plane_state);
2340                 else
2341                         ret = skl_build_plane_wm(crtc_state, plane_state);
2342                 if (ret)
2343                         return ret;
2344         }
2345
2346         crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2347
2348         return skl_wm_check_vblank(crtc_state);
2349 }
2350
2351 static void skl_ddb_entry_write(struct drm_i915_private *i915,
2352                                 i915_reg_t reg,
2353                                 const struct skl_ddb_entry *entry)
2354 {
2355         if (entry->end)
2356                 intel_de_write_fw(i915, reg,
2357                                   PLANE_BUF_END(entry->end - 1) |
2358                                   PLANE_BUF_START(entry->start));
2359         else
2360                 intel_de_write_fw(i915, reg, 0);
2361 }
2362
2363 static void skl_write_wm_level(struct drm_i915_private *i915,
2364                                i915_reg_t reg,
2365                                const struct skl_wm_level *level)
2366 {
2367         u32 val = 0;
2368
2369         if (level->enable)
2370                 val |= PLANE_WM_EN;
2371         if (level->ignore_lines)
2372                 val |= PLANE_WM_IGNORE_LINES;
2373         val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks);
2374         val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
2375
2376         intel_de_write_fw(i915, reg, val);
2377 }
2378
2379 void skl_write_plane_wm(struct intel_plane *plane,
2380                         const struct intel_crtc_state *crtc_state)
2381 {
2382         struct drm_i915_private *i915 = to_i915(plane->base.dev);
2383         enum plane_id plane_id = plane->id;
2384         enum pipe pipe = plane->pipe;
2385         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2386         const struct skl_ddb_entry *ddb =
2387                 &crtc_state->wm.skl.plane_ddb[plane_id];
2388         const struct skl_ddb_entry *ddb_y =
2389                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
2390         int level;
2391
2392         for (level = 0; level < i915->display.wm.num_levels; level++)
2393                 skl_write_wm_level(i915, PLANE_WM(pipe, plane_id, level),
2394                                    skl_plane_wm_level(pipe_wm, plane_id, level));
2395
2396         skl_write_wm_level(i915, PLANE_WM_TRANS(pipe, plane_id),
2397                            skl_plane_trans_wm(pipe_wm, plane_id));
2398
2399         if (HAS_HW_SAGV_WM(i915)) {
2400                 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2401
2402                 skl_write_wm_level(i915, PLANE_WM_SAGV(pipe, plane_id),
2403                                    &wm->sagv.wm0);
2404                 skl_write_wm_level(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id),
2405                                    &wm->sagv.trans_wm);
2406         }
2407
2408         skl_ddb_entry_write(i915,
2409                             PLANE_BUF_CFG(pipe, plane_id), ddb);
2410
2411         if (DISPLAY_VER(i915) < 11)
2412                 skl_ddb_entry_write(i915,
2413                                     PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y);
2414 }
2415
2416 void skl_write_cursor_wm(struct intel_plane *plane,
2417                          const struct intel_crtc_state *crtc_state)
2418 {
2419         struct drm_i915_private *i915 = to_i915(plane->base.dev);
2420         enum plane_id plane_id = plane->id;
2421         enum pipe pipe = plane->pipe;
2422         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2423         const struct skl_ddb_entry *ddb =
2424                 &crtc_state->wm.skl.plane_ddb[plane_id];
2425         int level;
2426
2427         for (level = 0; level < i915->display.wm.num_levels; level++)
2428                 skl_write_wm_level(i915, CUR_WM(pipe, level),
2429                                    skl_plane_wm_level(pipe_wm, plane_id, level));
2430
2431         skl_write_wm_level(i915, CUR_WM_TRANS(pipe),
2432                            skl_plane_trans_wm(pipe_wm, plane_id));
2433
2434         if (HAS_HW_SAGV_WM(i915)) {
2435                 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2436
2437                 skl_write_wm_level(i915, CUR_WM_SAGV(pipe),
2438                                    &wm->sagv.wm0);
2439                 skl_write_wm_level(i915, CUR_WM_SAGV_TRANS(pipe),
2440                                    &wm->sagv.trans_wm);
2441         }
2442
2443         skl_ddb_entry_write(i915, CUR_BUF_CFG(pipe), ddb);
2444 }
2445
2446 static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2447                                 const struct skl_wm_level *l2)
2448 {
2449         return l1->enable == l2->enable &&
2450                 l1->ignore_lines == l2->ignore_lines &&
2451                 l1->lines == l2->lines &&
2452                 l1->blocks == l2->blocks;
2453 }
2454
2455 static bool skl_plane_wm_equals(struct drm_i915_private *i915,
2456                                 const struct skl_plane_wm *wm1,
2457                                 const struct skl_plane_wm *wm2)
2458 {
2459         int level;
2460
2461         for (level = 0; level < i915->display.wm.num_levels; level++) {
2462                 /*
2463                  * We don't check uv_wm as the hardware doesn't actually
2464                  * use it. It only gets used for calculating the required
2465                  * ddb allocation.
2466                  */
2467                 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
2468                         return false;
2469         }
2470
2471         return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
2472                 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
2473                 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
2474 }
2475
2476 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2477                                     const struct skl_ddb_entry *b)
2478 {
2479         return a->start < b->end && b->start < a->end;
2480 }
2481
2482 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2483                                 const struct skl_ddb_entry *b)
2484 {
2485         if (a->end && b->end) {
2486                 a->start = min(a->start, b->start);
2487                 a->end = max(a->end, b->end);
2488         } else if (b->end) {
2489                 a->start = b->start;
2490                 a->end = b->end;
2491         }
2492 }
2493
2494 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2495                                  const struct skl_ddb_entry *entries,
2496                                  int num_entries, int ignore_idx)
2497 {
2498         int i;
2499
2500         for (i = 0; i < num_entries; i++) {
2501                 if (i != ignore_idx &&
2502                     skl_ddb_entries_overlap(ddb, &entries[i]))
2503                         return true;
2504         }
2505
2506         return false;
2507 }
2508
2509 static int
2510 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
2511                             struct intel_crtc_state *new_crtc_state)
2512 {
2513         struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
2514         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2515         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2516         struct intel_plane *plane;
2517
2518         for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2519                 struct intel_plane_state *plane_state;
2520                 enum plane_id plane_id = plane->id;
2521
2522                 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
2523                                         &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2524                     skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2525                                         &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2526                         continue;
2527
2528                 plane_state = intel_atomic_get_plane_state(state, plane);
2529                 if (IS_ERR(plane_state))
2530                         return PTR_ERR(plane_state);
2531
2532                 new_crtc_state->update_planes |= BIT(plane_id);
2533                 new_crtc_state->async_flip_planes = 0;
2534                 new_crtc_state->do_async_flip = false;
2535         }
2536
2537         return 0;
2538 }
2539
2540 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2541 {
2542         struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev);
2543         u8 enabled_slices;
2544         enum pipe pipe;
2545
2546         /*
2547          * FIXME: For now we always enable slice S1 as per
2548          * the Bspec display initialization sequence.
2549          */
2550         enabled_slices = BIT(DBUF_S1);
2551
2552         for_each_pipe(i915, pipe)
2553                 enabled_slices |= dbuf_state->slices[pipe];
2554
2555         return enabled_slices;
2556 }
2557
2558 static int
2559 skl_compute_ddb(struct intel_atomic_state *state)
2560 {
2561         struct drm_i915_private *i915 = to_i915(state->base.dev);
2562         const struct intel_dbuf_state *old_dbuf_state;
2563         struct intel_dbuf_state *new_dbuf_state = NULL;
2564         const struct intel_crtc_state *old_crtc_state;
2565         struct intel_crtc_state *new_crtc_state;
2566         struct intel_crtc *crtc;
2567         int ret, i;
2568
2569         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2570                 new_dbuf_state = intel_atomic_get_dbuf_state(state);
2571                 if (IS_ERR(new_dbuf_state))
2572                         return PTR_ERR(new_dbuf_state);
2573
2574                 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2575                 break;
2576         }
2577
2578         if (!new_dbuf_state)
2579                 return 0;
2580
2581         new_dbuf_state->active_pipes =
2582                 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
2583
2584         if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2585                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2586                 if (ret)
2587                         return ret;
2588         }
2589
2590         if (HAS_MBUS_JOINING(i915))
2591                 new_dbuf_state->joined_mbus =
2592                         adlp_check_mbus_joined(new_dbuf_state->active_pipes);
2593
2594         for_each_intel_crtc(&i915->drm, crtc) {
2595                 enum pipe pipe = crtc->pipe;
2596
2597                 new_dbuf_state->slices[pipe] =
2598                         skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
2599                                                 new_dbuf_state->joined_mbus);
2600
2601                 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2602                         continue;
2603
2604                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2605                 if (ret)
2606                         return ret;
2607         }
2608
2609         new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
2610
2611         if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2612             old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2613                 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
2614                 if (ret)
2615                         return ret;
2616
2617                 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2618                         /* TODO: Implement vblank synchronized MBUS joining changes */
2619                         ret = intel_modeset_all_pipes(state, "MBUS joining change");
2620                         if (ret)
2621                                 return ret;
2622                 }
2623
2624                 drm_dbg_kms(&i915->drm,
2625                             "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2626                             old_dbuf_state->enabled_slices,
2627                             new_dbuf_state->enabled_slices,
2628                             INTEL_INFO(i915)->display.dbuf.slice_mask,
2629                             str_yes_no(old_dbuf_state->joined_mbus),
2630                             str_yes_no(new_dbuf_state->joined_mbus));
2631         }
2632
2633         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2634                 enum pipe pipe = crtc->pipe;
2635
2636                 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
2637
2638                 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2639                         continue;
2640
2641                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2642                 if (ret)
2643                         return ret;
2644         }
2645
2646         for_each_intel_crtc(&i915->drm, crtc) {
2647                 ret = skl_crtc_allocate_ddb(state, crtc);
2648                 if (ret)
2649                         return ret;
2650         }
2651
2652         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2653                                             new_crtc_state, i) {
2654                 ret = skl_crtc_allocate_plane_ddb(state, crtc);
2655                 if (ret)
2656                         return ret;
2657
2658                 ret = skl_ddb_add_affected_planes(old_crtc_state,
2659                                                   new_crtc_state);
2660                 if (ret)
2661                         return ret;
2662         }
2663
2664         return 0;
2665 }
2666
2667 static char enast(bool enable)
2668 {
2669         return enable ? '*' : ' ';
2670 }
2671
2672 static void
2673 skl_print_wm_changes(struct intel_atomic_state *state)
2674 {
2675         struct drm_i915_private *i915 = to_i915(state->base.dev);
2676         const struct intel_crtc_state *old_crtc_state;
2677         const struct intel_crtc_state *new_crtc_state;
2678         struct intel_plane *plane;
2679         struct intel_crtc *crtc;
2680         int i;
2681
2682         if (!drm_debug_enabled(DRM_UT_KMS))
2683                 return;
2684
2685         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2686                                             new_crtc_state, i) {
2687                 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2688
2689                 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2690                 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2691
2692                 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2693                         enum plane_id plane_id = plane->id;
2694                         const struct skl_ddb_entry *old, *new;
2695
2696                         old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2697                         new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2698
2699                         if (skl_ddb_entry_equal(old, new))
2700                                 continue;
2701
2702                         drm_dbg_kms(&i915->drm,
2703                                     "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2704                                     plane->base.base.id, plane->base.name,
2705                                     old->start, old->end, new->start, new->end,
2706                                     skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2707                 }
2708
2709                 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2710                         enum plane_id plane_id = plane->id;
2711                         const struct skl_plane_wm *old_wm, *new_wm;
2712
2713                         old_wm = &old_pipe_wm->planes[plane_id];
2714                         new_wm = &new_pipe_wm->planes[plane_id];
2715
2716                         if (skl_plane_wm_equals(i915, old_wm, new_wm))
2717                                 continue;
2718
2719                         drm_dbg_kms(&i915->drm,
2720                                     "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2721                                     " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2722                                     plane->base.base.id, plane->base.name,
2723                                     enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2724                                     enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2725                                     enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2726                                     enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2727                                     enast(old_wm->trans_wm.enable),
2728                                     enast(old_wm->sagv.wm0.enable),
2729                                     enast(old_wm->sagv.trans_wm.enable),
2730                                     enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2731                                     enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2732                                     enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2733                                     enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2734                                     enast(new_wm->trans_wm.enable),
2735                                     enast(new_wm->sagv.wm0.enable),
2736                                     enast(new_wm->sagv.trans_wm.enable));
2737
2738                         drm_dbg_kms(&i915->drm,
2739                                     "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2740                                       " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2741                                     plane->base.base.id, plane->base.name,
2742                                     enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2743                                     enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2744                                     enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2745                                     enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2746                                     enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2747                                     enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2748                                     enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2749                                     enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2750                                     enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2751                                     enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2752                                     enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2753                                     enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2754                                     enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2755                                     enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2756                                     enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2757                                     enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2758                                     enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2759                                     enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2760                                     enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2761                                     enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2762                                     enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2763                                     enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2764
2765                         drm_dbg_kms(&i915->drm,
2766                                     "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2767                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2768                                     plane->base.base.id, plane->base.name,
2769                                     old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2770                                     old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2771                                     old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2772                                     old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2773                                     old_wm->trans_wm.blocks,
2774                                     old_wm->sagv.wm0.blocks,
2775                                     old_wm->sagv.trans_wm.blocks,
2776                                     new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2777                                     new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2778                                     new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2779                                     new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2780                                     new_wm->trans_wm.blocks,
2781                                     new_wm->sagv.wm0.blocks,
2782                                     new_wm->sagv.trans_wm.blocks);
2783
2784                         drm_dbg_kms(&i915->drm,
2785                                     "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2786                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2787                                     plane->base.base.id, plane->base.name,
2788                                     old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2789                                     old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2790                                     old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2791                                     old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2792                                     old_wm->trans_wm.min_ddb_alloc,
2793                                     old_wm->sagv.wm0.min_ddb_alloc,
2794                                     old_wm->sagv.trans_wm.min_ddb_alloc,
2795                                     new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2796                                     new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2797                                     new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2798                                     new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2799                                     new_wm->trans_wm.min_ddb_alloc,
2800                                     new_wm->sagv.wm0.min_ddb_alloc,
2801                                     new_wm->sagv.trans_wm.min_ddb_alloc);
2802                 }
2803         }
2804 }
2805
2806 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2807                                          const struct skl_pipe_wm *old_pipe_wm,
2808                                          const struct skl_pipe_wm *new_pipe_wm)
2809 {
2810         struct drm_i915_private *i915 = to_i915(plane->base.dev);
2811         int level;
2812
2813         for (level = 0; level < i915->display.wm.num_levels; level++) {
2814                 /*
2815                  * We don't check uv_wm as the hardware doesn't actually
2816                  * use it. It only gets used for calculating the required
2817                  * ddb allocation.
2818                  */
2819                 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
2820                                          skl_plane_wm_level(new_pipe_wm, plane->id, level)))
2821                         return false;
2822         }
2823
2824         if (HAS_HW_SAGV_WM(i915)) {
2825                 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2826                 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2827
2828                 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
2829                     !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
2830                         return false;
2831         }
2832
2833         return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
2834                                    skl_plane_trans_wm(new_pipe_wm, plane->id));
2835 }
2836
2837 /*
2838  * To make sure the cursor watermark registers are always consistent
2839  * with our computed state the following scenario needs special
2840  * treatment:
2841  *
2842  * 1. enable cursor
2843  * 2. move cursor entirely offscreen
2844  * 3. disable cursor
2845  *
2846  * Step 2. does call .disable_plane() but does not zero the watermarks
2847  * (since we consider an offscreen cursor still active for the purposes
2848  * of watermarks). Step 3. would not normally call .disable_plane()
2849  * because the actual plane visibility isn't changing, and we don't
2850  * deallocate the cursor ddb until the pipe gets disabled. So we must
2851  * force step 3. to call .disable_plane() to update the watermark
2852  * registers properly.
2853  *
2854  * Other planes do not suffer from this issues as their watermarks are
2855  * calculated based on the actual plane visibility. The only time this
2856  * can trigger for the other planes is during the initial readout as the
2857  * default value of the watermarks registers is not zero.
2858  */
2859 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2860                                       struct intel_crtc *crtc)
2861 {
2862         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2863         const struct intel_crtc_state *old_crtc_state =
2864                 intel_atomic_get_old_crtc_state(state, crtc);
2865         struct intel_crtc_state *new_crtc_state =
2866                 intel_atomic_get_new_crtc_state(state, crtc);
2867         struct intel_plane *plane;
2868
2869         for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2870                 struct intel_plane_state *plane_state;
2871                 enum plane_id plane_id = plane->id;
2872
2873                 /*
2874                  * Force a full wm update for every plane on modeset.
2875                  * Required because the reset value of the wm registers
2876                  * is non-zero, whereas we want all disabled planes to
2877                  * have zero watermarks. So if we turn off the relevant
2878                  * power well the hardware state will go out of sync
2879                  * with the software state.
2880                  */
2881                 if (!intel_crtc_needs_modeset(new_crtc_state) &&
2882                     skl_plane_selected_wm_equals(plane,
2883                                                  &old_crtc_state->wm.skl.optimal,
2884                                                  &new_crtc_state->wm.skl.optimal))
2885                         continue;
2886
2887                 plane_state = intel_atomic_get_plane_state(state, plane);
2888                 if (IS_ERR(plane_state))
2889                         return PTR_ERR(plane_state);
2890
2891                 new_crtc_state->update_planes |= BIT(plane_id);
2892                 new_crtc_state->async_flip_planes = 0;
2893                 new_crtc_state->do_async_flip = false;
2894         }
2895
2896         return 0;
2897 }
2898
2899 static int
2900 skl_compute_wm(struct intel_atomic_state *state)
2901 {
2902         struct intel_crtc *crtc;
2903         struct intel_crtc_state *new_crtc_state;
2904         int ret, i;
2905
2906         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2907                 ret = skl_build_pipe_wm(state, crtc);
2908                 if (ret)
2909                         return ret;
2910         }
2911
2912         ret = skl_compute_ddb(state);
2913         if (ret)
2914                 return ret;
2915
2916         ret = intel_compute_sagv_mask(state);
2917         if (ret)
2918                 return ret;
2919
2920         /*
2921          * skl_compute_ddb() will have adjusted the final watermarks
2922          * based on how much ddb is available. Now we can actually
2923          * check if the final watermarks changed.
2924          */
2925         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2926                 ret = skl_wm_add_affected_planes(state, crtc);
2927                 if (ret)
2928                         return ret;
2929         }
2930
2931         skl_print_wm_changes(state);
2932
2933         return 0;
2934 }
2935
2936 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
2937 {
2938         level->enable = val & PLANE_WM_EN;
2939         level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
2940         level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
2941         level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
2942 }
2943
2944 static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
2945                                      struct skl_pipe_wm *out)
2946 {
2947         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2948         enum pipe pipe = crtc->pipe;
2949         enum plane_id plane_id;
2950         int level;
2951         u32 val;
2952
2953         for_each_plane_id_on_crtc(crtc, plane_id) {
2954                 struct skl_plane_wm *wm = &out->planes[plane_id];
2955
2956                 for (level = 0; level < i915->display.wm.num_levels; level++) {
2957                         if (plane_id != PLANE_CURSOR)
2958                                 val = intel_de_read(i915, PLANE_WM(pipe, plane_id, level));
2959                         else
2960                                 val = intel_de_read(i915, CUR_WM(pipe, level));
2961
2962                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
2963                 }
2964
2965                 if (plane_id != PLANE_CURSOR)
2966                         val = intel_de_read(i915, PLANE_WM_TRANS(pipe, plane_id));
2967                 else
2968                         val = intel_de_read(i915, CUR_WM_TRANS(pipe));
2969
2970                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
2971
2972                 if (HAS_HW_SAGV_WM(i915)) {
2973                         if (plane_id != PLANE_CURSOR)
2974                                 val = intel_de_read(i915, PLANE_WM_SAGV(pipe, plane_id));
2975                         else
2976                                 val = intel_de_read(i915, CUR_WM_SAGV(pipe));
2977
2978                         skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
2979
2980                         if (plane_id != PLANE_CURSOR)
2981                                 val = intel_de_read(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id));
2982                         else
2983                                 val = intel_de_read(i915, CUR_WM_SAGV_TRANS(pipe));
2984
2985                         skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
2986                 } else if (DISPLAY_VER(i915) >= 12) {
2987                         wm->sagv.wm0 = wm->wm[0];
2988                         wm->sagv.trans_wm = wm->trans_wm;
2989                 }
2990         }
2991 }
2992
2993 static void skl_wm_get_hw_state(struct drm_i915_private *i915)
2994 {
2995         struct intel_dbuf_state *dbuf_state =
2996                 to_intel_dbuf_state(i915->display.dbuf.obj.state);
2997         struct intel_crtc *crtc;
2998
2999         if (HAS_MBUS_JOINING(i915))
3000                 dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN;
3001
3002         for_each_intel_crtc(&i915->drm, crtc) {
3003                 struct intel_crtc_state *crtc_state =
3004                         to_intel_crtc_state(crtc->base.state);
3005                 enum pipe pipe = crtc->pipe;
3006                 unsigned int mbus_offset;
3007                 enum plane_id plane_id;
3008                 u8 slices;
3009
3010                 memset(&crtc_state->wm.skl.optimal, 0,
3011                        sizeof(crtc_state->wm.skl.optimal));
3012                 if (crtc_state->hw.active)
3013                         skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
3014                 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3015
3016                 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3017
3018                 for_each_plane_id_on_crtc(crtc, plane_id) {
3019                         struct skl_ddb_entry *ddb =
3020                                 &crtc_state->wm.skl.plane_ddb[plane_id];
3021                         struct skl_ddb_entry *ddb_y =
3022                                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
3023
3024                         if (!crtc_state->hw.active)
3025                                 continue;
3026
3027                         skl_ddb_get_hw_plane_state(i915, crtc->pipe,
3028                                                    plane_id, ddb, ddb_y);
3029
3030                         skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
3031                         skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
3032                 }
3033
3034                 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3035
3036                 /*
3037                  * Used for checking overlaps, so we need absolute
3038                  * offsets instead of MBUS relative offsets.
3039                  */
3040                 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3041                                                  dbuf_state->joined_mbus);
3042                 mbus_offset = mbus_ddb_offset(i915, slices);
3043                 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3044                 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3045
3046                 /* The slices actually used by the planes on the pipe */
3047                 dbuf_state->slices[pipe] =
3048                         skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb);
3049
3050                 drm_dbg_kms(&i915->drm,
3051                             "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3052                             crtc->base.base.id, crtc->base.name,
3053                             dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3054                             dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3055                             str_yes_no(dbuf_state->joined_mbus));
3056         }
3057
3058         dbuf_state->enabled_slices = i915->display.dbuf.enabled_slices;
3059 }
3060
3061 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
3062 {
3063         const struct intel_dbuf_state *dbuf_state =
3064                 to_intel_dbuf_state(i915->display.dbuf.obj.state);
3065         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3066         struct intel_crtc *crtc;
3067
3068         for_each_intel_crtc(&i915->drm, crtc) {
3069                 const struct intel_crtc_state *crtc_state =
3070                         to_intel_crtc_state(crtc->base.state);
3071
3072                 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3073         }
3074
3075         for_each_intel_crtc(&i915->drm, crtc) {
3076                 const struct intel_crtc_state *crtc_state =
3077                         to_intel_crtc_state(crtc->base.state);
3078                 u8 slices;
3079
3080                 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3081                                                  dbuf_state->joined_mbus);
3082                 if (dbuf_state->slices[crtc->pipe] & ~slices)
3083                         return true;
3084
3085                 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
3086                                                 I915_MAX_PIPES, crtc->pipe))
3087                         return true;
3088         }
3089
3090         return false;
3091 }
3092
3093 static void skl_wm_sanitize(struct drm_i915_private *i915)
3094 {
3095         struct intel_crtc *crtc;
3096
3097         /*
3098          * On TGL/RKL (at least) the BIOS likes to assign the planes
3099          * to the wrong DBUF slices. This will cause an infinite loop
3100          * in skl_commit_modeset_enables() as it can't find a way to
3101          * transition between the old bogus DBUF layout to the new
3102          * proper DBUF layout without DBUF allocation overlaps between
3103          * the planes (which cannot be allowed or else the hardware
3104          * may hang). If we detect a bogus DBUF layout just turn off
3105          * all the planes so that skl_commit_modeset_enables() can
3106          * simply ignore them.
3107          */
3108         if (!skl_dbuf_is_misconfigured(i915))
3109                 return;
3110
3111         drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3112
3113         for_each_intel_crtc(&i915->drm, crtc) {
3114                 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3115                 const struct intel_plane_state *plane_state =
3116                         to_intel_plane_state(plane->base.state);
3117                 struct intel_crtc_state *crtc_state =
3118                         to_intel_crtc_state(crtc->base.state);
3119
3120                 if (plane_state->uapi.visible)
3121                         intel_plane_disable_noatomic(crtc, plane);
3122
3123                 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
3124
3125                 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3126         }
3127 }
3128
3129 static void skl_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3130 {
3131         skl_wm_get_hw_state(i915);
3132         skl_wm_sanitize(i915);
3133 }
3134
3135 void intel_wm_state_verify(struct intel_crtc *crtc,
3136                            struct intel_crtc_state *new_crtc_state)
3137 {
3138         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
3139         struct skl_hw_state {
3140                 struct skl_ddb_entry ddb[I915_MAX_PLANES];
3141                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3142                 struct skl_pipe_wm wm;
3143         } *hw;
3144         const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3145         struct intel_plane *plane;
3146         u8 hw_enabled_slices;
3147         int level;
3148
3149         if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
3150                 return;
3151
3152         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3153         if (!hw)
3154                 return;
3155
3156         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
3157
3158         skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y);
3159
3160         hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915);
3161
3162         if (DISPLAY_VER(i915) >= 11 &&
3163             hw_enabled_slices != i915->display.dbuf.enabled_slices)
3164                 drm_err(&i915->drm,
3165                         "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3166                         i915->display.dbuf.enabled_slices,
3167                         hw_enabled_slices);
3168
3169         for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
3170                 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3171                 const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3172
3173                 /* Watermarks */
3174                 for (level = 0; level < i915->display.wm.num_levels; level++) {
3175                         hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3176                         sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
3177
3178                         if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
3179                                 continue;
3180
3181                         drm_err(&i915->drm,
3182                                 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3183                                 plane->base.base.id, plane->base.name, level,
3184                                 sw_wm_level->enable,
3185                                 sw_wm_level->blocks,
3186                                 sw_wm_level->lines,
3187                                 hw_wm_level->enable,
3188                                 hw_wm_level->blocks,
3189                                 hw_wm_level->lines);
3190                 }
3191
3192                 hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3193                 sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
3194
3195                 if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3196                         drm_err(&i915->drm,
3197                                 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3198                                 plane->base.base.id, plane->base.name,
3199                                 sw_wm_level->enable,
3200                                 sw_wm_level->blocks,
3201                                 sw_wm_level->lines,
3202                                 hw_wm_level->enable,
3203                                 hw_wm_level->blocks,
3204                                 hw_wm_level->lines);
3205                 }
3206
3207                 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3208                 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3209
3210                 if (HAS_HW_SAGV_WM(i915) &&
3211                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3212                         drm_err(&i915->drm,
3213                                 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3214                                 plane->base.base.id, plane->base.name,
3215                                 sw_wm_level->enable,
3216                                 sw_wm_level->blocks,
3217                                 sw_wm_level->lines,
3218                                 hw_wm_level->enable,
3219                                 hw_wm_level->blocks,
3220                                 hw_wm_level->lines);
3221                 }
3222
3223                 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3224                 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3225
3226                 if (HAS_HW_SAGV_WM(i915) &&
3227                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3228                         drm_err(&i915->drm,
3229                                 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3230                                 plane->base.base.id, plane->base.name,
3231                                 sw_wm_level->enable,
3232                                 sw_wm_level->blocks,
3233                                 sw_wm_level->lines,
3234                                 hw_wm_level->enable,
3235                                 hw_wm_level->blocks,
3236                                 hw_wm_level->lines);
3237                 }
3238
3239                 /* DDB */
3240                 hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
3241                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
3242
3243                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
3244                         drm_err(&i915->drm,
3245                                 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
3246                                 plane->base.base.id, plane->base.name,
3247                                 sw_ddb_entry->start, sw_ddb_entry->end,
3248                                 hw_ddb_entry->start, hw_ddb_entry->end);
3249                 }
3250         }
3251
3252         kfree(hw);
3253 }
3254
3255 bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
3256 {
3257         return i915->display.wm.ipc_enabled;
3258 }
3259
3260 void skl_watermark_ipc_update(struct drm_i915_private *i915)
3261 {
3262         if (!HAS_IPC(i915))
3263                 return;
3264
3265         intel_de_rmw(i915, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3266                      skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
3267 }
3268
3269 static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
3270 {
3271         /* Display WA #0477 WaDisableIPC: skl */
3272         if (IS_SKYLAKE(i915))
3273                 return false;
3274
3275         /* Display WA #1141: SKL:all KBL:all CFL */
3276         if (IS_KABYLAKE(i915) ||
3277             IS_COFFEELAKE(i915) ||
3278             IS_COMETLAKE(i915))
3279                 return i915->dram_info.symmetric_memory;
3280
3281         return true;
3282 }
3283
3284 void skl_watermark_ipc_init(struct drm_i915_private *i915)
3285 {
3286         if (!HAS_IPC(i915))
3287                 return;
3288
3289         i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
3290
3291         skl_watermark_ipc_update(i915);
3292 }
3293
3294 static void
3295 adjust_wm_latency(struct drm_i915_private *i915,
3296                   u16 wm[], int num_levels, int read_latency)
3297 {
3298         bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
3299         int i, level;
3300
3301         /*
3302          * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3303          * need to be disabled. We make sure to sanitize the values out
3304          * of the punit to satisfy this requirement.
3305          */
3306         for (level = 1; level < num_levels; level++) {
3307                 if (wm[level] == 0) {
3308                         for (i = level + 1; i < num_levels; i++)
3309                                 wm[i] = 0;
3310
3311                         num_levels = level;
3312                         break;
3313                 }
3314         }
3315
3316         /*
3317          * WaWmMemoryReadLatency
3318          *
3319          * punit doesn't take into account the read latency so we need
3320          * to add proper adjustement to each valid level we retrieve
3321          * from the punit when level 0 response data is 0us.
3322          */
3323         if (wm[0] == 0) {
3324                 for (level = 0; level < num_levels; level++)
3325                         wm[level] += read_latency;
3326         }
3327
3328         /*
3329          * WA Level-0 adjustment for 16GB DIMMs: SKL+
3330          * If we could not get dimm info enable this WA to prevent from
3331          * any underrun. If not able to get Dimm info assume 16GB dimm
3332          * to avoid any underrun.
3333          */
3334         if (wm_lv_0_adjust_needed)
3335                 wm[0] += 1;
3336 }
3337
3338 static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3339 {
3340         int num_levels = i915->display.wm.num_levels;
3341         u32 val;
3342
3343         val = intel_de_read(i915, MTL_LATENCY_LP0_LP1);
3344         wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3345         wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3346
3347         val = intel_de_read(i915, MTL_LATENCY_LP2_LP3);
3348         wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3349         wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3350
3351         val = intel_de_read(i915, MTL_LATENCY_LP4_LP5);
3352         wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3353         wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3354
3355         adjust_wm_latency(i915, wm, num_levels, 6);
3356 }
3357
3358 static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3359 {
3360         int num_levels = i915->display.wm.num_levels;
3361         int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
3362         int mult = IS_DG2(i915) ? 2 : 1;
3363         u32 val;
3364         int ret;
3365
3366         /* read the first set of memory latencies[0:3] */
3367         val = 0; /* data0 to be programmed to 0 for first set */
3368         ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3369         if (ret) {
3370                 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3371                 return;
3372         }
3373
3374         wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3375         wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3376         wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3377         wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3378
3379         /* read the second set of memory latencies[4:7] */
3380         val = 1; /* data0 to be programmed to 1 for second set */
3381         ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3382         if (ret) {
3383                 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3384                 return;
3385         }
3386
3387         wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3388         wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3389         wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3390         wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3391
3392         adjust_wm_latency(i915, wm, num_levels, read_latency);
3393 }
3394
3395 static void skl_setup_wm_latency(struct drm_i915_private *i915)
3396 {
3397         if (HAS_HW_SAGV_WM(i915))
3398                 i915->display.wm.num_levels = 6;
3399         else
3400                 i915->display.wm.num_levels = 8;
3401
3402         if (DISPLAY_VER(i915) >= 14)
3403                 mtl_read_wm_latency(i915, i915->display.wm.skl_latency);
3404         else
3405                 skl_read_wm_latency(i915, i915->display.wm.skl_latency);
3406
3407         intel_print_wm_latency(i915, "Gen9 Plane", i915->display.wm.skl_latency);
3408 }
3409
3410 static const struct intel_wm_funcs skl_wm_funcs = {
3411         .compute_global_watermarks = skl_compute_wm,
3412         .get_hw_state = skl_wm_get_hw_state_and_sanitize,
3413 };
3414
3415 void skl_wm_init(struct drm_i915_private *i915)
3416 {
3417         intel_sagv_init(i915);
3418
3419         skl_setup_wm_latency(i915);
3420
3421         i915->display.funcs.wm = &skl_wm_funcs;
3422 }
3423
3424 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3425 {
3426         struct intel_dbuf_state *dbuf_state;
3427
3428         dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3429         if (!dbuf_state)
3430                 return NULL;
3431
3432         return &dbuf_state->base;
3433 }
3434
3435 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3436                                      struct intel_global_state *state)
3437 {
3438         kfree(state);
3439 }
3440
3441 static const struct intel_global_state_funcs intel_dbuf_funcs = {
3442         .atomic_duplicate_state = intel_dbuf_duplicate_state,
3443         .atomic_destroy_state = intel_dbuf_destroy_state,
3444 };
3445
3446 struct intel_dbuf_state *
3447 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3448 {
3449         struct drm_i915_private *i915 = to_i915(state->base.dev);
3450         struct intel_global_state *dbuf_state;
3451
3452         dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj);
3453         if (IS_ERR(dbuf_state))
3454                 return ERR_CAST(dbuf_state);
3455
3456         return to_intel_dbuf_state(dbuf_state);
3457 }
3458
3459 int intel_dbuf_init(struct drm_i915_private *i915)
3460 {
3461         struct intel_dbuf_state *dbuf_state;
3462
3463         dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3464         if (!dbuf_state)
3465                 return -ENOMEM;
3466
3467         intel_atomic_global_obj_init(i915, &i915->display.dbuf.obj,
3468                                      &dbuf_state->base, &intel_dbuf_funcs);
3469
3470         return 0;
3471 }
3472
3473 /*
3474  * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
3475  * update the request state of all DBUS slices.
3476  */
3477 static void update_mbus_pre_enable(struct intel_atomic_state *state)
3478 {
3479         struct drm_i915_private *i915 = to_i915(state->base.dev);
3480         u32 mbus_ctl, dbuf_min_tracker_val;
3481         enum dbuf_slice slice;
3482         const struct intel_dbuf_state *dbuf_state =
3483                 intel_atomic_get_new_dbuf_state(state);
3484
3485         if (!HAS_MBUS_JOINING(i915))
3486                 return;
3487
3488         /*
3489          * TODO: Implement vblank synchronized MBUS joining changes.
3490          * Must be properly coordinated with dbuf reprogramming.
3491          */
3492         if (dbuf_state->joined_mbus) {
3493                 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
3494                         MBUS_JOIN_PIPE_SELECT_NONE;
3495                 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
3496         } else {
3497                 mbus_ctl = MBUS_HASHING_MODE_2x2 |
3498                         MBUS_JOIN_PIPE_SELECT_NONE;
3499                 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
3500         }
3501
3502         intel_de_rmw(i915, MBUS_CTL,
3503                      MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3504                      MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
3505
3506         for_each_dbuf_slice(i915, slice)
3507                 intel_de_rmw(i915, DBUF_CTL_S(slice),
3508                              DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3509                              dbuf_min_tracker_val);
3510 }
3511
3512 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3513 {
3514         struct drm_i915_private *i915 = to_i915(state->base.dev);
3515         const struct intel_dbuf_state *new_dbuf_state =
3516                 intel_atomic_get_new_dbuf_state(state);
3517         const struct intel_dbuf_state *old_dbuf_state =
3518                 intel_atomic_get_old_dbuf_state(state);
3519
3520         if (!new_dbuf_state ||
3521             (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3522              new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3523                 return;
3524
3525         WARN_ON(!new_dbuf_state->base.changed);
3526
3527         update_mbus_pre_enable(state);
3528         gen9_dbuf_slices_update(i915,
3529                                 old_dbuf_state->enabled_slices |
3530                                 new_dbuf_state->enabled_slices);
3531 }
3532
3533 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3534 {
3535         struct drm_i915_private *i915 = to_i915(state->base.dev);
3536         const struct intel_dbuf_state *new_dbuf_state =
3537                 intel_atomic_get_new_dbuf_state(state);
3538         const struct intel_dbuf_state *old_dbuf_state =
3539                 intel_atomic_get_old_dbuf_state(state);
3540
3541         if (!new_dbuf_state ||
3542             (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3543              new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3544                 return;
3545
3546         WARN_ON(!new_dbuf_state->base.changed);
3547
3548         gen9_dbuf_slices_update(i915,
3549                                 new_dbuf_state->enabled_slices);
3550 }
3551
3552 static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3553 {
3554         switch (pipe) {
3555         case PIPE_A:
3556                 return !(active_pipes & BIT(PIPE_D));
3557         case PIPE_D:
3558                 return !(active_pipes & BIT(PIPE_A));
3559         case PIPE_B:
3560                 return !(active_pipes & BIT(PIPE_C));
3561         case PIPE_C:
3562                 return !(active_pipes & BIT(PIPE_B));
3563         default: /* to suppress compiler warning */
3564                 MISSING_CASE(pipe);
3565                 break;
3566         }
3567
3568         return false;
3569 }
3570
3571 void intel_mbus_dbox_update(struct intel_atomic_state *state)
3572 {
3573         struct drm_i915_private *i915 = to_i915(state->base.dev);
3574         const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3575         const struct intel_crtc_state *new_crtc_state;
3576         const struct intel_crtc *crtc;
3577         u32 val = 0;
3578         int i;
3579
3580         if (DISPLAY_VER(i915) < 11)
3581                 return;
3582
3583         new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3584         old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3585         if (!new_dbuf_state ||
3586             (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3587              new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3588                 return;
3589
3590         if (DISPLAY_VER(i915) >= 14)
3591                 val |= MBUS_DBOX_I_CREDIT(2);
3592
3593         if (DISPLAY_VER(i915) >= 12) {
3594                 val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3595                 val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3596                 val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3597         }
3598
3599         if (DISPLAY_VER(i915) >= 14)
3600                 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(12) :
3601                                                      MBUS_DBOX_A_CREDIT(8);
3602         else if (IS_ALDERLAKE_P(i915))
3603                 /* Wa_22010947358:adl-p */
3604                 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
3605                                                      MBUS_DBOX_A_CREDIT(4);
3606         else
3607                 val |= MBUS_DBOX_A_CREDIT(2);
3608
3609         if (DISPLAY_VER(i915) >= 14) {
3610                 val |= MBUS_DBOX_B_CREDIT(0xA);
3611         } else if (IS_ALDERLAKE_P(i915)) {
3612                 val |= MBUS_DBOX_BW_CREDIT(2);
3613                 val |= MBUS_DBOX_B_CREDIT(8);
3614         } else if (DISPLAY_VER(i915) >= 12) {
3615                 val |= MBUS_DBOX_BW_CREDIT(2);
3616                 val |= MBUS_DBOX_B_CREDIT(12);
3617         } else {
3618                 val |= MBUS_DBOX_BW_CREDIT(1);
3619                 val |= MBUS_DBOX_B_CREDIT(8);
3620         }
3621
3622         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3623                 u32 pipe_val = val;
3624
3625                 if (!new_crtc_state->hw.active)
3626                         continue;
3627
3628                 if (DISPLAY_VER(i915) >= 14) {
3629                         if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe,
3630                                                               new_dbuf_state->active_pipes))
3631                                 pipe_val |= MBUS_DBOX_BW_8CREDITS_MTL;
3632                         else
3633                                 pipe_val |= MBUS_DBOX_BW_4CREDITS_MTL;
3634                 }
3635
3636                 intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), pipe_val);
3637         }
3638 }
3639
3640 static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
3641 {
3642         struct drm_i915_private *i915 = m->private;
3643
3644         seq_printf(m, "Isochronous Priority Control: %s\n",
3645                    str_yes_no(skl_watermark_ipc_enabled(i915)));
3646         return 0;
3647 }
3648
3649 static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
3650 {
3651         struct drm_i915_private *i915 = inode->i_private;
3652
3653         return single_open(file, skl_watermark_ipc_status_show, i915);
3654 }
3655
3656 static ssize_t skl_watermark_ipc_status_write(struct file *file,
3657                                               const char __user *ubuf,
3658                                               size_t len, loff_t *offp)
3659 {
3660         struct seq_file *m = file->private_data;
3661         struct drm_i915_private *i915 = m->private;
3662         intel_wakeref_t wakeref;
3663         bool enable;
3664         int ret;
3665
3666         ret = kstrtobool_from_user(ubuf, len, &enable);
3667         if (ret < 0)
3668                 return ret;
3669
3670         with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
3671                 if (!skl_watermark_ipc_enabled(i915) && enable)
3672                         drm_info(&i915->drm,
3673                                  "Enabling IPC: WM will be proper only after next commit\n");
3674                 i915->display.wm.ipc_enabled = enable;
3675                 skl_watermark_ipc_update(i915);
3676         }
3677
3678         return len;
3679 }
3680
3681 static const struct file_operations skl_watermark_ipc_status_fops = {
3682         .owner = THIS_MODULE,
3683         .open = skl_watermark_ipc_status_open,
3684         .read = seq_read,
3685         .llseek = seq_lseek,
3686         .release = single_release,
3687         .write = skl_watermark_ipc_status_write
3688 };
3689
3690 static int intel_sagv_status_show(struct seq_file *m, void *unused)
3691 {
3692         struct drm_i915_private *i915 = m->private;
3693         static const char * const sagv_status[] = {
3694                 [I915_SAGV_UNKNOWN] = "unknown",
3695                 [I915_SAGV_DISABLED] = "disabled",
3696                 [I915_SAGV_ENABLED] = "enabled",
3697                 [I915_SAGV_NOT_CONTROLLED] = "not controlled",
3698         };
3699
3700         seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915)));
3701         seq_printf(m, "SAGV modparam: %s\n", str_enabled_disabled(i915->params.enable_sagv));
3702         seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
3703         seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
3704
3705         return 0;
3706 }
3707
3708 DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
3709
3710 void skl_watermark_debugfs_register(struct drm_i915_private *i915)
3711 {
3712         struct drm_minor *minor = i915->drm.primary;
3713
3714         if (HAS_IPC(i915))
3715                 debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
3716                                     &skl_watermark_ipc_status_fops);
3717
3718         if (HAS_SAGV(i915))
3719                 debugfs_create_file("i915_sagv_status", 0444, minor->debugfs_root, i915,
3720                                     &intel_sagv_status_fops);
3721 }