drivers/gpu/drm/i915/display/intel_bw.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <drm/drm_atomic_state_helper.h>
   7
   8 #include "intel_atomic.h"
   9 #include "intel_bw.h"
  10 #include "intel_cdclk.h"
  11 #include "intel_display_types.h"
  12 #include "intel_pm.h"
  13 #include "intel_sideband.h"
  14
  15 /* Parameters for Qclk Geyserville (QGV) */
  16 struct intel_qgv_point {
  17         u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
  18 };
  19
  20 struct intel_psf_gv_point {
  21         u8 clk; /* clock in multiples of 16.6666 MHz */
  22 };
  23
  24 struct intel_qgv_info {
  25         struct intel_qgv_point points[I915_NUM_QGV_POINTS];
  26         struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
  27         u8 num_points;
  28         u8 num_psf_points;
  29         u8 t_bl;
  30 };
  31
  32 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv,
  33                                           struct intel_qgv_point *sp,
  34                                           int point)
  35 {
  36         u32 dclk_ratio, dclk_reference;
  37         u32 val;
  38
  39         val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
  40         dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
  41         if (val & DG1_QCLK_REFERENCE)
  42                 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
  43         else
  44                 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
  45         sp->dclk = dclk_ratio * dclk_reference;
  46
  47         val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
  48         if (val & DG1_GEAR_TYPE)
  49                 sp->dclk *= 2;
  50
  51         if (sp->dclk == 0)
  52                 return -EINVAL;
  53
  54         val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
  55         sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
  56         sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
  57
  58         val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
  59         sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
  60         sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
  61
  62         sp->t_rc = sp->t_rp + sp->t_ras;
  63
  64         return 0;
  65 }
  66
  67 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv,
  68                                          struct intel_qgv_point *sp,
  69                                          int point)
  70 {
  71         u32 val = 0, val2 = 0;
  72         int ret;
  73
  74         ret = sandybridge_pcode_read(dev_priv,
  75                                      ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
  76                                      ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
  77                                      &val, &val2);
  78         if (ret)
  79                 return ret;
  80
  81         sp->dclk = val & 0xffff;
  82         sp->t_rp = (val & 0xff0000) >> 16;
  83         sp->t_rcd = (val & 0xff000000) >> 24;
  84
  85         sp->t_rdpre = val2 & 0xff;
  86         sp->t_ras = (val2 & 0xff00) >> 8;
  87
  88         sp->t_rc = sp->t_rp + sp->t_ras;
  89
  90         return 0;
  91 }
  92
  93 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
  94                                             struct intel_psf_gv_point *points)
  95 {
  96         u32 val = 0;
  97         int ret;
  98         int i;
  99
 100         ret = sandybridge_pcode_read(dev_priv,
 101                                      ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
 102                                      ADL_PCODE_MEM_SS_READ_PSF_GV_INFO,
 103                                      &val, NULL);
 104         if (ret)
 105                 return ret;
 106
 107         for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
 108                 points[i].clk = val & 0xff;
 109                 val >>= 8;
 110         }
 111
 112         return 0;
 113 }
 114
 115 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
 116                                   u32 points_mask)
 117 {
 118         int ret;
 119
 120         /* bspec says to keep retrying for at least 1 ms */
 121         ret = skl_pcode_request(dev_priv, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
 122                                 points_mask,
 123                                 ICL_PCODE_POINTS_RESTRICTED_MASK,
 124                                 ICL_PCODE_POINTS_RESTRICTED,
 125                                 1);
 126
 127         if (ret < 0) {
 128                 drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask);
 129                 return ret;
 130         }
 131
 132         return 0;
 133 }
 134
 135 static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
 136                               struct intel_qgv_info *qi)
 137 {
 138         const struct dram_info *dram_info = &dev_priv->dram_info;
 139         int i, ret;
 140
 141         qi->num_points = dram_info->num_qgv_points;
 142         qi->num_psf_points = dram_info->num_psf_gv_points;
 143
 144         if (DISPLAY_VER(dev_priv) == 12)
 145                 switch (dram_info->type) {
 146                 case INTEL_DRAM_DDR4:
 147                         qi->t_bl = 4;
 148                         break;
 149                 case INTEL_DRAM_DDR5:
 150                         qi->t_bl = 8;
 151                         break;
 152                 default:
 153                         qi->t_bl = 16;
 154                         break;
 155                 }
 156         else if (DISPLAY_VER(dev_priv) == 11)
 157                 qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
 158
 159         if (drm_WARN_ON(&dev_priv->drm,
 160                         qi->num_points > ARRAY_SIZE(qi->points)))
 161                 qi->num_points = ARRAY_SIZE(qi->points);
 162
 163         for (i = 0; i < qi->num_points; i++) {
 164                 struct intel_qgv_point *sp = &qi->points[i];
 165
 166                 if (IS_DG1(dev_priv))
 167                         ret = dg1_mchbar_read_qgv_point_info(dev_priv, sp, i);
 168                 else
 169                         ret = icl_pcode_read_qgv_point_info(dev_priv, sp, i);
 170
 171                 if (ret)
 172                         return ret;
 173
 174                 drm_dbg_kms(&dev_priv->drm,
 175                             "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
 176                             i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
 177                             sp->t_rcd, sp->t_rc);
 178         }
 179
 180         if (qi->num_psf_points > 0) {
 181                 ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points);
 182                 if (ret) {
 183                         drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
 184                         qi->num_psf_points = 0;
 185                 }
 186
 187                 for (i = 0; i < qi->num_psf_points; i++)
 188                         drm_dbg_kms(&dev_priv->drm,
 189                                     "PSF GV %d: CLK=%d \n",
 190                                     i, qi->psf_points[i].clk);
 191         }
 192
 193         return 0;
 194 }
 195
 196 static int icl_calc_bw(int dclk, int num, int den)
 197 {
 198         /* multiples of 16.666MHz (100/6) */
 199         return DIV_ROUND_CLOSEST(num * dclk * 100, den * 6);
 200 }
 201
 202 static int adl_calc_psf_bw(int clk)
 203 {
 204         /*
 205          * clk is multiples of 16.666MHz (100/6)
 206          * According to BSpec PSF GV bandwidth is
 207          * calculated as BW = 64 * clk * 16.666Mhz
 208          */
 209         return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
 210 }
 211
 212 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
 213 {
 214         u16 dclk = 0;
 215         int i;
 216
 217         for (i = 0; i < qi->num_points; i++)
 218                 dclk = max(dclk, qi->points[i].dclk);
 219
 220         return dclk;
 221 }
 222
 223 struct intel_sa_info {
 224         u16 displayrtids;
 225         u8 deburst, deprogbwlimit;
 226 };
 227
 228 static const struct intel_sa_info icl_sa_info = {
 229         .deburst = 8,
 230         .deprogbwlimit = 25, /* GB/s */
 231         .displayrtids = 128,
 232 };
 233
 234 static const struct intel_sa_info tgl_sa_info = {
 235         .deburst = 16,
 236         .deprogbwlimit = 34, /* GB/s */
 237         .displayrtids = 256,
 238 };
 239
 240 static const struct intel_sa_info rkl_sa_info = {
 241         .deburst = 16,
 242         .deprogbwlimit = 20, /* GB/s */
 243         .displayrtids = 128,
 244 };
 245
 246 static const struct intel_sa_info adls_sa_info = {
 247         .deburst = 16,
 248         .deprogbwlimit = 38, /* GB/s */
 249         .displayrtids = 256,
 250 };
 251
 252 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
 253 {
 254         struct intel_qgv_info qi = {};
 255         bool is_y_tile = true; /* assume y tile may be used */
 256         int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
 257         int deinterleave;
 258         int ipqdepth, ipqdepthpch;
 259         int dclk_max;
 260         int maxdebw;
 261         int i, ret;
 262
 263         ret = icl_get_qgv_points(dev_priv, &qi);
 264         if (ret) {
 265                 drm_dbg_kms(&dev_priv->drm,
 266                             "Failed to get memory subsystem information, ignoring bandwidth limits");
 267                 return ret;
 268         }
 269
 270         deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 271         dclk_max = icl_sagv_max_dclk(&qi);
 272
 273         ipqdepthpch = 16;
 274
 275         maxdebw = min(sa->deprogbwlimit * 1000,
 276                       icl_calc_bw(dclk_max, 16, 1) * 6 / 10); /* 60% */
 277         ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
 278
 279         for (i = 0; i < ARRAY_SIZE(dev_priv->max_bw); i++) {
 280                 struct intel_bw_info *bi = &dev_priv->max_bw[i];
 281                 int clpchgroup;
 282                 int j;
 283
 284                 clpchgroup = (sa->deburst * deinterleave / num_channels) << i;
 285                 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
 286
 287                 bi->num_qgv_points = qi.num_points;
 288                 bi->num_psf_gv_points = qi.num_psf_points;
 289
 290                 for (j = 0; j < qi.num_points; j++) {
 291                         const struct intel_qgv_point *sp = &qi.points[j];
 292                         int ct, bw;
 293
 294                         /*
 295                          * Max row cycle time
 296                          *
 297                          * FIXME what is the logic behind the
 298                          * assumed burst length?
 299                          */
 300                         ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
 301                                    (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
 302                         bw = icl_calc_bw(sp->dclk, clpchgroup * 32 * num_channels, ct);
 303
 304                         bi->deratedbw[j] = min(maxdebw,
 305                                                bw * 9 / 10); /* 90% */
 306
 307                         drm_dbg_kms(&dev_priv->drm,
 308                                     "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
 309                                     i, j, bi->num_planes, bi->deratedbw[j]);
 310                 }
 311
 312                 for (j = 0; j < qi.num_psf_points; j++) {
 313                         const struct intel_psf_gv_point *sp = &qi.psf_points[j];
 314
 315                         bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
 316
 317                         drm_dbg_kms(&dev_priv->drm,
 318                                     "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
 319                                     i, j, bi->num_planes, bi->psf_bw[j]);
 320                 }
 321
 322                 if (bi->num_planes == 1)
 323                         break;
 324         }
 325
 326         /*
 327          * In case if SAGV is disabled in BIOS, we always get 1
 328          * SAGV point, but we can't send PCode commands to restrict it
 329          * as it will fail and pointless anyway.
 330          */
 331         if (qi.num_points == 1)
 332                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
 333         else
 334                 dev_priv->sagv_status = I915_SAGV_ENABLED;
 335
 336         return 0;
 337 }
 338
 339 static void dg2_get_bw_info(struct drm_i915_private *i915)
 340 {
 341         struct intel_bw_info *bi = &i915->max_bw[0];
 342
 343         /*
 344          * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
 345          * that doesn't depend on the number of planes enabled.  Create a
 346          * single dummy QGV point to reflect that.  DG2-G10 platforms have a
 347          * constant 50 GB/s bandwidth, whereas DG2-G11 platforms have 38 GB/s.
 348          */
 349         bi->num_planes = 1;
 350         bi->num_qgv_points = 1;
 351         if (IS_DG2_G11(i915))
 352                 bi->deratedbw[0] = 38000;
 353         else
 354                 bi->deratedbw[0] = 50000;
 355
 356         i915->sagv_status = I915_SAGV_NOT_CONTROLLED;
 357 }
 358
 359 static unsigned int icl_max_bw(struct drm_i915_private *dev_priv,
 360                                int num_planes, int qgv_point)
 361 {
 362         int i;
 363
 364         /*
 365          * Let's return max bw for 0 planes
 366          */
 367         num_planes = max(1, num_planes);
 368
 369         for (i = 0; i < ARRAY_SIZE(dev_priv->max_bw); i++) {
 370                 const struct intel_bw_info *bi =
 371                         &dev_priv->max_bw[i];
 372
 373                 /*
 374                  * Pcode will not expose all QGV points when
 375                  * SAGV is forced to off/min/med/max.
 376                  */
 377                 if (qgv_point >= bi->num_qgv_points)
 378                         return UINT_MAX;
 379
 380                 if (num_planes >= bi->num_planes)
 381                         return bi->deratedbw[qgv_point];
 382         }
 383
 384         return 0;
 385 }
 386
 387 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
 388                                int psf_gv_point)
 389 {
 390         const struct intel_bw_info *bi =
 391                         &dev_priv->max_bw[0];
 392
 393         return bi->psf_bw[psf_gv_point];
 394 }
 395
 396 void intel_bw_init_hw(struct drm_i915_private *dev_priv)
 397 {
 398         if (!HAS_DISPLAY(dev_priv))
 399                 return;
 400
 401         if (IS_DG2(dev_priv))
 402                 dg2_get_bw_info(dev_priv);
 403         else if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv))
 404                 icl_get_bw_info(dev_priv, &adls_sa_info);
 405         else if (IS_ROCKETLAKE(dev_priv))
 406                 icl_get_bw_info(dev_priv, &rkl_sa_info);
 407         else if (DISPLAY_VER(dev_priv) == 12)
 408                 icl_get_bw_info(dev_priv, &tgl_sa_info);
 409         else if (DISPLAY_VER(dev_priv) == 11)
 410                 icl_get_bw_info(dev_priv, &icl_sa_info);
 411 }
 412
 413 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
 414 {
 415         /*
 416          * We assume cursors are small enough
 417          * to not not cause bandwidth problems.
 418          */
 419         return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
 420 }
 421
 422 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
 423 {
 424         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 425         unsigned int data_rate = 0;
 426         enum plane_id plane_id;
 427
 428         for_each_plane_id_on_crtc(crtc, plane_id) {
 429                 /*
 430                  * We assume cursors are small enough
 431                  * to not not cause bandwidth problems.
 432                  */
 433                 if (plane_id == PLANE_CURSOR)
 434                         continue;
 435
 436                 data_rate += crtc_state->data_rate[plane_id];
 437         }
 438
 439         return data_rate;
 440 }
 441
 442 void intel_bw_crtc_update(struct intel_bw_state *bw_state,
 443                           const struct intel_crtc_state *crtc_state)
 444 {
 445         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 446         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 447
 448         bw_state->data_rate[crtc->pipe] =
 449                 intel_bw_crtc_data_rate(crtc_state);
 450         bw_state->num_active_planes[crtc->pipe] =
 451                 intel_bw_crtc_num_active_planes(crtc_state);
 452
 453         drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
 454                     pipe_name(crtc->pipe),
 455                     bw_state->data_rate[crtc->pipe],
 456                     bw_state->num_active_planes[crtc->pipe]);
 457 }
 458
 459 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv,
 460                                                const struct intel_bw_state *bw_state)
 461 {
 462         unsigned int num_active_planes = 0;
 463         enum pipe pipe;
 464
 465         for_each_pipe(dev_priv, pipe)
 466                 num_active_planes += bw_state->num_active_planes[pipe];
 467
 468         return num_active_planes;
 469 }
 470
 471 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv,
 472                                        const struct intel_bw_state *bw_state)
 473 {
 474         unsigned int data_rate = 0;
 475         enum pipe pipe;
 476
 477         for_each_pipe(dev_priv, pipe)
 478                 data_rate += bw_state->data_rate[pipe];
 479
 480         if (DISPLAY_VER(dev_priv) >= 13 && intel_vtd_active())
 481                 data_rate = data_rate * 105 / 100;
 482
 483         return data_rate;
 484 }
 485
 486 struct intel_bw_state *
 487 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
 488 {
 489         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 490         struct intel_global_state *bw_state;
 491
 492         bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->bw_obj);
 493
 494         return to_intel_bw_state(bw_state);
 495 }
 496
 497 struct intel_bw_state *
 498 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
 499 {
 500         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 501         struct intel_global_state *bw_state;
 502
 503         bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->bw_obj);
 504
 505         return to_intel_bw_state(bw_state);
 506 }
 507
 508 struct intel_bw_state *
 509 intel_atomic_get_bw_state(struct intel_atomic_state *state)
 510 {
 511         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 512         struct intel_global_state *bw_state;
 513
 514         bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->bw_obj);
 515         if (IS_ERR(bw_state))
 516                 return ERR_CAST(bw_state);
 517
 518         return to_intel_bw_state(bw_state);
 519 }
 520
 521 int skl_bw_calc_min_cdclk(struct intel_atomic_state *state)
 522 {
 523         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 524         struct intel_bw_state *new_bw_state = NULL;
 525         struct intel_bw_state *old_bw_state = NULL;
 526         const struct intel_crtc_state *crtc_state;
 527         struct intel_crtc *crtc;
 528         int max_bw = 0;
 529         enum pipe pipe;
 530         int i;
 531
 532         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
 533                 enum plane_id plane_id;
 534                 struct intel_dbuf_bw *crtc_bw;
 535
 536                 new_bw_state = intel_atomic_get_bw_state(state);
 537                 if (IS_ERR(new_bw_state))
 538                         return PTR_ERR(new_bw_state);
 539
 540                 old_bw_state = intel_atomic_get_old_bw_state(state);
 541
 542                 crtc_bw = &new_bw_state->dbuf_bw[crtc->pipe];
 543
 544                 memset(&crtc_bw->used_bw, 0, sizeof(crtc_bw->used_bw));
 545
 546                 if (!crtc_state->hw.active)
 547                         continue;
 548
 549                 for_each_plane_id_on_crtc(crtc, plane_id) {
 550                         const struct skl_ddb_entry *plane_alloc =
 551                                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
 552                         const struct skl_ddb_entry *uv_plane_alloc =
 553                                 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
 554                         unsigned int data_rate = crtc_state->data_rate[plane_id];
 555                         unsigned int dbuf_mask = 0;
 556                         enum dbuf_slice slice;
 557
 558                         dbuf_mask |= skl_ddb_dbuf_slice_mask(dev_priv, plane_alloc);
 559                         dbuf_mask |= skl_ddb_dbuf_slice_mask(dev_priv, uv_plane_alloc);
 560
 561                         /*
 562                          * FIXME: To calculate that more properly we probably
 563                          * need to to split per plane data_rate into data_rate_y
 564                          * and data_rate_uv for multiplanar formats in order not
 565                          * to get accounted those twice if they happen to reside
 566                          * on different slices.
 567                          * However for pre-icl this would work anyway because
 568                          * we have only single slice and for icl+ uv plane has
 569                          * non-zero data rate.
 570                          * So in worst case those calculation are a bit
 571                          * pessimistic, which shouldn't pose any significant
 572                          * problem anyway.
 573                          */
 574                         for_each_dbuf_slice_in_mask(dev_priv, slice, dbuf_mask)
 575                                 crtc_bw->used_bw[slice] += data_rate;
 576                 }
 577         }
 578
 579         if (!old_bw_state)
 580                 return 0;
 581
 582         for_each_pipe(dev_priv, pipe) {
 583                 struct intel_dbuf_bw *crtc_bw;
 584                 enum dbuf_slice slice;
 585
 586                 crtc_bw = &new_bw_state->dbuf_bw[pipe];
 587
 588                 for_each_dbuf_slice(dev_priv, slice) {
 589                         /*
 590                          * Current experimental observations show that contrary
 591                          * to BSpec we get underruns once we exceed 64 * CDCLK
 592                          * for slices in total.
 593                          * As a temporary measure in order not to keep CDCLK
 594                          * bumped up all the time we calculate CDCLK according
 595                          * to this formula for  overall bw consumed by slices.
 596                          */
 597                         max_bw += crtc_bw->used_bw[slice];
 598                 }
 599         }
 600
 601         new_bw_state->min_cdclk = max_bw / 64;
 602
 603         if (new_bw_state->min_cdclk != old_bw_state->min_cdclk) {
 604                 int ret = intel_atomic_lock_global_state(&new_bw_state->base);
 605
 606                 if (ret)
 607                         return ret;
 608         }
 609
 610         return 0;
 611 }
 612
 613 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state)
 614 {
 615         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 616         struct intel_bw_state *new_bw_state = NULL;
 617         struct intel_bw_state *old_bw_state = NULL;
 618         const struct intel_crtc_state *crtc_state;
 619         struct intel_crtc *crtc;
 620         int min_cdclk = 0;
 621         enum pipe pipe;
 622         int i;
 623
 624         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
 625                 new_bw_state = intel_atomic_get_bw_state(state);
 626                 if (IS_ERR(new_bw_state))
 627                         return PTR_ERR(new_bw_state);
 628
 629                 old_bw_state = intel_atomic_get_old_bw_state(state);
 630         }
 631
 632         if (!old_bw_state)
 633                 return 0;
 634
 635         for_each_pipe(dev_priv, pipe) {
 636                 struct intel_cdclk_state *cdclk_state;
 637
 638                 cdclk_state = intel_atomic_get_new_cdclk_state(state);
 639                 if (!cdclk_state)
 640                         return 0;
 641
 642                 min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
 643         }
 644
 645         new_bw_state->min_cdclk = min_cdclk;
 646
 647         if (new_bw_state->min_cdclk != old_bw_state->min_cdclk) {
 648                 int ret = intel_atomic_lock_global_state(&new_bw_state->base);
 649
 650                 if (ret)
 651                         return ret;
 652         }
 653
 654         return 0;
 655 }
 656
 657 int intel_bw_atomic_check(struct intel_atomic_state *state)
 658 {
 659         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 660         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
 661         struct intel_bw_state *new_bw_state = NULL;
 662         const struct intel_bw_state *old_bw_state = NULL;
 663         unsigned int data_rate;
 664         unsigned int num_active_planes;
 665         struct intel_crtc *crtc;
 666         int i, ret;
 667         u32 allowed_points = 0;
 668         unsigned int max_bw_point = 0, max_bw = 0;
 669         unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points;
 670         unsigned int num_psf_gv_points = dev_priv->max_bw[0].num_psf_gv_points;
 671         u32 mask = 0;
 672
 673         /* FIXME earlier gens need some checks too */
 674         if (DISPLAY_VER(dev_priv) < 11)
 675                 return 0;
 676
 677         /*
 678          * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
 679          * it with failure if we try masking any unadvertised points.
 680          * So need to operate only with those returned from PCode.
 681          */
 682         if (num_qgv_points > 0)
 683                 mask |= REG_GENMASK(num_qgv_points - 1, 0);
 684
 685         if (num_psf_gv_points > 0)
 686                 mask |= REG_GENMASK(num_psf_gv_points - 1, 0) << ADLS_PSF_PT_SHIFT;
 687
 688         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
 689                                             new_crtc_state, i) {
 690                 unsigned int old_data_rate =
 691                         intel_bw_crtc_data_rate(old_crtc_state);
 692                 unsigned int new_data_rate =
 693                         intel_bw_crtc_data_rate(new_crtc_state);
 694                 unsigned int old_active_planes =
 695                         intel_bw_crtc_num_active_planes(old_crtc_state);
 696                 unsigned int new_active_planes =
 697                         intel_bw_crtc_num_active_planes(new_crtc_state);
 698
 699                 /*
 700                  * Avoid locking the bw state when
 701                  * nothing significant has changed.
 702                  */
 703                 if (old_data_rate == new_data_rate &&
 704                     old_active_planes == new_active_planes)
 705                         continue;
 706
 707                 new_bw_state = intel_atomic_get_bw_state(state);
 708                 if (IS_ERR(new_bw_state))
 709                         return PTR_ERR(new_bw_state);
 710
 711                 new_bw_state->data_rate[crtc->pipe] = new_data_rate;
 712                 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
 713
 714                 drm_dbg_kms(&dev_priv->drm,
 715                             "pipe %c data rate %u num active planes %u\n",
 716                             pipe_name(crtc->pipe),
 717                             new_bw_state->data_rate[crtc->pipe],
 718                             new_bw_state->num_active_planes[crtc->pipe]);
 719         }
 720
 721         if (!new_bw_state)
 722                 return 0;
 723
 724         ret = intel_atomic_lock_global_state(&new_bw_state->base);
 725         if (ret)
 726                 return ret;
 727
 728         data_rate = intel_bw_data_rate(dev_priv, new_bw_state);
 729         data_rate = DIV_ROUND_UP(data_rate, 1000);
 730
 731         num_active_planes = intel_bw_num_active_planes(dev_priv, new_bw_state);
 732
 733         for (i = 0; i < num_qgv_points; i++) {
 734                 unsigned int max_data_rate;
 735
 736                 max_data_rate = icl_max_bw(dev_priv, num_active_planes, i);
 737                 /*
 738                  * We need to know which qgv point gives us
 739                  * maximum bandwidth in order to disable SAGV
 740                  * if we find that we exceed SAGV block time
 741                  * with watermarks. By that moment we already
 742                  * have those, as it is calculated earlier in
 743                  * intel_atomic_check,
 744                  */
 745                 if (max_data_rate > max_bw) {
 746                         max_bw_point = i;
 747                         max_bw = max_data_rate;
 748                 }
 749                 if (max_data_rate >= data_rate)
 750                         allowed_points |= REG_FIELD_PREP(ADLS_QGV_PT_MASK, BIT(i));
 751
 752                 drm_dbg_kms(&dev_priv->drm, "QGV point %d: max bw %d required %d\n",
 753                             i, max_data_rate, data_rate);
 754         }
 755
 756         for (i = 0; i < num_psf_gv_points; i++) {
 757                 unsigned int max_data_rate = adl_psf_bw(dev_priv, i);
 758
 759                 if (max_data_rate >= data_rate)
 760                         allowed_points |= REG_FIELD_PREP(ADLS_PSF_PT_MASK, BIT(i));
 761
 762                 drm_dbg_kms(&dev_priv->drm, "PSF GV point %d: max bw %d"
 763                             " required %d\n",
 764                             i, max_data_rate, data_rate);
 765         }
 766
 767         /*
 768          * BSpec states that we always should have at least one allowed point
 769          * left, so if we couldn't - simply reject the configuration for obvious
 770          * reasons.
 771          */
 772         if ((allowed_points & ADLS_QGV_PT_MASK) == 0) {
 773                 drm_dbg_kms(&dev_priv->drm, "No QGV points provide sufficient memory"
 774                             " bandwidth %d for display configuration(%d active planes).\n",
 775                             data_rate, num_active_planes);
 776                 return -EINVAL;
 777         }
 778
 779         if (num_psf_gv_points > 0) {
 780                 if ((allowed_points & ADLS_PSF_PT_MASK) == 0) {
 781                         drm_dbg_kms(&dev_priv->drm, "No PSF GV points provide sufficient memory"
 782                                     " bandwidth %d for display configuration(%d active planes).\n",
 783                                     data_rate, num_active_planes);
 784                         return -EINVAL;
 785                 }
 786         }
 787
 788         /*
 789          * Leave only single point with highest bandwidth, if
 790          * we can't enable SAGV due to the increased memory latency it may
 791          * cause.
 792          */
 793         if (!intel_can_enable_sagv(dev_priv, new_bw_state)) {
 794                 allowed_points = BIT(max_bw_point);
 795                 drm_dbg_kms(&dev_priv->drm, "No SAGV, using single QGV point %d\n",
 796                             max_bw_point);
 797         }
 798         /*
 799          * We store the ones which need to be masked as that is what PCode
 800          * actually accepts as a parameter.
 801          */
 802         new_bw_state->qgv_points_mask = ~allowed_points & mask;
 803
 804         old_bw_state = intel_atomic_get_old_bw_state(state);
 805         /*
 806          * If the actual mask had changed we need to make sure that
 807          * the commits are serialized(in case this is a nomodeset, nonblocking)
 808          */
 809         if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
 810                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
 811                 if (ret)
 812                         return ret;
 813         }
 814
 815         return 0;
 816 }
 817
 818 static struct intel_global_state *
 819 intel_bw_duplicate_state(struct intel_global_obj *obj)
 820 {
 821         struct intel_bw_state *state;
 822
 823         state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
 824         if (!state)
 825                 return NULL;
 826
 827         return &state->base;
 828 }
 829
 830 static void intel_bw_destroy_state(struct intel_global_obj *obj,
 831                                    struct intel_global_state *state)
 832 {
 833         kfree(state);
 834 }
 835
 836 static const struct intel_global_state_funcs intel_bw_funcs = {
 837         .atomic_duplicate_state = intel_bw_duplicate_state,
 838         .atomic_destroy_state = intel_bw_destroy_state,
 839 };
 840
 841 int intel_bw_init(struct drm_i915_private *dev_priv)
 842 {
 843         struct intel_bw_state *state;
 844
 845         state = kzalloc(sizeof(*state), GFP_KERNEL);
 846         if (!state)
 847                 return -ENOMEM;
 848
 849         intel_atomic_global_obj_init(dev_priv, &dev_priv->bw_obj,
 850                                      &state->base, &intel_bw_funcs);
 851
 852         return 0;
 853 }