drivers/gpu/drm/vc4/vc4_kms.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2015 Broadcom
   4  */
   5
   6 /**
   7  * DOC: VC4 KMS
   8  *
   9  * This is the general code for implementing KMS mode setting that
  10  * doesn't clearly associate with any of the other objects (plane,
  11  * crtc, HDMI encoder).
  12  */
  13
  14 #include <linux/clk.h>
  15
  16 #include <drm/drm_atomic.h>
  17 #include <drm/drm_atomic_helper.h>
  18 #include <drm/drm_crtc.h>
  19 #include <drm/drm_gem_framebuffer_helper.h>
  20 #include <drm/drm_plane_helper.h>
  21 #include <drm/drm_probe_helper.h>
  22 #include <drm/drm_vblank.h>
  23
  24 #include "vc4_drv.h"
  25 #include "vc4_regs.h"
  26
  27 #define HVS_NUM_CHANNELS 3
  28
  29 struct vc4_ctm_state {
  30         struct drm_private_state base;
  31         struct drm_color_ctm *ctm;
  32         int fifo;
  33 };
  34
  35 static struct vc4_ctm_state *to_vc4_ctm_state(struct drm_private_state *priv)
  36 {
  37         return container_of(priv, struct vc4_ctm_state, base);
  38 }
  39
  40 struct vc4_hvs_state {
  41         struct drm_private_state base;
  42         unsigned long core_clock_rate;
  43
  44         struct {
  45                 unsigned in_use: 1;
  46                 unsigned long fifo_load;
  47                 struct drm_crtc_commit *pending_commit;
  48         } fifo_state[HVS_NUM_CHANNELS];
  49 };
  50
  51 static struct vc4_hvs_state *
  52 to_vc4_hvs_state(struct drm_private_state *priv)
  53 {
  54         return container_of(priv, struct vc4_hvs_state, base);
  55 }
  56
  57 struct vc4_load_tracker_state {
  58         struct drm_private_state base;
  59         u64 hvs_load;
  60         u64 membus_load;
  61 };
  62
  63 static struct vc4_load_tracker_state *
  64 to_vc4_load_tracker_state(struct drm_private_state *priv)
  65 {
  66         return container_of(priv, struct vc4_load_tracker_state, base);
  67 }
  68
  69 static struct vc4_ctm_state *vc4_get_ctm_state(struct drm_atomic_state *state,
  70                                                struct drm_private_obj *manager)
  71 {
  72         struct drm_device *dev = state->dev;
  73         struct vc4_dev *vc4 = to_vc4_dev(dev);
  74         struct drm_private_state *priv_state;
  75         int ret;
  76
  77         ret = drm_modeset_lock(&vc4->ctm_state_lock, state->acquire_ctx);
  78         if (ret)
  79                 return ERR_PTR(ret);
  80
  81         priv_state = drm_atomic_get_private_obj_state(state, manager);
  82         if (IS_ERR(priv_state))
  83                 return ERR_CAST(priv_state);
  84
  85         return to_vc4_ctm_state(priv_state);
  86 }
  87
  88 static struct drm_private_state *
  89 vc4_ctm_duplicate_state(struct drm_private_obj *obj)
  90 {
  91         struct vc4_ctm_state *state;
  92
  93         state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
  94         if (!state)
  95                 return NULL;
  96
  97         __drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
  98
  99         return &state->base;
 100 }
 101
 102 static void vc4_ctm_destroy_state(struct drm_private_obj *obj,
 103                                   struct drm_private_state *state)
 104 {
 105         struct vc4_ctm_state *ctm_state = to_vc4_ctm_state(state);
 106
 107         kfree(ctm_state);
 108 }
 109
 110 static const struct drm_private_state_funcs vc4_ctm_state_funcs = {
 111         .atomic_duplicate_state = vc4_ctm_duplicate_state,
 112         .atomic_destroy_state = vc4_ctm_destroy_state,
 113 };
 114
 115 static void vc4_ctm_obj_fini(struct drm_device *dev, void *unused)
 116 {
 117         struct vc4_dev *vc4 = to_vc4_dev(dev);
 118
 119         drm_atomic_private_obj_fini(&vc4->ctm_manager);
 120 }
 121
 122 static int vc4_ctm_obj_init(struct vc4_dev *vc4)
 123 {
 124         struct vc4_ctm_state *ctm_state;
 125
 126         drm_modeset_lock_init(&vc4->ctm_state_lock);
 127
 128         ctm_state = kzalloc(sizeof(*ctm_state), GFP_KERNEL);
 129         if (!ctm_state)
 130                 return -ENOMEM;
 131
 132         drm_atomic_private_obj_init(&vc4->base, &vc4->ctm_manager, &ctm_state->base,
 133                                     &vc4_ctm_state_funcs);
 134
 135         return drmm_add_action_or_reset(&vc4->base, vc4_ctm_obj_fini, NULL);
 136 }
 137
 138 /* Converts a DRM S31.32 value to the HW S0.9 format. */
 139 static u16 vc4_ctm_s31_32_to_s0_9(u64 in)
 140 {
 141         u16 r;
 142
 143         /* Sign bit. */
 144         r = in & BIT_ULL(63) ? BIT(9) : 0;
 145
 146         if ((in & GENMASK_ULL(62, 32)) > 0) {
 147                 /* We have zero integer bits so we can only saturate here. */
 148                 r |= GENMASK(8, 0);
 149         } else {
 150                 /* Otherwise take the 9 most important fractional bits. */
 151                 r |= (in >> 23) & GENMASK(8, 0);
 152         }
 153
 154         return r;
 155 }
 156
 157 static void
 158 vc4_ctm_commit(struct vc4_dev *vc4, struct drm_atomic_state *state)
 159 {
 160         struct vc4_hvs *hvs = vc4->hvs;
 161         struct vc4_ctm_state *ctm_state = to_vc4_ctm_state(vc4->ctm_manager.state);
 162         struct drm_color_ctm *ctm = ctm_state->ctm;
 163
 164         if (vc4->firmware_kms)
 165                 return;
 166
 167         if (ctm_state->fifo) {
 168                 HVS_WRITE(SCALER_OLEDCOEF2,
 169                           VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[0]),
 170                                         SCALER_OLEDCOEF2_R_TO_R) |
 171                           VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[3]),
 172                                         SCALER_OLEDCOEF2_R_TO_G) |
 173                           VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[6]),
 174                                         SCALER_OLEDCOEF2_R_TO_B));
 175                 HVS_WRITE(SCALER_OLEDCOEF1,
 176                           VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[1]),
 177                                         SCALER_OLEDCOEF1_G_TO_R) |
 178                           VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[4]),
 179                                         SCALER_OLEDCOEF1_G_TO_G) |
 180                           VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[7]),
 181                                         SCALER_OLEDCOEF1_G_TO_B));
 182                 HVS_WRITE(SCALER_OLEDCOEF0,
 183                           VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[2]),
 184                                         SCALER_OLEDCOEF0_B_TO_R) |
 185                           VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[5]),
 186                                         SCALER_OLEDCOEF0_B_TO_G) |
 187                           VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[8]),
 188                                         SCALER_OLEDCOEF0_B_TO_B));
 189         }
 190
 191         HVS_WRITE(SCALER_OLEDOFFS,
 192                   VC4_SET_FIELD(ctm_state->fifo, SCALER_OLEDOFFS_DISPFIFO));
 193 }
 194
 195 static struct vc4_hvs_state *
 196 vc4_hvs_get_new_global_state(struct drm_atomic_state *state)
 197 {
 198         struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 199         struct drm_private_state *priv_state;
 200
 201         priv_state = drm_atomic_get_new_private_obj_state(state, &vc4->hvs_channels);
 202         if (IS_ERR(priv_state))
 203                 return ERR_CAST(priv_state);
 204
 205         return to_vc4_hvs_state(priv_state);
 206 }
 207
 208 static struct vc4_hvs_state *
 209 vc4_hvs_get_old_global_state(struct drm_atomic_state *state)
 210 {
 211         struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 212         struct drm_private_state *priv_state;
 213
 214         priv_state = drm_atomic_get_old_private_obj_state(state, &vc4->hvs_channels);
 215         if (IS_ERR(priv_state))
 216                 return ERR_CAST(priv_state);
 217
 218         return to_vc4_hvs_state(priv_state);
 219 }
 220
 221 static struct vc4_hvs_state *
 222 vc4_hvs_get_global_state(struct drm_atomic_state *state)
 223 {
 224         struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 225         struct drm_private_state *priv_state;
 226
 227         priv_state = drm_atomic_get_private_obj_state(state, &vc4->hvs_channels);
 228         if (IS_ERR(priv_state))
 229                 return ERR_CAST(priv_state);
 230
 231         return to_vc4_hvs_state(priv_state);
 232 }
 233
 234 static void vc4_hvs_pv_muxing_commit(struct vc4_dev *vc4,
 235                                      struct drm_atomic_state *state)
 236 {
 237         struct vc4_hvs *hvs = vc4->hvs;
 238         struct drm_crtc_state *crtc_state;
 239         struct drm_crtc *crtc;
 240         unsigned int i;
 241
 242         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 243                 struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 244                 struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
 245                 u32 dispctrl;
 246                 u32 dsp3_mux;
 247
 248                 if (!crtc_state->active)
 249                         continue;
 250
 251                 if (vc4_state->assigned_channel != 2)
 252                         continue;
 253
 254                 /*
 255                  * SCALER_DISPCTRL_DSP3 = X, where X < 2 means 'connect DSP3 to
 256                  * FIFO X'.
 257                  * SCALER_DISPCTRL_DSP3 = 3 means 'disable DSP 3'.
 258                  *
 259                  * DSP3 is connected to FIFO2 unless the transposer is
 260                  * enabled. In this case, FIFO 2 is directly accessed by the
 261                  * TXP IP, and we need to disable the FIFO2 -> pixelvalve1
 262                  * route.
 263                  */
 264                 if (vc4_crtc->feeds_txp)
 265                         dsp3_mux = VC4_SET_FIELD(3, SCALER_DISPCTRL_DSP3_MUX);
 266                 else
 267                         dsp3_mux = VC4_SET_FIELD(2, SCALER_DISPCTRL_DSP3_MUX);
 268
 269                 dispctrl = HVS_READ(SCALER_DISPCTRL) &
 270                            ~SCALER_DISPCTRL_DSP3_MUX_MASK;
 271                 HVS_WRITE(SCALER_DISPCTRL, dispctrl | dsp3_mux);
 272         }
 273 }
 274
 275 static void vc5_hvs_pv_muxing_commit(struct vc4_dev *vc4,
 276                                      struct drm_atomic_state *state)
 277 {
 278         struct vc4_hvs *hvs = vc4->hvs;
 279         struct drm_crtc_state *crtc_state;
 280         struct drm_crtc *crtc;
 281         unsigned char mux;
 282         unsigned int i;
 283         u32 reg;
 284
 285         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 286                 struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
 287                 struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 288                 unsigned int channel = vc4_state->assigned_channel;
 289
 290                 if (!vc4_state->update_muxing)
 291                         continue;
 292
 293                 switch (vc4_crtc->data->hvs_output) {
 294                 case 2:
 295                         WARN_ON(VC4_GET_FIELD(HVS_READ(SCALER_DISPCTRL),
 296                                               SCALER_DISPCTRL_DSP3_MUX) == channel);
 297
 298                         mux = (channel == 2) ? 0 : 1;
 299                         reg = HVS_READ(SCALER_DISPECTRL);
 300                         HVS_WRITE(SCALER_DISPECTRL,
 301                                   (reg & ~SCALER_DISPECTRL_DSP2_MUX_MASK) |
 302                                   VC4_SET_FIELD(mux, SCALER_DISPECTRL_DSP2_MUX));
 303                         break;
 304
 305                 case 3:
 306                         if (channel == VC4_HVS_CHANNEL_DISABLED)
 307                                 mux = 3;
 308                         else
 309                                 mux = channel;
 310
 311                         reg = HVS_READ(SCALER_DISPCTRL);
 312                         HVS_WRITE(SCALER_DISPCTRL,
 313                                   (reg & ~SCALER_DISPCTRL_DSP3_MUX_MASK) |
 314                                   VC4_SET_FIELD(mux, SCALER_DISPCTRL_DSP3_MUX));
 315                         break;
 316
 317                 case 4:
 318                         if (channel == VC4_HVS_CHANNEL_DISABLED)
 319                                 mux = 3;
 320                         else
 321                                 mux = channel;
 322
 323                         reg = HVS_READ(SCALER_DISPEOLN);
 324                         HVS_WRITE(SCALER_DISPEOLN,
 325                                   (reg & ~SCALER_DISPEOLN_DSP4_MUX_MASK) |
 326                                   VC4_SET_FIELD(mux, SCALER_DISPEOLN_DSP4_MUX));
 327
 328                         break;
 329
 330                 case 5:
 331                         if (channel == VC4_HVS_CHANNEL_DISABLED)
 332                                 mux = 3;
 333                         else
 334                                 mux = channel;
 335
 336                         reg = HVS_READ(SCALER_DISPDITHER);
 337                         HVS_WRITE(SCALER_DISPDITHER,
 338                                   (reg & ~SCALER_DISPDITHER_DSP5_MUX_MASK) |
 339                                   VC4_SET_FIELD(mux, SCALER_DISPDITHER_DSP5_MUX));
 340                         break;
 341
 342                 default:
 343                         break;
 344                 }
 345         }
 346 }
 347
 348 static void vc4_atomic_commit_tail(struct drm_atomic_state *state)
 349 {
 350         struct drm_device *dev = state->dev;
 351         struct vc4_dev *vc4 = to_vc4_dev(dev);
 352         struct vc4_hvs *hvs = vc4->hvs;
 353         struct drm_crtc_state *new_crtc_state;
 354         struct vc4_hvs_state *new_hvs_state;
 355         struct drm_crtc *crtc;
 356         struct vc4_hvs_state *old_hvs_state;
 357         unsigned long max_clock_rate = hvs ? clk_get_max_rate(hvs->core_clk) : 0;
 358         unsigned int channel;
 359         int i;
 360
 361         old_hvs_state = vc4_hvs_get_old_global_state(state);
 362         if (WARN_ON(IS_ERR(old_hvs_state)))
 363                 return;
 364
 365         new_hvs_state = vc4_hvs_get_new_global_state(state);
 366         if (WARN_ON(IS_ERR(new_hvs_state)))
 367                 return;
 368
 369         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
 370                 struct vc4_crtc_state *vc4_crtc_state;
 371
 372                 if (!new_crtc_state->commit || vc4->firmware_kms)
 373                         continue;
 374
 375                 vc4_crtc_state = to_vc4_crtc_state(new_crtc_state);
 376                 vc4_hvs_mask_underrun(hvs, vc4_crtc_state->assigned_channel);
 377         }
 378
 379         for (channel = 0; channel < HVS_NUM_CHANNELS; channel++) {
 380                 struct drm_crtc_commit *commit;
 381                 int ret;
 382
 383                 if (!old_hvs_state->fifo_state[channel].in_use)
 384                         continue;
 385
 386                 commit = old_hvs_state->fifo_state[channel].pending_commit;
 387                 if (!commit)
 388                         continue;
 389
 390                 ret = drm_crtc_commit_wait(commit);
 391                 if (ret)
 392                         drm_err(dev, "Timed out waiting for commit\n");
 393
 394                 drm_crtc_commit_put(commit);
 395                 old_hvs_state->fifo_state[channel].pending_commit = NULL;
 396         }
 397
 398         if (vc4->is_vc5 && !vc4->firmware_kms) {
 399                 unsigned long state_rate = max(old_hvs_state->core_clock_rate,
 400                                                new_hvs_state->core_clock_rate);
 401                 unsigned long core_rate = clamp_t(unsigned long, state_rate,
 402                                                   500000000, max_clock_rate);
 403
 404                 WARN_ON(clk_set_min_rate(hvs->core_clk, core_rate));
 405         }
 406
 407         drm_atomic_helper_commit_modeset_disables(dev, state);
 408
 409         vc4_ctm_commit(vc4, state);
 410
 411         if (!vc4->firmware_kms) {
 412                 if (vc4->is_vc5)
 413                         vc5_hvs_pv_muxing_commit(vc4, state);
 414                 else
 415                         vc4_hvs_pv_muxing_commit(vc4, state);
 416         }
 417
 418         drm_atomic_helper_commit_planes(dev, state, 0);
 419
 420         drm_atomic_helper_commit_modeset_enables(dev, state);
 421
 422         drm_atomic_helper_fake_vblank(state);
 423
 424         drm_atomic_helper_commit_hw_done(state);
 425
 426         drm_atomic_helper_wait_for_flip_done(dev, state);
 427
 428         drm_atomic_helper_cleanup_planes(dev, state);
 429
 430         if (vc4->is_vc5 && !vc4->firmware_kms) {
 431                 unsigned long core_rate = min_t(unsigned long,
 432                                                max_clock_rate,
 433                                                new_hvs_state->core_clock_rate);
 434
 435                 drm_dbg(dev, "Running the core clock at %lu Hz\n", core_rate);
 436
 437                 WARN_ON(clk_set_min_rate(hvs->core_clk, core_rate));
 438
 439                 drm_dbg(dev, "Core clock actual rate: %lu Hz\n",
 440                         clk_get_rate(hvs->core_clk));
 441         }
 442 }
 443
 444 static int vc4_atomic_commit_setup(struct drm_atomic_state *state)
 445 {
 446         struct drm_device *dev = state->dev;
 447         struct vc4_dev *vc4 = to_vc4_dev(dev);
 448         struct drm_crtc_state *crtc_state;
 449         struct vc4_hvs_state *hvs_state;
 450         struct drm_crtc *crtc;
 451         unsigned int i;
 452
 453         /* We know for sure we don't want an async update here. Set
 454          * state->legacy_cursor_update to false to prevent
 455          * drm_atomic_helper_setup_commit() from auto-completing
 456          * commit->flip_done.
 457          */
 458         if (!vc4->firmware_kms)
 459                 state->legacy_cursor_update = false;
 460
 461         hvs_state = vc4_hvs_get_new_global_state(state);
 462         if (WARN_ON(IS_ERR(hvs_state)))
 463                 return PTR_ERR(hvs_state);
 464
 465         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 466                 struct vc4_crtc_state *vc4_crtc_state =
 467                         to_vc4_crtc_state(crtc_state);
 468                 unsigned int channel =
 469                         vc4_crtc_state->assigned_channel;
 470
 471                 if (channel == VC4_HVS_CHANNEL_DISABLED)
 472                         continue;
 473
 474                 if (!hvs_state->fifo_state[channel].in_use)
 475                         continue;
 476
 477                 hvs_state->fifo_state[channel].pending_commit =
 478                         drm_crtc_commit_get(crtc_state->commit);
 479         }
 480
 481         return 0;
 482 }
 483
 484 static struct drm_framebuffer *vc4_fb_create(struct drm_device *dev,
 485                                              struct drm_file *file_priv,
 486                                              const struct drm_mode_fb_cmd2 *mode_cmd)
 487 {
 488         struct vc4_dev *vc4 = to_vc4_dev(dev);
 489         struct drm_mode_fb_cmd2 mode_cmd_local;
 490
 491         if (WARN_ON_ONCE(vc4->is_vc5))
 492                 return ERR_PTR(-ENODEV);
 493
 494         /* If the user didn't specify a modifier, use the
 495          * vc4_set_tiling_ioctl() state for the BO.
 496          */
 497         if (!(mode_cmd->flags & DRM_MODE_FB_MODIFIERS)) {
 498                 struct drm_gem_object *gem_obj;
 499                 struct vc4_bo *bo;
 500
 501                 gem_obj = drm_gem_object_lookup(file_priv,
 502                                                 mode_cmd->handles[0]);
 503                 if (!gem_obj) {
 504                         DRM_DEBUG("Failed to look up GEM BO %d\n",
 505                                   mode_cmd->handles[0]);
 506                         return ERR_PTR(-ENOENT);
 507                 }
 508                 bo = to_vc4_bo(gem_obj);
 509
 510                 mode_cmd_local = *mode_cmd;
 511
 512                 if (bo->t_format) {
 513                         mode_cmd_local.modifier[0] =
 514                                 DRM_FORMAT_MOD_BROADCOM_VC4_T_TILED;
 515                 } else {
 516                         mode_cmd_local.modifier[0] = DRM_FORMAT_MOD_NONE;
 517                 }
 518
 519                 drm_gem_object_put(gem_obj);
 520
 521                 mode_cmd = &mode_cmd_local;
 522         }
 523
 524         return drm_gem_fb_create(dev, file_priv, mode_cmd);
 525 }
 526
 527 /* Our CTM has some peculiar limitations: we can only enable it for one CRTC
 528  * at a time and the HW only supports S0.9 scalars. To account for the latter,
 529  * we don't allow userland to set a CTM that we have no hope of approximating.
 530  */
 531 static int
 532 vc4_ctm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 533 {
 534         struct vc4_dev *vc4 = to_vc4_dev(dev);
 535         struct vc4_ctm_state *ctm_state = NULL;
 536         struct drm_crtc *crtc;
 537         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 538         struct drm_color_ctm *ctm;
 539         int i;
 540
 541         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 542                 /* CTM is being disabled. */
 543                 if (!new_crtc_state->ctm && old_crtc_state->ctm) {
 544                         ctm_state = vc4_get_ctm_state(state, &vc4->ctm_manager);
 545                         if (IS_ERR(ctm_state))
 546                                 return PTR_ERR(ctm_state);
 547                         ctm_state->fifo = 0;
 548                 }
 549         }
 550
 551         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 552                 if (new_crtc_state->ctm == old_crtc_state->ctm)
 553                         continue;
 554
 555                 if (!ctm_state) {
 556                         ctm_state = vc4_get_ctm_state(state, &vc4->ctm_manager);
 557                         if (IS_ERR(ctm_state))
 558                                 return PTR_ERR(ctm_state);
 559                 }
 560
 561                 /* CTM is being enabled or the matrix changed. */
 562                 if (new_crtc_state->ctm) {
 563                         struct vc4_crtc_state *vc4_crtc_state =
 564                                 to_vc4_crtc_state(new_crtc_state);
 565
 566                         /* fifo is 1-based since 0 disables CTM. */
 567                         int fifo = vc4_crtc_state->assigned_channel + 1;
 568
 569                         /* Check userland isn't trying to turn on CTM for more
 570                          * than one CRTC at a time.
 571                          */
 572                         if (ctm_state->fifo && ctm_state->fifo != fifo) {
 573                                 DRM_DEBUG_DRIVER("Too many CTM configured\n");
 574                                 return -EINVAL;
 575                         }
 576
 577                         /* Check we can approximate the specified CTM.
 578                          * We disallow scalars |c| > 1.0 since the HW has
 579                          * no integer bits.
 580                          */
 581                         ctm = new_crtc_state->ctm->data;
 582                         for (i = 0; i < ARRAY_SIZE(ctm->matrix); i++) {
 583                                 u64 val = ctm->matrix[i];
 584
 585                                 val &= ~BIT_ULL(63);
 586                                 if (val > BIT_ULL(32))
 587                                         return -EINVAL;
 588                         }
 589
 590                         ctm_state->fifo = fifo;
 591                         ctm_state->ctm = ctm;
 592                 }
 593         }
 594
 595         return 0;
 596 }
 597
 598 static int vc4_load_tracker_atomic_check(struct drm_atomic_state *state)
 599 {
 600         struct drm_plane_state *old_plane_state, *new_plane_state;
 601         struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 602         struct vc4_load_tracker_state *load_state;
 603         struct drm_private_state *priv_state;
 604         struct drm_plane *plane;
 605         int i;
 606
 607         priv_state = drm_atomic_get_private_obj_state(state,
 608                                                       &vc4->load_tracker);
 609         if (IS_ERR(priv_state))
 610                 return PTR_ERR(priv_state);
 611
 612         load_state = to_vc4_load_tracker_state(priv_state);
 613         for_each_oldnew_plane_in_state(state, plane, old_plane_state,
 614                                        new_plane_state, i) {
 615                 struct vc4_plane_state *vc4_plane_state;
 616
 617                 if (old_plane_state->fb && old_plane_state->crtc) {
 618                         vc4_plane_state = to_vc4_plane_state(old_plane_state);
 619                         load_state->membus_load -= vc4_plane_state->membus_load;
 620                         load_state->hvs_load -= vc4_plane_state->hvs_load;
 621                 }
 622
 623                 if (new_plane_state->fb && new_plane_state->crtc) {
 624                         vc4_plane_state = to_vc4_plane_state(new_plane_state);
 625                         load_state->membus_load += vc4_plane_state->membus_load;
 626                         load_state->hvs_load += vc4_plane_state->hvs_load;
 627                 }
 628         }
 629
 630         /* Don't check the load when the tracker is disabled. */
 631         if (!vc4->load_tracker_enabled)
 632                 return 0;
 633
 634         /* The absolute limit is 2Gbyte/sec, but let's take a margin to let
 635          * the system work when other blocks are accessing the memory.
 636          */
 637         if (load_state->membus_load > SZ_1G + SZ_512M)
 638                 return -ENOSPC;
 639
 640         /* HVS clock is supposed to run @ 250Mhz, let's take a margin and
 641          * consider the maximum number of cycles is 240M.
 642          */
 643         if (load_state->hvs_load > 240000000ULL)
 644                 return -ENOSPC;
 645
 646         return 0;
 647 }
 648
 649 static struct drm_private_state *
 650 vc4_load_tracker_duplicate_state(struct drm_private_obj *obj)
 651 {
 652         struct vc4_load_tracker_state *state;
 653
 654         state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
 655         if (!state)
 656                 return NULL;
 657
 658         __drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 659
 660         return &state->base;
 661 }
 662
 663 static void vc4_load_tracker_destroy_state(struct drm_private_obj *obj,
 664                                            struct drm_private_state *state)
 665 {
 666         struct vc4_load_tracker_state *load_state;
 667
 668         load_state = to_vc4_load_tracker_state(state);
 669         kfree(load_state);
 670 }
 671
 672 static const struct drm_private_state_funcs vc4_load_tracker_state_funcs = {
 673         .atomic_duplicate_state = vc4_load_tracker_duplicate_state,
 674         .atomic_destroy_state = vc4_load_tracker_destroy_state,
 675 };
 676
 677 static void vc4_load_tracker_obj_fini(struct drm_device *dev, void *unused)
 678 {
 679         struct vc4_dev *vc4 = to_vc4_dev(dev);
 680
 681         drm_atomic_private_obj_fini(&vc4->load_tracker);
 682 }
 683
 684 static int vc4_load_tracker_obj_init(struct vc4_dev *vc4)
 685 {
 686         struct vc4_load_tracker_state *load_state;
 687
 688         load_state = kzalloc(sizeof(*load_state), GFP_KERNEL);
 689         if (!load_state)
 690                 return -ENOMEM;
 691
 692         drm_atomic_private_obj_init(&vc4->base, &vc4->load_tracker,
 693                                     &load_state->base,
 694                                     &vc4_load_tracker_state_funcs);
 695
 696         return drmm_add_action_or_reset(&vc4->base, vc4_load_tracker_obj_fini, NULL);
 697 }
 698
 699 static struct drm_private_state *
 700 vc4_hvs_channels_duplicate_state(struct drm_private_obj *obj)
 701 {
 702         struct vc4_hvs_state *old_state = to_vc4_hvs_state(obj->state);
 703         struct vc4_hvs_state *state;
 704         unsigned int i;
 705
 706         state = kzalloc(sizeof(*state), GFP_KERNEL);
 707         if (!state)
 708                 return NULL;
 709
 710         __drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 711
 712         for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 713                 state->fifo_state[i].in_use = old_state->fifo_state[i].in_use;
 714                 state->fifo_state[i].fifo_load = old_state->fifo_state[i].fifo_load;
 715         }
 716
 717         state->core_clock_rate = old_state->core_clock_rate;
 718
 719         return &state->base;
 720 }
 721
 722 static void vc4_hvs_channels_destroy_state(struct drm_private_obj *obj,
 723                                            struct drm_private_state *state)
 724 {
 725         struct vc4_hvs_state *hvs_state = to_vc4_hvs_state(state);
 726         unsigned int i;
 727
 728         for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 729                 if (!hvs_state->fifo_state[i].pending_commit)
 730                         continue;
 731
 732                 drm_crtc_commit_put(hvs_state->fifo_state[i].pending_commit);
 733         }
 734
 735         kfree(hvs_state);
 736 }
 737
 738 static const struct drm_private_state_funcs vc4_hvs_state_funcs = {
 739         .atomic_duplicate_state = vc4_hvs_channels_duplicate_state,
 740         .atomic_destroy_state = vc4_hvs_channels_destroy_state,
 741 };
 742
 743 static void vc4_hvs_channels_obj_fini(struct drm_device *dev, void *unused)
 744 {
 745         struct vc4_dev *vc4 = to_vc4_dev(dev);
 746
 747         drm_atomic_private_obj_fini(&vc4->hvs_channels);
 748 }
 749
 750 static int vc4_hvs_channels_obj_init(struct vc4_dev *vc4)
 751 {
 752         struct vc4_hvs_state *state;
 753
 754         state = kzalloc(sizeof(*state), GFP_KERNEL);
 755         if (!state)
 756                 return -ENOMEM;
 757
 758         drm_atomic_private_obj_init(&vc4->base, &vc4->hvs_channels,
 759                                     &state->base,
 760                                     &vc4_hvs_state_funcs);
 761
 762         return drmm_add_action_or_reset(&vc4->base, vc4_hvs_channels_obj_fini, NULL);
 763 }
 764
 765 /*
 766  * The BCM2711 HVS has up to 7 outputs connected to the pixelvalves and
 767  * the TXP (and therefore all the CRTCs found on that platform).
 768  *
 769  * The naive (and our initial) implementation would just iterate over
 770  * all the active CRTCs, try to find a suitable FIFO, and then remove it
 771  * from the pool of available FIFOs. However, there are a few corner
 772  * cases that need to be considered:
 773  *
 774  * - When running in a dual-display setup (so with two CRTCs involved),
 775  *   we can update the state of a single CRTC (for example by changing
 776  *   its mode using xrandr under X11) without affecting the other. In
 777  *   this case, the other CRTC wouldn't be in the state at all, so we
 778  *   need to consider all the running CRTCs in the DRM device to assign
 779  *   a FIFO, not just the one in the state.
 780  *
 781  * - To fix the above, we can't use drm_atomic_get_crtc_state on all
 782  *   enabled CRTCs to pull their CRTC state into the global state, since
 783  *   a page flip would start considering their vblank to complete. Since
 784  *   we don't have a guarantee that they are actually active, that
 785  *   vblank might never happen, and shouldn't even be considered if we
 786  *   want to do a page flip on a single CRTC. That can be tested by
 787  *   doing a modetest -v first on HDMI1 and then on HDMI0.
 788  *
 789  * - Since we need the pixelvalve to be disabled and enabled back when
 790  *   the FIFO is changed, we should keep the FIFO assigned for as long
 791  *   as the CRTC is enabled, only considering it free again once that
 792  *   CRTC has been disabled. This can be tested by booting X11 on a
 793  *   single display, and changing the resolution down and then back up.
 794  */
 795 static int vc4_pv_muxing_atomic_check(struct drm_device *dev,
 796                                       struct drm_atomic_state *state)
 797 {
 798         struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 799         struct vc4_hvs_state *hvs_new_state;
 800         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 801         struct drm_crtc *crtc;
 802         unsigned int unassigned_channels = 0;
 803         unsigned int i;
 804
 805         hvs_new_state = vc4_hvs_get_global_state(state);
 806         if (IS_ERR(hvs_new_state))
 807                 return PTR_ERR(hvs_new_state);
 808
 809         for (i = 0; i < ARRAY_SIZE(hvs_new_state->fifo_state); i++)
 810                 if (!hvs_new_state->fifo_state[i].in_use)
 811                         unassigned_channels |= BIT(i);
 812
 813         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 814                 struct vc4_crtc_state *old_vc4_crtc_state =
 815                         to_vc4_crtc_state(old_crtc_state);
 816                 struct vc4_crtc_state *new_vc4_crtc_state =
 817                         to_vc4_crtc_state(new_crtc_state);
 818                 struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 819                 unsigned int matching_channels;
 820                 unsigned int channel;
 821
 822                 if (vc4->firmware_kms)
 823                         continue;
 824
 825                 drm_dbg(dev, "%s: Trying to find a channel.\n", crtc->name);
 826
 827                 /* Nothing to do here, let's skip it */
 828                 if (old_crtc_state->enable == new_crtc_state->enable) {
 829                         if (new_crtc_state->enable)
 830                                 drm_dbg(dev, "%s: Already enabled, reusing channel %d.\n",
 831                                         crtc->name, new_vc4_crtc_state->assigned_channel);
 832                         else
 833                                 drm_dbg(dev, "%s: Disabled, ignoring.\n", crtc->name);
 834
 835                         continue;
 836                 }
 837
 838                 /* Muxing will need to be modified, mark it as such */
 839                 new_vc4_crtc_state->update_muxing = true;
 840
 841                 /* If we're disabling our CRTC, we put back our channel */
 842                 if (!new_crtc_state->enable) {
 843                         channel = old_vc4_crtc_state->assigned_channel;
 844
 845                         drm_dbg(dev, "%s: Disabling, Freeing channel %d\n",
 846                                 crtc->name, channel);
 847
 848                         hvs_new_state->fifo_state[channel].in_use = false;
 849                         new_vc4_crtc_state->assigned_channel = VC4_HVS_CHANNEL_DISABLED;
 850                         continue;
 851                 }
 852
 853                 /*
 854                  * The problem we have to solve here is that we have
 855                  * up to 7 encoders, connected to up to 6 CRTCs.
 856                  *
 857                  * Those CRTCs, depending on the instance, can be
 858                  * routed to 1, 2 or 3 HVS FIFOs, and we need to set
 859                  * the change the muxing between FIFOs and outputs in
 860                  * the HVS accordingly.
 861                  *
 862                  * It would be pretty hard to come up with an
 863                  * algorithm that would generically solve
 864                  * this. However, the current routing trees we support
 865                  * allow us to simplify a bit the problem.
 866                  *
 867                  * Indeed, with the current supported layouts, if we
 868                  * try to assign in the ascending crtc index order the
 869                  * FIFOs, we can't fall into the situation where an
 870                  * earlier CRTC that had multiple routes is assigned
 871                  * one that was the only option for a later CRTC.
 872                  *
 873                  * If the layout changes and doesn't give us that in
 874                  * the future, we will need to have something smarter,
 875                  * but it works so far.
 876                  */
 877                 matching_channels = unassigned_channels & vc4_crtc->data->hvs_available_channels;
 878                 if (!matching_channels)
 879                         return -EINVAL;
 880
 881                 channel = ffs(matching_channels) - 1;
 882
 883                 drm_dbg(dev, "Assigned HVS channel %d to CRTC %s\n", channel, crtc->name);
 884                 new_vc4_crtc_state->assigned_channel = channel;
 885                 unassigned_channels &= ~BIT(channel);
 886                 hvs_new_state->fifo_state[channel].in_use = true;
 887         }
 888
 889         return 0;
 890 }
 891
 892 static int
 893 vc4_core_clock_atomic_check(struct drm_atomic_state *state)
 894 {
 895         struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 896         struct drm_private_state *priv_state;
 897         struct vc4_hvs_state *hvs_new_state;
 898         struct vc4_load_tracker_state *load_state;
 899         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 900         struct drm_crtc *crtc;
 901         unsigned int num_outputs;
 902         unsigned long pixel_rate;
 903         unsigned long cob_rate;
 904         unsigned int i;
 905
 906         priv_state = drm_atomic_get_private_obj_state(state,
 907                                                       &vc4->load_tracker);
 908         if (IS_ERR(priv_state))
 909                 return PTR_ERR(priv_state);
 910
 911         load_state = to_vc4_load_tracker_state(priv_state);
 912
 913         hvs_new_state = vc4_hvs_get_global_state(state);
 914         if (IS_ERR(hvs_new_state))
 915                 return PTR_ERR(hvs_new_state);
 916
 917         for_each_oldnew_crtc_in_state(state, crtc,
 918                                       old_crtc_state,
 919                                       new_crtc_state,
 920                                       i) {
 921                 if (old_crtc_state->active) {
 922                         struct vc4_crtc_state *old_vc4_state =
 923                                 to_vc4_crtc_state(old_crtc_state);
 924                         unsigned int channel = old_vc4_state->assigned_channel;
 925
 926                         hvs_new_state->fifo_state[channel].fifo_load = 0;
 927                 }
 928
 929                 if (new_crtc_state->active) {
 930                         struct vc4_crtc_state *new_vc4_state =
 931                                 to_vc4_crtc_state(new_crtc_state);
 932                         unsigned int channel = new_vc4_state->assigned_channel;
 933
 934                         hvs_new_state->fifo_state[channel].fifo_load =
 935                                 new_vc4_state->hvs_load;
 936                 }
 937         }
 938
 939         cob_rate = 0;
 940         num_outputs = 0;
 941         for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 942                 if (!hvs_new_state->fifo_state[i].in_use)
 943                         continue;
 944
 945                 num_outputs++;
 946                 cob_rate = max_t(unsigned long,
 947                                  hvs_new_state->fifo_state[i].fifo_load,
 948                                  cob_rate);
 949         }
 950
 951         pixel_rate = load_state->hvs_load;
 952         if (num_outputs > 1) {
 953                 pixel_rate = (pixel_rate * 40) / 100;
 954         } else {
 955                 pixel_rate = (pixel_rate * 60) / 100;
 956         }
 957
 958         hvs_new_state->core_clock_rate = max(cob_rate, pixel_rate);
 959
 960         return 0;
 961 }
 962
 963
 964 static int
 965 vc4_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 966 {
 967         int ret;
 968
 969         ret = vc4_pv_muxing_atomic_check(dev, state);
 970         if (ret)
 971                 return ret;
 972
 973         ret = vc4_ctm_atomic_check(dev, state);
 974         if (ret < 0)
 975                 return ret;
 976
 977         ret = drm_atomic_helper_check(dev, state);
 978         if (ret)
 979                 return ret;
 980
 981         ret = vc4_load_tracker_atomic_check(state);
 982         if (ret)
 983                 return ret;
 984
 985         return vc4_core_clock_atomic_check(state);
 986 }
 987
 988 static struct drm_mode_config_helper_funcs vc4_mode_config_helpers = {
 989         .atomic_commit_setup    = vc4_atomic_commit_setup,
 990         .atomic_commit_tail     = vc4_atomic_commit_tail,
 991 };
 992
 993 static const struct drm_mode_config_funcs vc4_mode_funcs = {
 994         .atomic_check = vc4_atomic_check,
 995         .atomic_commit = drm_atomic_helper_commit,
 996         .fb_create = vc4_fb_create,
 997 };
 998
 999 static const struct drm_mode_config_funcs vc5_mode_funcs = {
1000         .atomic_check = vc4_atomic_check,
1001         .atomic_commit = drm_atomic_helper_commit,
1002         .fb_create = drm_gem_fb_create,
1003 };
1004
1005 int vc4_kms_load(struct drm_device *dev)
1006 {
1007         struct vc4_dev *vc4 = to_vc4_dev(dev);
1008         int ret;
1009
1010         /*
1011          * The limits enforced by the load tracker aren't relevant for
1012          * the BCM2711, but the load tracker computations are used for
1013          * the core clock rate calculation.
1014          */
1015         if (!vc4->is_vc5) {
1016                 /* Start with the load tracker enabled. Can be
1017                  * disabled through the debugfs load_tracker file.
1018                  */
1019                 vc4->load_tracker_enabled = true;
1020         }
1021
1022         /* Set support for vblank irq fast disable, before drm_vblank_init() */
1023         dev->vblank_disable_immediate = true;
1024
1025         ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
1026         if (ret < 0) {
1027                 dev_err(dev->dev, "failed to initialize vblank\n");
1028                 return ret;
1029         }
1030
1031         if (vc4->is_vc5) {
1032                 dev->mode_config.max_width = 7680;
1033                 dev->mode_config.max_height = 7680;
1034         } else {
1035                 dev->mode_config.max_width = 2048;
1036                 dev->mode_config.max_height = 2048;
1037         }
1038
1039         dev->mode_config.funcs = vc4->is_vc5 ? &vc5_mode_funcs : &vc4_mode_funcs;
1040         dev->mode_config.helper_private = &vc4_mode_config_helpers;
1041         dev->mode_config.preferred_depth = 24;
1042         dev->mode_config.async_page_flip = true;
1043         dev->mode_config.normalize_zpos = true;
1044
1045         ret = vc4_ctm_obj_init(vc4);
1046         if (ret)
1047                 return ret;
1048
1049         ret = vc4_load_tracker_obj_init(vc4);
1050         if (ret)
1051                 return ret;
1052
1053         ret = vc4_hvs_channels_obj_init(vc4);
1054         if (ret)
1055                 return ret;
1056
1057         drm_mode_config_reset(dev);
1058
1059         drm_kms_helper_poll_init(dev);
1060
1061         return 0;
1062 }