drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
   4  * Copyright (c) 2014-2021 The Linux Foundation. All rights reserved.
   5  * Copyright (C) 2013 Red Hat
   6  * Author: Rob Clark <robdclark@gmail.com>
   7  */
   8
   9 #define pr_fmt(fmt)     "[drm:%s:%d] " fmt, __func__, __LINE__
  10 #include <linux/sort.h>
  11 #include <linux/debugfs.h>
  12 #include <linux/ktime.h>
  13 #include <linux/bits.h>
  14
  15 #include <drm/drm_atomic.h>
  16 #include <drm/drm_blend.h>
  17 #include <drm/drm_crtc.h>
  18 #include <drm/drm_flip_work.h>
  19 #include <drm/drm_framebuffer.h>
  20 #include <drm/drm_mode.h>
  21 #include <drm/drm_probe_helper.h>
  22 #include <drm/drm_rect.h>
  23 #include <drm/drm_vblank.h>
  24 #include <drm/drm_self_refresh_helper.h>
  25
  26 #include "dpu_kms.h"
  27 #include "dpu_hw_lm.h"
  28 #include "dpu_hw_ctl.h"
  29 #include "dpu_hw_dspp.h"
  30 #include "dpu_crtc.h"
  31 #include "dpu_plane.h"
  32 #include "dpu_encoder.h"
  33 #include "dpu_vbif.h"
  34 #include "dpu_core_perf.h"
  35 #include "dpu_trace.h"
  36
  37 /* layer mixer index on dpu_crtc */
  38 #define LEFT_MIXER 0
  39 #define RIGHT_MIXER 1
  40
  41 /* timeout in ms waiting for frame done */
  42 #define DPU_CRTC_FRAME_DONE_TIMEOUT_MS  60
  43
  44 #define CONVERT_S3_15(val) \
  45         (((((u64)val) & ~BIT_ULL(63)) >> 17) & GENMASK_ULL(17, 0))
  46
  47 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
  48 {
  49         struct msm_drm_private *priv = crtc->dev->dev_private;
  50
  51         return to_dpu_kms(priv->kms);
  52 }
  53
  54 static void dpu_crtc_destroy(struct drm_crtc *crtc)
  55 {
  56         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
  57
  58         if (!crtc)
  59                 return;
  60
  61         drm_crtc_cleanup(crtc);
  62         kfree(dpu_crtc);
  63 }
  64
  65 static struct drm_encoder *get_encoder_from_crtc(struct drm_crtc *crtc)
  66 {
  67         struct drm_device *dev = crtc->dev;
  68         struct drm_encoder *encoder;
  69
  70         drm_for_each_encoder(encoder, dev)
  71                 if (encoder->crtc == crtc)
  72                         return encoder;
  73
  74         return NULL;
  75 }
  76
  77 static enum dpu_crtc_crc_source dpu_crtc_parse_crc_source(const char *src_name)
  78 {
  79         if (!src_name ||
  80             !strcmp(src_name, "none"))
  81                 return DPU_CRTC_CRC_SOURCE_NONE;
  82         if (!strcmp(src_name, "auto") ||
  83             !strcmp(src_name, "lm"))
  84                 return DPU_CRTC_CRC_SOURCE_LAYER_MIXER;
  85         if (!strcmp(src_name, "encoder"))
  86                 return DPU_CRTC_CRC_SOURCE_ENCODER;
  87
  88         return DPU_CRTC_CRC_SOURCE_INVALID;
  89 }
  90
  91 static int dpu_crtc_verify_crc_source(struct drm_crtc *crtc,
  92                 const char *src_name, size_t *values_cnt)
  93 {
  94         enum dpu_crtc_crc_source source = dpu_crtc_parse_crc_source(src_name);
  95         struct dpu_crtc_state *crtc_state = to_dpu_crtc_state(crtc->state);
  96
  97         if (source < 0) {
  98                 DRM_DEBUG_DRIVER("Invalid source %s for CRTC%d\n", src_name, crtc->index);
  99                 return -EINVAL;
 100         }
 101
 102         if (source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER) {
 103                 *values_cnt = crtc_state->num_mixers;
 104         } else if (source == DPU_CRTC_CRC_SOURCE_ENCODER) {
 105                 struct drm_encoder *drm_enc;
 106
 107                 *values_cnt = 0;
 108
 109                 drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask)
 110                         *values_cnt += dpu_encoder_get_crc_values_cnt(drm_enc);
 111         }
 112
 113         return 0;
 114 }
 115
 116 static void dpu_crtc_setup_lm_misr(struct dpu_crtc_state *crtc_state)
 117 {
 118         struct dpu_crtc_mixer *m;
 119         int i;
 120
 121         for (i = 0; i < crtc_state->num_mixers; ++i) {
 122                 m = &crtc_state->mixers[i];
 123
 124                 if (!m->hw_lm || !m->hw_lm->ops.setup_misr)
 125                         continue;
 126
 127                 /* Calculate MISR over 1 frame */
 128                 m->hw_lm->ops.setup_misr(m->hw_lm, true, 1);
 129         }
 130 }
 131
 132 static void dpu_crtc_setup_encoder_misr(struct drm_crtc *crtc)
 133 {
 134         struct drm_encoder *drm_enc;
 135
 136         drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask)
 137                 dpu_encoder_setup_misr(drm_enc);
 138 }
 139
 140 static int dpu_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
 141 {
 142         enum dpu_crtc_crc_source source = dpu_crtc_parse_crc_source(src_name);
 143         enum dpu_crtc_crc_source current_source;
 144         struct dpu_crtc_state *crtc_state;
 145         struct drm_device *drm_dev = crtc->dev;
 146
 147         bool was_enabled;
 148         bool enable = false;
 149         int ret = 0;
 150
 151         if (source < 0) {
 152                 DRM_DEBUG_DRIVER("Invalid CRC source %s for CRTC%d\n", src_name, crtc->index);
 153                 return -EINVAL;
 154         }
 155
 156         ret = drm_modeset_lock(&crtc->mutex, NULL);
 157
 158         if (ret)
 159                 return ret;
 160
 161         enable = (source != DPU_CRTC_CRC_SOURCE_NONE);
 162         crtc_state = to_dpu_crtc_state(crtc->state);
 163
 164         spin_lock_irq(&drm_dev->event_lock);
 165         current_source = crtc_state->crc_source;
 166         spin_unlock_irq(&drm_dev->event_lock);
 167
 168         was_enabled = (current_source != DPU_CRTC_CRC_SOURCE_NONE);
 169
 170         if (!was_enabled && enable) {
 171                 ret = drm_crtc_vblank_get(crtc);
 172
 173                 if (ret)
 174                         goto cleanup;
 175
 176         } else if (was_enabled && !enable) {
 177                 drm_crtc_vblank_put(crtc);
 178         }
 179
 180         spin_lock_irq(&drm_dev->event_lock);
 181         crtc_state->crc_source = source;
 182         spin_unlock_irq(&drm_dev->event_lock);
 183
 184         crtc_state->crc_frame_skip_count = 0;
 185
 186         if (source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER)
 187                 dpu_crtc_setup_lm_misr(crtc_state);
 188         else if (source == DPU_CRTC_CRC_SOURCE_ENCODER)
 189                 dpu_crtc_setup_encoder_misr(crtc);
 190         else
 191                 ret = -EINVAL;
 192
 193 cleanup:
 194         drm_modeset_unlock(&crtc->mutex);
 195
 196         return ret;
 197 }
 198
 199 static u32 dpu_crtc_get_vblank_counter(struct drm_crtc *crtc)
 200 {
 201         struct drm_encoder *encoder = get_encoder_from_crtc(crtc);
 202         if (!encoder) {
 203                 DRM_ERROR("no encoder found for crtc %d\n", crtc->index);
 204                 return 0;
 205         }
 206
 207         return dpu_encoder_get_vsync_count(encoder);
 208 }
 209
 210 static int dpu_crtc_get_lm_crc(struct drm_crtc *crtc,
 211                 struct dpu_crtc_state *crtc_state)
 212 {
 213         struct dpu_crtc_mixer *m;
 214         u32 crcs[CRTC_DUAL_MIXERS];
 215
 216         int rc = 0;
 217         int i;
 218
 219         BUILD_BUG_ON(ARRAY_SIZE(crcs) != ARRAY_SIZE(crtc_state->mixers));
 220
 221         for (i = 0; i < crtc_state->num_mixers; ++i) {
 222
 223                 m = &crtc_state->mixers[i];
 224
 225                 if (!m->hw_lm || !m->hw_lm->ops.collect_misr)
 226                         continue;
 227
 228                 rc = m->hw_lm->ops.collect_misr(m->hw_lm, &crcs[i]);
 229
 230                 if (rc) {
 231                         if (rc != -ENODATA)
 232                                 DRM_DEBUG_DRIVER("MISR read failed\n");
 233                         return rc;
 234                 }
 235         }
 236
 237         return drm_crtc_add_crc_entry(crtc, true,
 238                         drm_crtc_accurate_vblank_count(crtc), crcs);
 239 }
 240
 241 static int dpu_crtc_get_encoder_crc(struct drm_crtc *crtc)
 242 {
 243         struct drm_encoder *drm_enc;
 244         int rc, pos = 0;
 245         u32 crcs[INTF_MAX];
 246
 247         drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask) {
 248                 rc = dpu_encoder_get_crc(drm_enc, crcs, pos);
 249                 if (rc < 0) {
 250                         if (rc != -ENODATA)
 251                                 DRM_DEBUG_DRIVER("MISR read failed\n");
 252
 253                         return rc;
 254                 }
 255
 256                 pos += rc;
 257         }
 258
 259         return drm_crtc_add_crc_entry(crtc, true,
 260                         drm_crtc_accurate_vblank_count(crtc), crcs);
 261 }
 262
 263 static int dpu_crtc_get_crc(struct drm_crtc *crtc)
 264 {
 265         struct dpu_crtc_state *crtc_state = to_dpu_crtc_state(crtc->state);
 266
 267         /* Skip first 2 frames in case of "uncooked" CRCs */
 268         if (crtc_state->crc_frame_skip_count < 2) {
 269                 crtc_state->crc_frame_skip_count++;
 270                 return 0;
 271         }
 272
 273         if (crtc_state->crc_source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER)
 274                 return dpu_crtc_get_lm_crc(crtc, crtc_state);
 275         else if (crtc_state->crc_source == DPU_CRTC_CRC_SOURCE_ENCODER)
 276                 return dpu_crtc_get_encoder_crc(crtc);
 277
 278         return -EINVAL;
 279 }
 280
 281 static bool dpu_crtc_get_scanout_position(struct drm_crtc *crtc,
 282                                            bool in_vblank_irq,
 283                                            int *vpos, int *hpos,
 284                                            ktime_t *stime, ktime_t *etime,
 285                                            const struct drm_display_mode *mode)
 286 {
 287         unsigned int pipe = crtc->index;
 288         struct drm_encoder *encoder;
 289         int line, vsw, vbp, vactive_start, vactive_end, vfp_end;
 290
 291         encoder = get_encoder_from_crtc(crtc);
 292         if (!encoder) {
 293                 DRM_ERROR("no encoder found for crtc %d\n", pipe);
 294                 return false;
 295         }
 296
 297         vsw = mode->crtc_vsync_end - mode->crtc_vsync_start;
 298         vbp = mode->crtc_vtotal - mode->crtc_vsync_end;
 299
 300         /*
 301          * the line counter is 1 at the start of the VSYNC pulse and VTOTAL at
 302          * the end of VFP. Translate the porch values relative to the line
 303          * counter positions.
 304          */
 305
 306         vactive_start = vsw + vbp + 1;
 307         vactive_end = vactive_start + mode->crtc_vdisplay;
 308
 309         /* last scan line before VSYNC */
 310         vfp_end = mode->crtc_vtotal;
 311
 312         if (stime)
 313                 *stime = ktime_get();
 314
 315         line = dpu_encoder_get_linecount(encoder);
 316
 317         if (line < vactive_start)
 318                 line -= vactive_start;
 319         else if (line > vactive_end)
 320                 line = line - vfp_end - vactive_start;
 321         else
 322                 line -= vactive_start;
 323
 324         *vpos = line;
 325         *hpos = 0;
 326
 327         if (etime)
 328                 *etime = ktime_get();
 329
 330         return true;
 331 }
 332
 333 static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer,
 334                 struct dpu_plane_state *pstate, struct dpu_format *format)
 335 {
 336         struct dpu_hw_mixer *lm = mixer->hw_lm;
 337         uint32_t blend_op;
 338         uint32_t fg_alpha, bg_alpha;
 339
 340         fg_alpha = pstate->base.alpha >> 8;
 341         bg_alpha = 0xff - fg_alpha;
 342
 343         /* default to opaque blending */
 344         if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PIXEL_NONE ||
 345             !format->alpha_enable) {
 346                 blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
 347                         DPU_BLEND_BG_ALPHA_BG_CONST;
 348         } else if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PREMULTI) {
 349                 blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
 350                         DPU_BLEND_BG_ALPHA_FG_PIXEL;
 351                 if (fg_alpha != 0xff) {
 352                         bg_alpha = fg_alpha;
 353                         blend_op |= DPU_BLEND_BG_MOD_ALPHA |
 354                                     DPU_BLEND_BG_INV_MOD_ALPHA;
 355                 } else {
 356                         blend_op |= DPU_BLEND_BG_INV_ALPHA;
 357                 }
 358         } else {
 359                 /* coverage blending */
 360                 blend_op = DPU_BLEND_FG_ALPHA_FG_PIXEL |
 361                         DPU_BLEND_BG_ALPHA_FG_PIXEL;
 362                 if (fg_alpha != 0xff) {
 363                         bg_alpha = fg_alpha;
 364                         blend_op |= DPU_BLEND_FG_MOD_ALPHA |
 365                                     DPU_BLEND_FG_INV_MOD_ALPHA |
 366                                     DPU_BLEND_BG_MOD_ALPHA |
 367                                     DPU_BLEND_BG_INV_MOD_ALPHA;
 368                 } else {
 369                         blend_op |= DPU_BLEND_BG_INV_ALPHA;
 370                 }
 371         }
 372
 373         lm->ops.setup_blend_config(lm, pstate->stage,
 374                                 fg_alpha, bg_alpha, blend_op);
 375
 376         DRM_DEBUG_ATOMIC("format:%p4cc, alpha_en:%u blend_op:0x%x\n",
 377                   &format->base.pixel_format, format->alpha_enable, blend_op);
 378 }
 379
 380 static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc)
 381 {
 382         struct dpu_crtc_state *crtc_state;
 383         int lm_idx, lm_horiz_position;
 384
 385         crtc_state = to_dpu_crtc_state(crtc->state);
 386
 387         lm_horiz_position = 0;
 388         for (lm_idx = 0; lm_idx < crtc_state->num_mixers; lm_idx++) {
 389                 const struct drm_rect *lm_roi = &crtc_state->lm_bounds[lm_idx];
 390                 struct dpu_hw_mixer *hw_lm = crtc_state->mixers[lm_idx].hw_lm;
 391                 struct dpu_hw_mixer_cfg cfg;
 392
 393                 if (!lm_roi || !drm_rect_visible(lm_roi))
 394                         continue;
 395
 396                 cfg.out_width = drm_rect_width(lm_roi);
 397                 cfg.out_height = drm_rect_height(lm_roi);
 398                 cfg.right_mixer = lm_horiz_position++;
 399                 cfg.flags = 0;
 400                 hw_lm->ops.setup_mixer_out(hw_lm, &cfg);
 401         }
 402 }
 403
 404 static void _dpu_crtc_blend_setup_pipe(struct drm_crtc *crtc,
 405                                        struct drm_plane *plane,
 406                                        struct dpu_crtc_mixer *mixer,
 407                                        u32 num_mixers,
 408                                        enum dpu_stage stage,
 409                                        struct dpu_format *format,
 410                                        uint64_t modifier,
 411                                        struct dpu_sw_pipe *pipe,
 412                                        unsigned int stage_idx,
 413                                        struct dpu_hw_stage_cfg *stage_cfg
 414                                       )
 415 {
 416         uint32_t lm_idx;
 417         enum dpu_sspp sspp_idx;
 418         struct drm_plane_state *state;
 419
 420         sspp_idx = pipe->sspp->idx;
 421
 422         state = plane->state;
 423
 424         trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
 425                                    state, to_dpu_plane_state(state), stage_idx,
 426                                    format->base.pixel_format,
 427                                    modifier);
 428
 429         DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d multirect_idx %d\n",
 430                          crtc->base.id,
 431                          stage,
 432                          plane->base.id,
 433                          sspp_idx - SSPP_NONE,
 434                          state->fb ? state->fb->base.id : -1,
 435                          pipe->multirect_index);
 436
 437         stage_cfg->stage[stage][stage_idx] = sspp_idx;
 438         stage_cfg->multirect_index[stage][stage_idx] = pipe->multirect_index;
 439
 440         /* blend config update */
 441         for (lm_idx = 0; lm_idx < num_mixers; lm_idx++)
 442                 mixer[lm_idx].lm_ctl->ops.update_pending_flush_sspp(mixer[lm_idx].lm_ctl, sspp_idx);
 443 }
 444
 445 static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 446         struct dpu_crtc *dpu_crtc, struct dpu_crtc_mixer *mixer,
 447         struct dpu_hw_stage_cfg *stage_cfg)
 448 {
 449         struct drm_plane *plane;
 450         struct drm_framebuffer *fb;
 451         struct drm_plane_state *state;
 452         struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 453         struct dpu_plane_state *pstate = NULL;
 454         struct dpu_format *format;
 455         struct dpu_hw_ctl *ctl = mixer->lm_ctl;
 456
 457         uint32_t lm_idx;
 458         bool bg_alpha_enable = false;
 459         DECLARE_BITMAP(fetch_active, SSPP_MAX);
 460
 461         memset(fetch_active, 0, sizeof(fetch_active));
 462         drm_atomic_crtc_for_each_plane(plane, crtc) {
 463                 state = plane->state;
 464                 if (!state)
 465                         continue;
 466
 467                 if (!state->visible)
 468                         continue;
 469
 470                 pstate = to_dpu_plane_state(state);
 471                 fb = state->fb;
 472
 473                 format = to_dpu_format(msm_framebuffer_format(pstate->base.fb));
 474
 475                 if (pstate->stage == DPU_STAGE_BASE && format->alpha_enable)
 476                         bg_alpha_enable = true;
 477
 478                 set_bit(pstate->pipe.sspp->idx, fetch_active);
 479                 _dpu_crtc_blend_setup_pipe(crtc, plane,
 480                                            mixer, cstate->num_mixers,
 481                                            pstate->stage,
 482                                            format, fb ? fb->modifier : 0,
 483                                            &pstate->pipe, 0, stage_cfg);
 484
 485                 if (pstate->r_pipe.sspp) {
 486                         set_bit(pstate->r_pipe.sspp->idx, fetch_active);
 487                         _dpu_crtc_blend_setup_pipe(crtc, plane,
 488                                                    mixer, cstate->num_mixers,
 489                                                    pstate->stage,
 490                                                    format, fb ? fb->modifier : 0,
 491                                                    &pstate->r_pipe, 1, stage_cfg);
 492                 }
 493
 494                 /* blend config update */
 495                 for (lm_idx = 0; lm_idx < cstate->num_mixers; lm_idx++) {
 496                         _dpu_crtc_setup_blend_cfg(mixer + lm_idx, pstate, format);
 497
 498                         if (bg_alpha_enable && !format->alpha_enable)
 499                                 mixer[lm_idx].mixer_op_mode = 0;
 500                         else
 501                                 mixer[lm_idx].mixer_op_mode |=
 502                                                 1 << pstate->stage;
 503                 }
 504         }
 505
 506         if (ctl->ops.set_active_pipes)
 507                 ctl->ops.set_active_pipes(ctl, fetch_active);
 508
 509         _dpu_crtc_program_lm_output_roi(crtc);
 510 }
 511
 512 /**
 513  * _dpu_crtc_blend_setup - configure crtc mixers
 514  * @crtc: Pointer to drm crtc structure
 515  */
 516 static void _dpu_crtc_blend_setup(struct drm_crtc *crtc)
 517 {
 518         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 519         struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 520         struct dpu_crtc_mixer *mixer = cstate->mixers;
 521         struct dpu_hw_ctl *ctl;
 522         struct dpu_hw_mixer *lm;
 523         struct dpu_hw_stage_cfg stage_cfg;
 524         int i;
 525
 526         DRM_DEBUG_ATOMIC("%s\n", dpu_crtc->name);
 527
 528         for (i = 0; i < cstate->num_mixers; i++) {
 529                 mixer[i].mixer_op_mode = 0;
 530                 if (mixer[i].lm_ctl->ops.clear_all_blendstages)
 531                         mixer[i].lm_ctl->ops.clear_all_blendstages(
 532                                         mixer[i].lm_ctl);
 533         }
 534
 535         /* initialize stage cfg */
 536         memset(&stage_cfg, 0, sizeof(struct dpu_hw_stage_cfg));
 537
 538         _dpu_crtc_blend_setup_mixer(crtc, dpu_crtc, mixer, &stage_cfg);
 539
 540         for (i = 0; i < cstate->num_mixers; i++) {
 541                 ctl = mixer[i].lm_ctl;
 542                 lm = mixer[i].hw_lm;
 543
 544                 lm->ops.setup_alpha_out(lm, mixer[i].mixer_op_mode);
 545
 546                 /* stage config flush mask */
 547                 ctl->ops.update_pending_flush_mixer(ctl,
 548                         mixer[i].hw_lm->idx);
 549
 550                 DRM_DEBUG_ATOMIC("lm %d, op_mode 0x%X, ctl %d\n",
 551                         mixer[i].hw_lm->idx - LM_0,
 552                         mixer[i].mixer_op_mode,
 553                         ctl->idx - CTL_0);
 554
 555                 ctl->ops.setup_blendstage(ctl, mixer[i].hw_lm->idx,
 556                         &stage_cfg);
 557         }
 558 }
 559
 560 /**
 561  *  _dpu_crtc_complete_flip - signal pending page_flip events
 562  * Any pending vblank events are added to the vblank_event_list
 563  * so that the next vblank interrupt shall signal them.
 564  * However PAGE_FLIP events are not handled through the vblank_event_list.
 565  * This API signals any pending PAGE_FLIP events requested through
 566  * DRM_IOCTL_MODE_PAGE_FLIP and are cached in the dpu_crtc->event.
 567  * @crtc: Pointer to drm crtc structure
 568  */
 569 static void _dpu_crtc_complete_flip(struct drm_crtc *crtc)
 570 {
 571         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 572         struct drm_device *dev = crtc->dev;
 573         unsigned long flags;
 574
 575         spin_lock_irqsave(&dev->event_lock, flags);
 576         if (dpu_crtc->event) {
 577                 DRM_DEBUG_VBL("%s: send event: %pK\n", dpu_crtc->name,
 578                               dpu_crtc->event);
 579                 trace_dpu_crtc_complete_flip(DRMID(crtc));
 580                 drm_crtc_send_vblank_event(crtc, dpu_crtc->event);
 581                 dpu_crtc->event = NULL;
 582         }
 583         spin_unlock_irqrestore(&dev->event_lock, flags);
 584 }
 585
 586 enum dpu_intf_mode dpu_crtc_get_intf_mode(struct drm_crtc *crtc)
 587 {
 588         struct drm_encoder *encoder;
 589
 590         /*
 591          * TODO: This function is called from dpu debugfs and as part of atomic
 592          * check. When called from debugfs, the crtc->mutex must be held to
 593          * read crtc->state. However reading crtc->state from atomic check isn't
 594          * allowed (unless you have a good reason, a big comment, and a deep
 595          * understanding of how the atomic/modeset locks work (<- and this is
 596          * probably not possible)). So we'll keep the WARN_ON here for now, but
 597          * really we need to figure out a better way to track our operating mode
 598          */
 599         WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
 600
 601         /* TODO: Returns the first INTF_MODE, could there be multiple values? */
 602         drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
 603                 return dpu_encoder_get_intf_mode(encoder);
 604
 605         return INTF_MODE_NONE;
 606 }
 607
 608 void dpu_crtc_vblank_callback(struct drm_crtc *crtc)
 609 {
 610         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 611
 612         /* keep statistics on vblank callback - with auto reset via debugfs */
 613         if (ktime_compare(dpu_crtc->vblank_cb_time, ktime_set(0, 0)) == 0)
 614                 dpu_crtc->vblank_cb_time = ktime_get();
 615         else
 616                 dpu_crtc->vblank_cb_count++;
 617
 618         dpu_crtc_get_crc(crtc);
 619
 620         drm_crtc_handle_vblank(crtc);
 621         trace_dpu_crtc_vblank_cb(DRMID(crtc));
 622 }
 623
 624 static void dpu_crtc_frame_event_work(struct kthread_work *work)
 625 {
 626         struct dpu_crtc_frame_event *fevent = container_of(work,
 627                         struct dpu_crtc_frame_event, work);
 628         struct drm_crtc *crtc = fevent->crtc;
 629         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 630         unsigned long flags;
 631         bool frame_done = false;
 632
 633         DPU_ATRACE_BEGIN("crtc_frame_event");
 634
 635         DRM_DEBUG_ATOMIC("crtc%d event:%u ts:%lld\n", crtc->base.id, fevent->event,
 636                         ktime_to_ns(fevent->ts));
 637
 638         if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
 639                                 | DPU_ENCODER_FRAME_EVENT_ERROR
 640                                 | DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)) {
 641
 642                 if (atomic_read(&dpu_crtc->frame_pending) < 1) {
 643                         /* ignore vblank when not pending */
 644                 } else if (atomic_dec_return(&dpu_crtc->frame_pending) == 0) {
 645                         /* release bandwidth and other resources */
 646                         trace_dpu_crtc_frame_event_done(DRMID(crtc),
 647                                                         fevent->event);
 648                         dpu_core_perf_crtc_release_bw(crtc);
 649                 } else {
 650                         trace_dpu_crtc_frame_event_more_pending(DRMID(crtc),
 651                                                                 fevent->event);
 652                 }
 653
 654                 if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
 655                                         | DPU_ENCODER_FRAME_EVENT_ERROR))
 656                         frame_done = true;
 657         }
 658
 659         if (fevent->event & DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)
 660                 DPU_ERROR("crtc%d ts:%lld received panel dead event\n",
 661                                 crtc->base.id, ktime_to_ns(fevent->ts));
 662
 663         if (frame_done)
 664                 complete_all(&dpu_crtc->frame_done_comp);
 665
 666         spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
 667         list_add_tail(&fevent->list, &dpu_crtc->frame_event_list);
 668         spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
 669         DPU_ATRACE_END("crtc_frame_event");
 670 }
 671
 672 /*
 673  * dpu_crtc_frame_event_cb - crtc frame event callback API. CRTC module
 674  * registers this API to encoder for all frame event callbacks like
 675  * frame_error, frame_done, idle_timeout, etc. Encoder may call different events
 676  * from different context - IRQ, user thread, commit_thread, etc. Each event
 677  * should be carefully reviewed and should be processed in proper task context
 678  * to avoid schedulin delay or properly manage the irq context's bottom half
 679  * processing.
 680  */
 681 static void dpu_crtc_frame_event_cb(void *data, u32 event)
 682 {
 683         struct drm_crtc *crtc = (struct drm_crtc *)data;
 684         struct dpu_crtc *dpu_crtc;
 685         struct msm_drm_private *priv;
 686         struct dpu_crtc_frame_event *fevent;
 687         unsigned long flags;
 688         u32 crtc_id;
 689
 690         /* Nothing to do on idle event */
 691         if (event & DPU_ENCODER_FRAME_EVENT_IDLE)
 692                 return;
 693
 694         dpu_crtc = to_dpu_crtc(crtc);
 695         priv = crtc->dev->dev_private;
 696         crtc_id = drm_crtc_index(crtc);
 697
 698         trace_dpu_crtc_frame_event_cb(DRMID(crtc), event);
 699
 700         spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
 701         fevent = list_first_entry_or_null(&dpu_crtc->frame_event_list,
 702                         struct dpu_crtc_frame_event, list);
 703         if (fevent)
 704                 list_del_init(&fevent->list);
 705         spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
 706
 707         if (!fevent) {
 708                 DRM_ERROR_RATELIMITED("crtc%d event %d overflow\n", crtc->base.id, event);
 709                 return;
 710         }
 711
 712         fevent->event = event;
 713         fevent->crtc = crtc;
 714         fevent->ts = ktime_get();
 715         kthread_queue_work(priv->event_thread[crtc_id].worker, &fevent->work);
 716 }
 717
 718 void dpu_crtc_complete_commit(struct drm_crtc *crtc)
 719 {
 720         trace_dpu_crtc_complete_commit(DRMID(crtc));
 721         dpu_core_perf_crtc_update(crtc, 0, false);
 722         _dpu_crtc_complete_flip(crtc);
 723 }
 724
 725 static void _dpu_crtc_setup_lm_bounds(struct drm_crtc *crtc,
 726                 struct drm_crtc_state *state)
 727 {
 728         struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
 729         struct drm_display_mode *adj_mode = &state->adjusted_mode;
 730         u32 crtc_split_width = adj_mode->hdisplay / cstate->num_mixers;
 731         int i;
 732
 733         for (i = 0; i < cstate->num_mixers; i++) {
 734                 struct drm_rect *r = &cstate->lm_bounds[i];
 735                 r->x1 = crtc_split_width * i;
 736                 r->y1 = 0;
 737                 r->x2 = r->x1 + crtc_split_width;
 738                 r->y2 = adj_mode->vdisplay;
 739
 740                 trace_dpu_crtc_setup_lm_bounds(DRMID(crtc), i, r);
 741         }
 742 }
 743
 744 static void _dpu_crtc_get_pcc_coeff(struct drm_crtc_state *state,
 745                 struct dpu_hw_pcc_cfg *cfg)
 746 {
 747         struct drm_color_ctm *ctm;
 748
 749         memset(cfg, 0, sizeof(struct dpu_hw_pcc_cfg));
 750
 751         ctm = (struct drm_color_ctm *)state->ctm->data;
 752
 753         if (!ctm)
 754                 return;
 755
 756         cfg->r.r = CONVERT_S3_15(ctm->matrix[0]);
 757         cfg->g.r = CONVERT_S3_15(ctm->matrix[1]);
 758         cfg->b.r = CONVERT_S3_15(ctm->matrix[2]);
 759
 760         cfg->r.g = CONVERT_S3_15(ctm->matrix[3]);
 761         cfg->g.g = CONVERT_S3_15(ctm->matrix[4]);
 762         cfg->b.g = CONVERT_S3_15(ctm->matrix[5]);
 763
 764         cfg->r.b = CONVERT_S3_15(ctm->matrix[6]);
 765         cfg->g.b = CONVERT_S3_15(ctm->matrix[7]);
 766         cfg->b.b = CONVERT_S3_15(ctm->matrix[8]);
 767 }
 768
 769 static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc)
 770 {
 771         struct drm_crtc_state *state = crtc->state;
 772         struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 773         struct dpu_crtc_mixer *mixer = cstate->mixers;
 774         struct dpu_hw_pcc_cfg cfg;
 775         struct dpu_hw_ctl *ctl;
 776         struct dpu_hw_dspp *dspp;
 777         int i;
 778
 779
 780         if (!state->color_mgmt_changed && !drm_atomic_crtc_needs_modeset(state))
 781                 return;
 782
 783         for (i = 0; i < cstate->num_mixers; i++) {
 784                 ctl = mixer[i].lm_ctl;
 785                 dspp = mixer[i].hw_dspp;
 786
 787                 if (!dspp || !dspp->ops.setup_pcc)
 788                         continue;
 789
 790                 if (!state->ctm) {
 791                         dspp->ops.setup_pcc(dspp, NULL);
 792                 } else {
 793                         _dpu_crtc_get_pcc_coeff(state, &cfg);
 794                         dspp->ops.setup_pcc(dspp, &cfg);
 795                 }
 796
 797                 /* stage config flush mask */
 798                 ctl->ops.update_pending_flush_dspp(ctl,
 799                         mixer[i].hw_dspp->idx, DPU_DSPP_PCC);
 800         }
 801 }
 802
 803 static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
 804                 struct drm_atomic_state *state)
 805 {
 806         struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 807         struct drm_encoder *encoder;
 808
 809         if (!crtc->state->enable) {
 810                 DRM_DEBUG_ATOMIC("crtc%d -> enable %d, skip atomic_begin\n",
 811                                 crtc->base.id, crtc->state->enable);
 812                 return;
 813         }
 814
 815         DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
 816
 817         _dpu_crtc_setup_lm_bounds(crtc, crtc->state);
 818
 819         /* encoder will trigger pending mask now */
 820         drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
 821                 dpu_encoder_trigger_kickoff_pending(encoder);
 822
 823         /*
 824          * If no mixers have been allocated in dpu_crtc_atomic_check(),
 825          * it means we are trying to flush a CRTC whose state is disabled:
 826          * nothing else needs to be done.
 827          */
 828         if (unlikely(!cstate->num_mixers))
 829                 return;
 830
 831         _dpu_crtc_blend_setup(crtc);
 832
 833         _dpu_crtc_setup_cp_blocks(crtc);
 834
 835         /*
 836          * PP_DONE irq is only used by command mode for now.
 837          * It is better to request pending before FLUSH and START trigger
 838          * to make sure no pp_done irq missed.
 839          * This is safe because no pp_done will happen before SW trigger
 840          * in command mode.
 841          */
 842 }
 843
 844 static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
 845                 struct drm_atomic_state *state)
 846 {
 847         struct dpu_crtc *dpu_crtc;
 848         struct drm_device *dev;
 849         struct drm_plane *plane;
 850         struct msm_drm_private *priv;
 851         unsigned long flags;
 852         struct dpu_crtc_state *cstate;
 853
 854         if (!crtc->state->enable) {
 855                 DRM_DEBUG_ATOMIC("crtc%d -> enable %d, skip atomic_flush\n",
 856                                 crtc->base.id, crtc->state->enable);
 857                 return;
 858         }
 859
 860         DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
 861
 862         dpu_crtc = to_dpu_crtc(crtc);
 863         cstate = to_dpu_crtc_state(crtc->state);
 864         dev = crtc->dev;
 865         priv = dev->dev_private;
 866
 867         if (crtc->index >= ARRAY_SIZE(priv->event_thread)) {
 868                 DPU_ERROR("invalid crtc index[%d]\n", crtc->index);
 869                 return;
 870         }
 871
 872         WARN_ON(dpu_crtc->event);
 873         spin_lock_irqsave(&dev->event_lock, flags);
 874         dpu_crtc->event = crtc->state->event;
 875         crtc->state->event = NULL;
 876         spin_unlock_irqrestore(&dev->event_lock, flags);
 877
 878         /*
 879          * If no mixers has been allocated in dpu_crtc_atomic_check(),
 880          * it means we are trying to flush a CRTC whose state is disabled:
 881          * nothing else needs to be done.
 882          */
 883         if (unlikely(!cstate->num_mixers))
 884                 return;
 885
 886         /* update performance setting before crtc kickoff */
 887         dpu_core_perf_crtc_update(crtc, 1, false);
 888
 889         /*
 890          * Final plane updates: Give each plane a chance to complete all
 891          *                      required writes/flushing before crtc's "flush
 892          *                      everything" call below.
 893          */
 894         drm_atomic_crtc_for_each_plane(plane, crtc) {
 895                 if (dpu_crtc->smmu_state.transition_error)
 896                         dpu_plane_set_error(plane, true);
 897                 dpu_plane_flush(plane);
 898         }
 899
 900         /* Kickoff will be scheduled by outer layer */
 901 }
 902
 903 /**
 904  * dpu_crtc_destroy_state - state destroy hook
 905  * @crtc: drm CRTC
 906  * @state: CRTC state object to release
 907  */
 908 static void dpu_crtc_destroy_state(struct drm_crtc *crtc,
 909                 struct drm_crtc_state *state)
 910 {
 911         struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
 912
 913         DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
 914
 915         __drm_atomic_helper_crtc_destroy_state(state);
 916
 917         kfree(cstate);
 918 }
 919
 920 static int _dpu_crtc_wait_for_frame_done(struct drm_crtc *crtc)
 921 {
 922         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 923         int ret, rc = 0;
 924
 925         if (!atomic_read(&dpu_crtc->frame_pending)) {
 926                 DRM_DEBUG_ATOMIC("no frames pending\n");
 927                 return 0;
 928         }
 929
 930         DPU_ATRACE_BEGIN("frame done completion wait");
 931         ret = wait_for_completion_timeout(&dpu_crtc->frame_done_comp,
 932                         msecs_to_jiffies(DPU_CRTC_FRAME_DONE_TIMEOUT_MS));
 933         if (!ret) {
 934                 DRM_ERROR("frame done wait timed out, ret:%d\n", ret);
 935                 rc = -ETIMEDOUT;
 936         }
 937         DPU_ATRACE_END("frame done completion wait");
 938
 939         return rc;
 940 }
 941
 942 void dpu_crtc_commit_kickoff(struct drm_crtc *crtc)
 943 {
 944         struct drm_encoder *encoder;
 945         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 946         struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
 947         struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 948
 949         /*
 950          * If no mixers has been allocated in dpu_crtc_atomic_check(),
 951          * it means we are trying to start a CRTC whose state is disabled:
 952          * nothing else needs to be done.
 953          */
 954         if (unlikely(!cstate->num_mixers))
 955                 return;
 956
 957         DPU_ATRACE_BEGIN("crtc_commit");
 958
 959         drm_for_each_encoder_mask(encoder, crtc->dev,
 960                         crtc->state->encoder_mask) {
 961                 if (!dpu_encoder_is_valid_for_commit(encoder)) {
 962                         DRM_DEBUG_ATOMIC("invalid FB not kicking off crtc\n");
 963                         goto end;
 964                 }
 965         }
 966         /*
 967          * Encoder will flush/start now, unless it has a tx pending. If so, it
 968          * may delay and flush at an irq event (e.g. ppdone)
 969          */
 970         drm_for_each_encoder_mask(encoder, crtc->dev,
 971                                   crtc->state->encoder_mask)
 972                 dpu_encoder_prepare_for_kickoff(encoder);
 973
 974         if (atomic_inc_return(&dpu_crtc->frame_pending) == 1) {
 975                 /* acquire bandwidth and other resources */
 976                 DRM_DEBUG_ATOMIC("crtc%d first commit\n", crtc->base.id);
 977         } else
 978                 DRM_DEBUG_ATOMIC("crtc%d commit\n", crtc->base.id);
 979
 980         dpu_crtc->play_count++;
 981
 982         dpu_vbif_clear_errors(dpu_kms);
 983
 984         drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
 985                 dpu_encoder_kickoff(encoder);
 986
 987         reinit_completion(&dpu_crtc->frame_done_comp);
 988
 989 end:
 990         DPU_ATRACE_END("crtc_commit");
 991 }
 992
 993 static void dpu_crtc_reset(struct drm_crtc *crtc)
 994 {
 995         struct dpu_crtc_state *cstate = kzalloc(sizeof(*cstate), GFP_KERNEL);
 996
 997         if (crtc->state)
 998                 dpu_crtc_destroy_state(crtc, crtc->state);
 999
1000         if (cstate)
1001                 __drm_atomic_helper_crtc_reset(crtc, &cstate->base);
1002         else
1003                 __drm_atomic_helper_crtc_reset(crtc, NULL);
1004 }
1005
1006 /**
1007  * dpu_crtc_duplicate_state - state duplicate hook
1008  * @crtc: Pointer to drm crtc structure
1009  */
1010 static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc)
1011 {
1012         struct dpu_crtc_state *cstate, *old_cstate = to_dpu_crtc_state(crtc->state);
1013
1014         cstate = kmemdup(old_cstate, sizeof(*old_cstate), GFP_KERNEL);
1015         if (!cstate) {
1016                 DPU_ERROR("failed to allocate state\n");
1017                 return NULL;
1018         }
1019
1020         /* duplicate base helper */
1021         __drm_atomic_helper_crtc_duplicate_state(crtc, &cstate->base);
1022
1023         return &cstate->base;
1024 }
1025
1026 static void dpu_crtc_atomic_print_state(struct drm_printer *p,
1027                                         const struct drm_crtc_state *state)
1028 {
1029         const struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
1030         int i;
1031
1032         for (i = 0; i < cstate->num_mixers; i++) {
1033                 drm_printf(p, "\tlm[%d]=%d\n", i, cstate->mixers[i].hw_lm->idx - LM_0);
1034                 drm_printf(p, "\tctl[%d]=%d\n", i, cstate->mixers[i].lm_ctl->idx - CTL_0);
1035                 if (cstate->mixers[i].hw_dspp)
1036                         drm_printf(p, "\tdspp[%d]=%d\n", i, cstate->mixers[i].hw_dspp->idx - DSPP_0);
1037         }
1038 }
1039
1040 static void dpu_crtc_disable(struct drm_crtc *crtc,
1041                              struct drm_atomic_state *state)
1042 {
1043         struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
1044                                                                               crtc);
1045         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1046         struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
1047         struct drm_encoder *encoder;
1048         unsigned long flags;
1049         bool release_bandwidth = false;
1050
1051         DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1052
1053         /* If disable is triggered while in self refresh mode,
1054          * reset the encoder software state so that in enable
1055          * it won't trigger a warn while assigning crtc.
1056          */
1057         if (old_crtc_state->self_refresh_active) {
1058                 drm_for_each_encoder_mask(encoder, crtc->dev,
1059                                         old_crtc_state->encoder_mask) {
1060                         dpu_encoder_assign_crtc(encoder, NULL);
1061                 }
1062                 return;
1063         }
1064
1065         /* Disable/save vblank irq handling */
1066         drm_crtc_vblank_off(crtc);
1067
1068         drm_for_each_encoder_mask(encoder, crtc->dev,
1069                                   old_crtc_state->encoder_mask) {
1070                 /* in video mode, we hold an extra bandwidth reference
1071                  * as we cannot drop bandwidth at frame-done if any
1072                  * crtc is being used in video mode.
1073                  */
1074                 if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
1075                         release_bandwidth = true;
1076
1077                 /*
1078                  * If disable is triggered during psr active(e.g: screen dim in PSR),
1079                  * we will need encoder->crtc connection to process the device sleep &
1080                  * preserve it during psr sequence.
1081                  */
1082                 if (!crtc->state->self_refresh_active)
1083                         dpu_encoder_assign_crtc(encoder, NULL);
1084         }
1085
1086         /* wait for frame_event_done completion */
1087         if (_dpu_crtc_wait_for_frame_done(crtc))
1088                 DPU_ERROR("crtc%d wait for frame done failed;frame_pending%d\n",
1089                                 crtc->base.id,
1090                                 atomic_read(&dpu_crtc->frame_pending));
1091
1092         trace_dpu_crtc_disable(DRMID(crtc), false, dpu_crtc);
1093         dpu_crtc->enabled = false;
1094
1095         if (atomic_read(&dpu_crtc->frame_pending)) {
1096                 trace_dpu_crtc_disable_frame_pending(DRMID(crtc),
1097                                      atomic_read(&dpu_crtc->frame_pending));
1098                 if (release_bandwidth)
1099                         dpu_core_perf_crtc_release_bw(crtc);
1100                 atomic_set(&dpu_crtc->frame_pending, 0);
1101         }
1102
1103         dpu_core_perf_crtc_update(crtc, 0, true);
1104
1105         drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
1106                 dpu_encoder_register_frame_event_callback(encoder, NULL, NULL);
1107
1108         memset(cstate->mixers, 0, sizeof(cstate->mixers));
1109         cstate->num_mixers = 0;
1110
1111         /* disable clk & bw control until clk & bw properties are set */
1112         cstate->bw_control = false;
1113         cstate->bw_split_vote = false;
1114
1115         if (crtc->state->event && !crtc->state->active) {
1116                 spin_lock_irqsave(&crtc->dev->event_lock, flags);
1117                 drm_crtc_send_vblank_event(crtc, crtc->state->event);
1118                 crtc->state->event = NULL;
1119                 spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
1120         }
1121
1122         pm_runtime_put_sync(crtc->dev->dev);
1123 }
1124
1125 static void dpu_crtc_enable(struct drm_crtc *crtc,
1126                 struct drm_atomic_state *state)
1127 {
1128         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1129         struct drm_encoder *encoder;
1130         bool request_bandwidth = false;
1131         struct drm_crtc_state *old_crtc_state;
1132
1133         old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc);
1134
1135         pm_runtime_get_sync(crtc->dev->dev);
1136
1137         DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1138
1139         drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask) {
1140                 /* in video mode, we hold an extra bandwidth reference
1141                  * as we cannot drop bandwidth at frame-done if any
1142                  * crtc is being used in video mode.
1143                  */
1144                 if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
1145                         request_bandwidth = true;
1146                 dpu_encoder_register_frame_event_callback(encoder,
1147                                 dpu_crtc_frame_event_cb, (void *)crtc);
1148         }
1149
1150         if (request_bandwidth)
1151                 atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
1152
1153         trace_dpu_crtc_enable(DRMID(crtc), true, dpu_crtc);
1154         dpu_crtc->enabled = true;
1155
1156         if (!old_crtc_state->self_refresh_active) {
1157                 drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
1158                         dpu_encoder_assign_crtc(encoder, crtc);
1159         }
1160
1161         /* Enable/restore vblank irq handling */
1162         drm_crtc_vblank_on(crtc);
1163 }
1164
1165 static bool dpu_crtc_needs_dirtyfb(struct drm_crtc_state *cstate)
1166 {
1167         struct drm_crtc *crtc = cstate->crtc;
1168         struct drm_encoder *encoder;
1169
1170         if (cstate->self_refresh_active)
1171                 return true;
1172
1173         drm_for_each_encoder_mask (encoder, crtc->dev, cstate->encoder_mask) {
1174                 if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_CMD) {
1175                         return true;
1176                 }
1177         }
1178
1179         return false;
1180 }
1181
1182 static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
1183                 struct drm_atomic_state *state)
1184 {
1185         struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
1186                                                                           crtc);
1187         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1188         struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc_state);
1189
1190         const struct drm_plane_state *pstate;
1191         struct drm_plane *plane;
1192
1193         int rc = 0;
1194
1195         bool needs_dirtyfb = dpu_crtc_needs_dirtyfb(crtc_state);
1196
1197         if (!crtc_state->enable || !drm_atomic_crtc_effectively_active(crtc_state)) {
1198                 DRM_DEBUG_ATOMIC("crtc%d -> enable %d, active %d, skip atomic_check\n",
1199                                 crtc->base.id, crtc_state->enable,
1200                                 crtc_state->active);
1201                 memset(&cstate->new_perf, 0, sizeof(cstate->new_perf));
1202                 return 0;
1203         }
1204
1205         DRM_DEBUG_ATOMIC("%s: check\n", dpu_crtc->name);
1206
1207         /* force a full mode set if active state changed */
1208         if (crtc_state->active_changed)
1209                 crtc_state->mode_changed = true;
1210
1211         if (cstate->num_mixers)
1212                 _dpu_crtc_setup_lm_bounds(crtc, crtc_state);
1213
1214         /* FIXME: move this to dpu_plane_atomic_check? */
1215         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
1216                 struct dpu_plane_state *dpu_pstate = to_dpu_plane_state(pstate);
1217
1218                 if (IS_ERR_OR_NULL(pstate)) {
1219                         rc = PTR_ERR(pstate);
1220                         DPU_ERROR("%s: failed to get plane%d state, %d\n",
1221                                         dpu_crtc->name, plane->base.id, rc);
1222                         return rc;
1223                 }
1224
1225                 if (!pstate->visible)
1226                         continue;
1227
1228                 dpu_pstate->needs_dirtyfb = needs_dirtyfb;
1229         }
1230
1231         atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
1232
1233         rc = dpu_core_perf_crtc_check(crtc, crtc_state);
1234         if (rc) {
1235                 DPU_ERROR("crtc%d failed performance check %d\n",
1236                                 crtc->base.id, rc);
1237                 return rc;
1238         }
1239
1240         return 0;
1241 }
1242
1243 int dpu_crtc_vblank(struct drm_crtc *crtc, bool en)
1244 {
1245         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1246         struct drm_encoder *enc;
1247
1248         trace_dpu_crtc_vblank(DRMID(&dpu_crtc->base), en, dpu_crtc);
1249
1250         /*
1251          * Normally we would iterate through encoder_mask in crtc state to find
1252          * attached encoders. In this case, we might be disabling vblank _after_
1253          * encoder_mask has been cleared.
1254          *
1255          * Instead, we "assign" a crtc to the encoder in enable and clear it in
1256          * disable (which is also after encoder_mask is cleared). So instead of
1257          * using encoder mask, we'll ask the encoder to toggle itself iff it's
1258          * currently assigned to our crtc.
1259          *
1260          * Note also that this function cannot be called while crtc is disabled
1261          * since we use drm_crtc_vblank_on/off. So we don't need to worry
1262          * about the assigned crtcs being inconsistent with the current state
1263          * (which means no need to worry about modeset locks).
1264          */
1265         list_for_each_entry(enc, &crtc->dev->mode_config.encoder_list, head) {
1266                 trace_dpu_crtc_vblank_enable(DRMID(crtc), DRMID(enc), en,
1267                                              dpu_crtc);
1268
1269                 dpu_encoder_toggle_vblank_for_crtc(enc, crtc, en);
1270         }
1271
1272         return 0;
1273 }
1274
1275 #ifdef CONFIG_DEBUG_FS
1276 static int _dpu_debugfs_status_show(struct seq_file *s, void *data)
1277 {
1278         struct dpu_crtc *dpu_crtc;
1279         struct dpu_plane_state *pstate = NULL;
1280         struct dpu_crtc_mixer *m;
1281
1282         struct drm_crtc *crtc;
1283         struct drm_plane *plane;
1284         struct drm_display_mode *mode;
1285         struct drm_framebuffer *fb;
1286         struct drm_plane_state *state;
1287         struct dpu_crtc_state *cstate;
1288
1289         int i, out_width;
1290
1291         dpu_crtc = s->private;
1292         crtc = &dpu_crtc->base;
1293
1294         drm_modeset_lock_all(crtc->dev);
1295         cstate = to_dpu_crtc_state(crtc->state);
1296
1297         mode = &crtc->state->adjusted_mode;
1298         out_width = mode->hdisplay / cstate->num_mixers;
1299
1300         seq_printf(s, "crtc:%d width:%d height:%d\n", crtc->base.id,
1301                                 mode->hdisplay, mode->vdisplay);
1302
1303         seq_puts(s, "\n");
1304
1305         for (i = 0; i < cstate->num_mixers; ++i) {
1306                 m = &cstate->mixers[i];
1307                 seq_printf(s, "\tmixer:%d ctl:%d width:%d height:%d\n",
1308                         m->hw_lm->idx - LM_0, m->lm_ctl->idx - CTL_0,
1309                         out_width, mode->vdisplay);
1310         }
1311
1312         seq_puts(s, "\n");
1313
1314         drm_atomic_crtc_for_each_plane(plane, crtc) {
1315                 pstate = to_dpu_plane_state(plane->state);
1316                 state = plane->state;
1317
1318                 if (!pstate || !state)
1319                         continue;
1320
1321                 seq_printf(s, "\tplane:%u stage:%d\n", plane->base.id,
1322                         pstate->stage);
1323
1324                 if (plane->state->fb) {
1325                         fb = plane->state->fb;
1326
1327                         seq_printf(s, "\tfb:%d image format:%4.4s wxh:%ux%u ",
1328                                 fb->base.id, (char *) &fb->format->format,
1329                                 fb->width, fb->height);
1330                         for (i = 0; i < ARRAY_SIZE(fb->format->cpp); ++i)
1331                                 seq_printf(s, "cpp[%d]:%u ",
1332                                                 i, fb->format->cpp[i]);
1333                         seq_puts(s, "\n\t");
1334
1335                         seq_printf(s, "modifier:%8llu ", fb->modifier);
1336                         seq_puts(s, "\n");
1337
1338                         seq_puts(s, "\t");
1339                         for (i = 0; i < ARRAY_SIZE(fb->pitches); i++)
1340                                 seq_printf(s, "pitches[%d]:%8u ", i,
1341                                                         fb->pitches[i]);
1342                         seq_puts(s, "\n");
1343
1344                         seq_puts(s, "\t");
1345                         for (i = 0; i < ARRAY_SIZE(fb->offsets); i++)
1346                                 seq_printf(s, "offsets[%d]:%8u ", i,
1347                                                         fb->offsets[i]);
1348                         seq_puts(s, "\n");
1349                 }
1350
1351                 seq_printf(s, "\tsrc_x:%4d src_y:%4d src_w:%4d src_h:%4d\n",
1352                         state->src_x, state->src_y, state->src_w, state->src_h);
1353
1354                 seq_printf(s, "\tdst x:%4d dst_y:%4d dst_w:%4d dst_h:%4d\n",
1355                         state->crtc_x, state->crtc_y, state->crtc_w,
1356                         state->crtc_h);
1357                 seq_printf(s, "\tsspp[0]:%s\n",
1358                            pstate->pipe.sspp->cap->name);
1359                 seq_printf(s, "\tmultirect[0]: mode: %d index: %d\n",
1360                         pstate->pipe.multirect_mode, pstate->pipe.multirect_index);
1361                 if (pstate->r_pipe.sspp) {
1362                         seq_printf(s, "\tsspp[1]:%s\n",
1363                                    pstate->r_pipe.sspp->cap->name);
1364                         seq_printf(s, "\tmultirect[1]: mode: %d index: %d\n",
1365                                    pstate->r_pipe.multirect_mode, pstate->r_pipe.multirect_index);
1366                 }
1367
1368                 seq_puts(s, "\n");
1369         }
1370         if (dpu_crtc->vblank_cb_count) {
1371                 ktime_t diff = ktime_sub(ktime_get(), dpu_crtc->vblank_cb_time);
1372                 s64 diff_ms = ktime_to_ms(diff);
1373                 s64 fps = diff_ms ? div_s64(
1374                                 dpu_crtc->vblank_cb_count * 1000, diff_ms) : 0;
1375
1376                 seq_printf(s,
1377                         "vblank fps:%lld count:%u total:%llums total_framecount:%llu\n",
1378                                 fps, dpu_crtc->vblank_cb_count,
1379                                 ktime_to_ms(diff), dpu_crtc->play_count);
1380
1381                 /* reset time & count for next measurement */
1382                 dpu_crtc->vblank_cb_count = 0;
1383                 dpu_crtc->vblank_cb_time = ktime_set(0, 0);
1384         }
1385
1386         drm_modeset_unlock_all(crtc->dev);
1387
1388         return 0;
1389 }
1390
1391 DEFINE_SHOW_ATTRIBUTE(_dpu_debugfs_status);
1392
1393 static int dpu_crtc_debugfs_state_show(struct seq_file *s, void *v)
1394 {
1395         struct drm_crtc *crtc = s->private;
1396         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1397
1398         seq_printf(s, "client type: %d\n", dpu_crtc_get_client_type(crtc));
1399         seq_printf(s, "intf_mode: %d\n", dpu_crtc_get_intf_mode(crtc));
1400         seq_printf(s, "core_clk_rate: %llu\n",
1401                         dpu_crtc->cur_perf.core_clk_rate);
1402         seq_printf(s, "bw_ctl: %llu\n", dpu_crtc->cur_perf.bw_ctl);
1403         seq_printf(s, "max_per_pipe_ib: %llu\n",
1404                                 dpu_crtc->cur_perf.max_per_pipe_ib);
1405
1406         return 0;
1407 }
1408 DEFINE_SHOW_ATTRIBUTE(dpu_crtc_debugfs_state);
1409
1410 static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1411 {
1412         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1413
1414         debugfs_create_file("status", 0400,
1415                         crtc->debugfs_entry,
1416                         dpu_crtc, &_dpu_debugfs_status_fops);
1417         debugfs_create_file("state", 0600,
1418                         crtc->debugfs_entry,
1419                         &dpu_crtc->base,
1420                         &dpu_crtc_debugfs_state_fops);
1421
1422         return 0;
1423 }
1424 #else
1425 static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1426 {
1427         return 0;
1428 }
1429 #endif /* CONFIG_DEBUG_FS */
1430
1431 static int dpu_crtc_late_register(struct drm_crtc *crtc)
1432 {
1433         return _dpu_crtc_init_debugfs(crtc);
1434 }
1435
1436 static const struct drm_crtc_funcs dpu_crtc_funcs = {
1437         .set_config = drm_atomic_helper_set_config,
1438         .destroy = dpu_crtc_destroy,
1439         .page_flip = drm_atomic_helper_page_flip,
1440         .reset = dpu_crtc_reset,
1441         .atomic_duplicate_state = dpu_crtc_duplicate_state,
1442         .atomic_destroy_state = dpu_crtc_destroy_state,
1443         .atomic_print_state = dpu_crtc_atomic_print_state,
1444         .late_register = dpu_crtc_late_register,
1445         .verify_crc_source = dpu_crtc_verify_crc_source,
1446         .set_crc_source = dpu_crtc_set_crc_source,
1447         .enable_vblank  = msm_crtc_enable_vblank,
1448         .disable_vblank = msm_crtc_disable_vblank,
1449         .get_vblank_timestamp = drm_crtc_vblank_helper_get_vblank_timestamp,
1450         .get_vblank_counter = dpu_crtc_get_vblank_counter,
1451 };
1452
1453 static const struct drm_crtc_helper_funcs dpu_crtc_helper_funcs = {
1454         .atomic_disable = dpu_crtc_disable,
1455         .atomic_enable = dpu_crtc_enable,
1456         .atomic_check = dpu_crtc_atomic_check,
1457         .atomic_begin = dpu_crtc_atomic_begin,
1458         .atomic_flush = dpu_crtc_atomic_flush,
1459         .get_scanout_position = dpu_crtc_get_scanout_position,
1460 };
1461
1462 /* initialize crtc */
1463 struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane,
1464                                 struct drm_plane *cursor)
1465 {
1466         struct msm_drm_private *priv = dev->dev_private;
1467         struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms);
1468         struct drm_crtc *crtc = NULL;
1469         struct dpu_crtc *dpu_crtc = NULL;
1470         int i, ret;
1471
1472         dpu_crtc = kzalloc(sizeof(*dpu_crtc), GFP_KERNEL);
1473         if (!dpu_crtc)
1474                 return ERR_PTR(-ENOMEM);
1475
1476         crtc = &dpu_crtc->base;
1477         crtc->dev = dev;
1478
1479         spin_lock_init(&dpu_crtc->spin_lock);
1480         atomic_set(&dpu_crtc->frame_pending, 0);
1481
1482         init_completion(&dpu_crtc->frame_done_comp);
1483
1484         INIT_LIST_HEAD(&dpu_crtc->frame_event_list);
1485
1486         for (i = 0; i < ARRAY_SIZE(dpu_crtc->frame_events); i++) {
1487                 INIT_LIST_HEAD(&dpu_crtc->frame_events[i].list);
1488                 list_add(&dpu_crtc->frame_events[i].list,
1489                                 &dpu_crtc->frame_event_list);
1490                 kthread_init_work(&dpu_crtc->frame_events[i].work,
1491                                 dpu_crtc_frame_event_work);
1492         }
1493
1494         drm_crtc_init_with_planes(dev, crtc, plane, cursor, &dpu_crtc_funcs,
1495                                 NULL);
1496
1497         drm_crtc_helper_add(crtc, &dpu_crtc_helper_funcs);
1498
1499         if (dpu_kms->catalog->dspp_count)
1500                 drm_crtc_enable_color_mgmt(crtc, 0, true, 0);
1501
1502         /* save user friendly CRTC name for later */
1503         snprintf(dpu_crtc->name, DPU_CRTC_NAME_SIZE, "crtc%u", crtc->base.id);
1504
1505         /* initialize event handling */
1506         spin_lock_init(&dpu_crtc->event_lock);
1507
1508         ret = drm_self_refresh_helper_init(crtc);
1509         if (ret) {
1510                 DPU_ERROR("Failed to initialize %s with self-refresh helpers %d\n",
1511                         crtc->name, ret);
1512                 return ERR_PTR(ret);
1513         }
1514
1515         DRM_DEBUG_KMS("%s: successfully initialized crtc\n", dpu_crtc->name);
1516         return crtc;
1517 }