drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c

   1 /*
   2  * Copyright 2021 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include "amdgpu.h"
  25 #include "amdgpu_jpeg.h"
  26 #include "amdgpu_pm.h"
  27 #include "soc15.h"
  28 #include "soc15d.h"
  29 #include "jpeg_v2_0.h"
  30 #include "jpeg_v4_0.h"
  31 #include "mmsch_v4_0.h"
  32
  33 #include "vcn/vcn_4_0_0_offset.h"
  34 #include "vcn/vcn_4_0_0_sh_mask.h"
  35 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
  36
  37 #define regUVD_JPEG_PITCH_INTERNAL_OFFSET                  0x401f
  38
  39 static int jpeg_v4_0_start_sriov(struct amdgpu_device *adev);
  40 static void jpeg_v4_0_set_dec_ring_funcs(struct amdgpu_device *adev);
  41 static void jpeg_v4_0_set_irq_funcs(struct amdgpu_device *adev);
  42 static int jpeg_v4_0_set_powergating_state(void *handle,
  43                                 enum amd_powergating_state state);
  44 static void jpeg_v4_0_set_ras_funcs(struct amdgpu_device *adev);
  45
  46 static void jpeg_v4_0_dec_ring_set_wptr(struct amdgpu_ring *ring);
  47
  48 /**
  49  * jpeg_v4_0_early_init - set function pointers
  50  *
  51  * @handle: amdgpu_device pointer
  52  *
  53  * Set ring and irq function pointers
  54  */
  55 static int jpeg_v4_0_early_init(void *handle)
  56 {
  57         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  58
  59
  60         adev->jpeg.num_jpeg_inst = 1;
  61
  62         jpeg_v4_0_set_dec_ring_funcs(adev);
  63         jpeg_v4_0_set_irq_funcs(adev);
  64         jpeg_v4_0_set_ras_funcs(adev);
  65
  66         return 0;
  67 }
  68
  69 /**
  70  * jpeg_v4_0_sw_init - sw init for JPEG block
  71  *
  72  * @handle: amdgpu_device pointer
  73  *
  74  * Load firmware and sw initialization
  75  */
  76 static int jpeg_v4_0_sw_init(void *handle)
  77 {
  78         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  79         struct amdgpu_ring *ring;
  80         int r;
  81
  82         /* JPEG TRAP */
  83         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
  84                 VCN_4_0__SRCID__JPEG_DECODE, &adev->jpeg.inst->irq);
  85         if (r)
  86                 return r;
  87
  88         /* JPEG DJPEG POISON EVENT */
  89         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
  90                         VCN_4_0__SRCID_DJPEG0_POISON, &adev->jpeg.inst->irq);
  91         if (r)
  92                 return r;
  93
  94         /* JPEG EJPEG POISON EVENT */
  95         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
  96                         VCN_4_0__SRCID_EJPEG0_POISON, &adev->jpeg.inst->irq);
  97         if (r)
  98                 return r;
  99
 100         r = amdgpu_jpeg_sw_init(adev);
 101         if (r)
 102                 return r;
 103
 104         r = amdgpu_jpeg_resume(adev);
 105         if (r)
 106                 return r;
 107
 108         ring = &adev->jpeg.inst->ring_dec;
 109         ring->use_doorbell = true;
 110         ring->doorbell_index = amdgpu_sriov_vf(adev) ? (((adev->doorbell_index.vcn.vcn_ring0_1) << 1) + 4) : ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1);
 111         ring->vm_hub = AMDGPU_MMHUB_0;
 112
 113         sprintf(ring->name, "jpeg_dec");
 114         r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq, 0,
 115                              AMDGPU_RING_PRIO_DEFAULT, NULL);
 116         if (r)
 117                 return r;
 118
 119         adev->jpeg.internal.jpeg_pitch = regUVD_JPEG_PITCH_INTERNAL_OFFSET;
 120         adev->jpeg.inst->external.jpeg_pitch = SOC15_REG_OFFSET(JPEG, 0, regUVD_JPEG_PITCH);
 121
 122         r = amdgpu_jpeg_ras_sw_init(adev);
 123         if (r)
 124                 return r;
 125
 126         return 0;
 127 }
 128
 129 /**
 130  * jpeg_v4_0_sw_fini - sw fini for JPEG block
 131  *
 132  * @handle: amdgpu_device pointer
 133  *
 134  * JPEG suspend and free up sw allocation
 135  */
 136 static int jpeg_v4_0_sw_fini(void *handle)
 137 {
 138         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 139         int r;
 140
 141         r = amdgpu_jpeg_suspend(adev);
 142         if (r)
 143                 return r;
 144
 145         r = amdgpu_jpeg_sw_fini(adev);
 146
 147         return r;
 148 }
 149
 150 /**
 151  * jpeg_v4_0_hw_init - start and test JPEG block
 152  *
 153  * @handle: amdgpu_device pointer
 154  *
 155  */
 156 static int jpeg_v4_0_hw_init(void *handle)
 157 {
 158         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 159         struct amdgpu_ring *ring = &adev->jpeg.inst->ring_dec;
 160         int r;
 161
 162         if (amdgpu_sriov_vf(adev)) {
 163                 r = jpeg_v4_0_start_sriov(adev);
 164                 if (r)
 165                         return r;
 166                 ring->wptr = 0;
 167                 ring->wptr_old = 0;
 168                 jpeg_v4_0_dec_ring_set_wptr(ring);
 169                 ring->sched.ready = true;
 170         } else {
 171                 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
 172                                                 (adev->doorbell_index.vcn.vcn_ring0_1 << 1), 0);
 173
 174                 WREG32_SOC15(VCN, 0, regVCN_JPEG_DB_CTRL,
 175                         ring->doorbell_index << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
 176                         VCN_JPEG_DB_CTRL__EN_MASK);
 177
 178                 r = amdgpu_ring_test_helper(ring);
 179                 if (r)
 180                         return r;
 181         }
 182
 183         DRM_DEV_INFO(adev->dev, "JPEG decode initialized successfully.\n");
 184
 185         return 0;
 186 }
 187
 188 /**
 189  * jpeg_v4_0_hw_fini - stop the hardware block
 190  *
 191  * @handle: amdgpu_device pointer
 192  *
 193  * Stop the JPEG block, mark ring as not ready any more
 194  */
 195 static int jpeg_v4_0_hw_fini(void *handle)
 196 {
 197         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 198
 199         cancel_delayed_work_sync(&adev->vcn.idle_work);
 200         if (!amdgpu_sriov_vf(adev)) {
 201                 if (adev->jpeg.cur_state != AMD_PG_STATE_GATE &&
 202                         RREG32_SOC15(JPEG, 0, regUVD_JRBC_STATUS))
 203                         jpeg_v4_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
 204         }
 205         amdgpu_irq_put(adev, &adev->jpeg.inst->irq, 0);
 206
 207         return 0;
 208 }
 209
 210 /**
 211  * jpeg_v4_0_suspend - suspend JPEG block
 212  *
 213  * @handle: amdgpu_device pointer
 214  *
 215  * HW fini and suspend JPEG block
 216  */
 217 static int jpeg_v4_0_suspend(void *handle)
 218 {
 219         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 220         int r;
 221
 222         r = jpeg_v4_0_hw_fini(adev);
 223         if (r)
 224                 return r;
 225
 226         r = amdgpu_jpeg_suspend(adev);
 227
 228         return r;
 229 }
 230
 231 /**
 232  * jpeg_v4_0_resume - resume JPEG block
 233  *
 234  * @handle: amdgpu_device pointer
 235  *
 236  * Resume firmware and hw init JPEG block
 237  */
 238 static int jpeg_v4_0_resume(void *handle)
 239 {
 240         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 241         int r;
 242
 243         r = amdgpu_jpeg_resume(adev);
 244         if (r)
 245                 return r;
 246
 247         r = jpeg_v4_0_hw_init(adev);
 248
 249         return r;
 250 }
 251
 252 static void jpeg_v4_0_disable_clock_gating(struct amdgpu_device *adev)
 253 {
 254         uint32_t data = 0;
 255
 256         data = RREG32_SOC15(JPEG, 0, regJPEG_CGC_CTRL);
 257         if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
 258                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 259                 data &= (~JPEG_CGC_CTRL__JPEG_DEC_MODE_MASK);
 260         } else {
 261                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 262         }
 263
 264         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 265         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 266         WREG32_SOC15(JPEG, 0, regJPEG_CGC_CTRL, data);
 267
 268         data = RREG32_SOC15(JPEG, 0, regJPEG_CGC_GATE);
 269         data &= ~(JPEG_CGC_GATE__JPEG_DEC_MASK
 270                 | JPEG_CGC_GATE__JPEG2_DEC_MASK
 271                 | JPEG_CGC_GATE__JMCIF_MASK
 272                 | JPEG_CGC_GATE__JRBBM_MASK);
 273         WREG32_SOC15(JPEG, 0, regJPEG_CGC_GATE, data);
 274 }
 275
 276 static void jpeg_v4_0_enable_clock_gating(struct amdgpu_device *adev)
 277 {
 278         uint32_t data = 0;
 279
 280         data = RREG32_SOC15(JPEG, 0, regJPEG_CGC_CTRL);
 281         if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
 282                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 283                 data |= JPEG_CGC_CTRL__JPEG_DEC_MODE_MASK;
 284         } else {
 285                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 286         }
 287
 288         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 289         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 290         WREG32_SOC15(JPEG, 0, regJPEG_CGC_CTRL, data);
 291
 292         data = RREG32_SOC15(JPEG, 0, regJPEG_CGC_GATE);
 293         data |= (JPEG_CGC_GATE__JPEG_DEC_MASK
 294                 |JPEG_CGC_GATE__JPEG2_DEC_MASK
 295                 |JPEG_CGC_GATE__JMCIF_MASK
 296                 |JPEG_CGC_GATE__JRBBM_MASK);
 297         WREG32_SOC15(JPEG, 0, regJPEG_CGC_GATE, data);
 298 }
 299
 300 static int jpeg_v4_0_disable_static_power_gating(struct amdgpu_device *adev)
 301 {
 302         if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
 303                 uint32_t data = 0;
 304                 int r = 0;
 305
 306                 data = 1 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT;
 307                 WREG32(SOC15_REG_OFFSET(JPEG, 0, regUVD_PGFSM_CONFIG), data);
 308
 309                 r = SOC15_WAIT_ON_RREG(JPEG, 0,
 310                         regUVD_PGFSM_STATUS, UVD_PGFSM_STATUS_UVDJ_PWR_ON,
 311                         UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
 312
 313                 if (r) {
 314                         DRM_DEV_ERROR(adev->dev, "amdgpu: JPEG disable power gating failed\n");
 315                         return r;
 316                 }
 317         }
 318
 319         /* disable anti hang mechanism */
 320         WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regUVD_JPEG_POWER_STATUS), 0,
 321                 ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
 322
 323         /* keep the JPEG in static PG mode */
 324         WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regUVD_JPEG_POWER_STATUS), 0,
 325                 ~UVD_JPEG_POWER_STATUS__JPEG_PG_MODE_MASK);
 326
 327         return 0;
 328 }
 329
 330 static int jpeg_v4_0_enable_static_power_gating(struct amdgpu_device *adev)
 331 {
 332         /* enable anti hang mechanism */
 333         WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regUVD_JPEG_POWER_STATUS),
 334                 UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK,
 335                 ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
 336
 337         if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
 338                 uint32_t data = 0;
 339                 int r = 0;
 340
 341                 data = 2 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT;
 342                 WREG32(SOC15_REG_OFFSET(JPEG, 0, regUVD_PGFSM_CONFIG), data);
 343
 344                 r = SOC15_WAIT_ON_RREG(JPEG, 0, regUVD_PGFSM_STATUS,
 345                         (2 << UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT),
 346                         UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
 347
 348                 if (r) {
 349                         DRM_DEV_ERROR(adev->dev, "amdgpu: JPEG enable power gating failed\n");
 350                         return r;
 351                 }
 352         }
 353
 354         return 0;
 355 }
 356
 357 /**
 358  * jpeg_v4_0_start - start JPEG block
 359  *
 360  * @adev: amdgpu_device pointer
 361  *
 362  * Setup and start the JPEG block
 363  */
 364 static int jpeg_v4_0_start(struct amdgpu_device *adev)
 365 {
 366         struct amdgpu_ring *ring = &adev->jpeg.inst->ring_dec;
 367         int r;
 368
 369         if (adev->pm.dpm_enabled)
 370                 amdgpu_dpm_enable_jpeg(adev, true);
 371
 372         /* disable power gating */
 373         r = jpeg_v4_0_disable_static_power_gating(adev);
 374         if (r)
 375                 return r;
 376
 377         /* JPEG disable CGC */
 378         jpeg_v4_0_disable_clock_gating(adev);
 379
 380         /* MJPEG global tiling registers */
 381         WREG32_SOC15(JPEG, 0, regJPEG_DEC_GFX10_ADDR_CONFIG,
 382                 adev->gfx.config.gb_addr_config);
 383
 384
 385         /* enable JMI channel */
 386         WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI_CNTL), 0,
 387                 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 388
 389         /* enable System Interrupt for JRBC */
 390         WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regJPEG_SYS_INT_EN),
 391                 JPEG_SYS_INT_EN__DJRBC_MASK,
 392                 ~JPEG_SYS_INT_EN__DJRBC_MASK);
 393
 394         WREG32_SOC15(JPEG, 0, regUVD_LMI_JRBC_RB_VMID, 0);
 395         WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_CNTL, (0x00000001L | 0x00000002L));
 396         WREG32_SOC15(JPEG, 0, regUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
 397                 lower_32_bits(ring->gpu_addr));
 398         WREG32_SOC15(JPEG, 0, regUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
 399                 upper_32_bits(ring->gpu_addr));
 400         WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_RPTR, 0);
 401         WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_WPTR, 0);
 402         WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_CNTL, 0x00000002L);
 403         WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_SIZE, ring->ring_size / 4);
 404         ring->wptr = RREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_WPTR);
 405
 406         return 0;
 407 }
 408
 409 static int jpeg_v4_0_start_sriov(struct amdgpu_device *adev)
 410 {
 411         struct amdgpu_ring *ring;
 412         uint64_t ctx_addr;
 413         uint32_t param, resp, expected;
 414         uint32_t tmp, timeout;
 415
 416         struct amdgpu_mm_table *table = &adev->virt.mm_table;
 417         uint32_t *table_loc;
 418         uint32_t table_size;
 419         uint32_t size, size_dw;
 420         uint32_t init_status;
 421
 422         struct mmsch_v4_0_cmd_direct_write
 423                 direct_wt = { {0} };
 424         struct mmsch_v4_0_cmd_end end = { {0} };
 425         struct mmsch_v4_0_init_header header;
 426
 427         direct_wt.cmd_header.command_type =
 428                 MMSCH_COMMAND__DIRECT_REG_WRITE;
 429         end.cmd_header.command_type =
 430                 MMSCH_COMMAND__END;
 431
 432         header.version = MMSCH_VERSION;
 433         header.total_size = RREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_SIZE);
 434
 435         header.jpegdec.init_status = 0;
 436         header.jpegdec.table_offset = 0;
 437         header.jpegdec.table_size = 0;
 438
 439         table_loc = (uint32_t *)table->cpu_addr;
 440         table_loc += header.total_size;
 441
 442         table_size = 0;
 443
 444         ring = &adev->jpeg.inst->ring_dec;
 445
 446         MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(JPEG, 0,
 447                 regUVD_LMI_JRBC_RB_64BIT_BAR_LOW),
 448                 lower_32_bits(ring->gpu_addr));
 449         MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(JPEG, 0,
 450                 regUVD_LMI_JRBC_RB_64BIT_BAR_HIGH),
 451                 upper_32_bits(ring->gpu_addr));
 452         MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(JPEG, 0,
 453                 regUVD_JRBC_RB_SIZE), ring->ring_size / 4);
 454
 455         /* add end packet */
 456         MMSCH_V4_0_INSERT_END();
 457
 458         /* refine header */
 459         header.jpegdec.init_status = 0;
 460         header.jpegdec.table_offset = header.total_size;
 461         header.jpegdec.table_size = table_size;
 462         header.total_size += table_size;
 463
 464         /* Update init table header in memory */
 465         size = sizeof(struct mmsch_v4_0_init_header);
 466         table_loc = (uint32_t *)table->cpu_addr;
 467         memcpy((void *)table_loc, &header, size);
 468
 469         /* message MMSCH (in VCN[0]) to initialize this client
 470          * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
 471          * of memory descriptor location
 472          */
 473         ctx_addr = table->gpu_addr;
 474         WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
 475         WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
 476
 477         /* 2, update vmid of descriptor */
 478         tmp = RREG32_SOC15(VCN, 0, regMMSCH_VF_VMID);
 479         tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
 480         /* use domain0 for MM scheduler */
 481         tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
 482         WREG32_SOC15(VCN, 0, regMMSCH_VF_VMID, tmp);
 483
 484         /* 3, notify mmsch about the size of this descriptor */
 485         size = header.total_size;
 486         WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_SIZE, size);
 487
 488         /* 4, set resp to zero */
 489         WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP, 0);
 490
 491         /* 5, kick off the initialization and wait until
 492          * MMSCH_VF_MAILBOX_RESP becomes non-zero
 493          */
 494         param = 0x00000001;
 495         WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_HOST, param);
 496         tmp = 0;
 497         timeout = 1000;
 498         resp = 0;
 499         expected = MMSCH_VF_MAILBOX_RESP__OK;
 500         init_status = ((struct mmsch_v4_0_init_header *)(table_loc))->jpegdec.init_status;
 501         while (resp != expected) {
 502                 resp = RREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP);
 503
 504                 if (resp != 0)
 505                         break;
 506                 udelay(10);
 507                 tmp = tmp + 10;
 508                 if (tmp >= timeout) {
 509                         DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
 510                                 " waiting for regMMSCH_VF_MAILBOX_RESP "\
 511                                 "(expected=0x%08x, readback=0x%08x)\n",
 512                                 tmp, expected, resp);
 513                         return -EBUSY;
 514                 }
 515         }
 516         if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE && init_status != MMSCH_VF_ENGINE_STATUS__PASS)
 517                 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init status for jpeg: %x\n", resp, init_status);
 518
 519         return 0;
 520
 521 }
 522
 523 /**
 524  * jpeg_v4_0_stop - stop JPEG block
 525  *
 526  * @adev: amdgpu_device pointer
 527  *
 528  * stop the JPEG block
 529  */
 530 static int jpeg_v4_0_stop(struct amdgpu_device *adev)
 531 {
 532         int r;
 533
 534         /* reset JMI */
 535         WREG32_P(SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI_CNTL),
 536                 UVD_JMI_CNTL__SOFT_RESET_MASK,
 537                 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 538
 539         jpeg_v4_0_enable_clock_gating(adev);
 540
 541         /* enable power gating */
 542         r = jpeg_v4_0_enable_static_power_gating(adev);
 543         if (r)
 544                 return r;
 545
 546         if (adev->pm.dpm_enabled)
 547                 amdgpu_dpm_enable_jpeg(adev, false);
 548
 549         return 0;
 550 }
 551
 552 /**
 553  * jpeg_v4_0_dec_ring_get_rptr - get read pointer
 554  *
 555  * @ring: amdgpu_ring pointer
 556  *
 557  * Returns the current hardware read pointer
 558  */
 559 static uint64_t jpeg_v4_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
 560 {
 561         struct amdgpu_device *adev = ring->adev;
 562
 563         return RREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_RPTR);
 564 }
 565
 566 /**
 567  * jpeg_v4_0_dec_ring_get_wptr - get write pointer
 568  *
 569  * @ring: amdgpu_ring pointer
 570  *
 571  * Returns the current hardware write pointer
 572  */
 573 static uint64_t jpeg_v4_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
 574 {
 575         struct amdgpu_device *adev = ring->adev;
 576
 577         if (ring->use_doorbell)
 578                 return *ring->wptr_cpu_addr;
 579         else
 580                 return RREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_WPTR);
 581 }
 582
 583 /**
 584  * jpeg_v4_0_dec_ring_set_wptr - set write pointer
 585  *
 586  * @ring: amdgpu_ring pointer
 587  *
 588  * Commits the write pointer to the hardware
 589  */
 590 static void jpeg_v4_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
 591 {
 592         struct amdgpu_device *adev = ring->adev;
 593
 594         if (ring->use_doorbell) {
 595                 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
 596                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
 597         } else {
 598                 WREG32_SOC15(JPEG, 0, regUVD_JRBC_RB_WPTR, lower_32_bits(ring->wptr));
 599         }
 600 }
 601
 602 static bool jpeg_v4_0_is_idle(void *handle)
 603 {
 604         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 605         int ret = 1;
 606
 607         ret &= (((RREG32_SOC15(JPEG, 0, regUVD_JRBC_STATUS) &
 608                 UVD_JRBC_STATUS__RB_JOB_DONE_MASK) ==
 609                 UVD_JRBC_STATUS__RB_JOB_DONE_MASK));
 610
 611         return ret;
 612 }
 613
 614 static int jpeg_v4_0_wait_for_idle(void *handle)
 615 {
 616         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 617
 618         return SOC15_WAIT_ON_RREG(JPEG, 0, regUVD_JRBC_STATUS,
 619                 UVD_JRBC_STATUS__RB_JOB_DONE_MASK,
 620                 UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
 621 }
 622
 623 static int jpeg_v4_0_set_clockgating_state(void *handle,
 624                                           enum amd_clockgating_state state)
 625 {
 626         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 627         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
 628
 629         if (enable) {
 630                 if (!jpeg_v4_0_is_idle(handle))
 631                         return -EBUSY;
 632                 jpeg_v4_0_enable_clock_gating(adev);
 633         } else {
 634                 jpeg_v4_0_disable_clock_gating(adev);
 635         }
 636
 637         return 0;
 638 }
 639
 640 static int jpeg_v4_0_set_powergating_state(void *handle,
 641                                           enum amd_powergating_state state)
 642 {
 643         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 644         int ret;
 645
 646         if (amdgpu_sriov_vf(adev)) {
 647                 adev->jpeg.cur_state = AMD_PG_STATE_UNGATE;
 648                 return 0;
 649         }
 650
 651         if (state == adev->jpeg.cur_state)
 652                 return 0;
 653
 654         if (state == AMD_PG_STATE_GATE)
 655                 ret = jpeg_v4_0_stop(adev);
 656         else
 657                 ret = jpeg_v4_0_start(adev);
 658
 659         if (!ret)
 660                 adev->jpeg.cur_state = state;
 661
 662         return ret;
 663 }
 664
 665 static int jpeg_v4_0_set_interrupt_state(struct amdgpu_device *adev,
 666                                         struct amdgpu_irq_src *source,
 667                                         unsigned type,
 668                                         enum amdgpu_interrupt_state state)
 669 {
 670         return 0;
 671 }
 672
 673 static int jpeg_v4_0_process_interrupt(struct amdgpu_device *adev,
 674                                       struct amdgpu_irq_src *source,
 675                                       struct amdgpu_iv_entry *entry)
 676 {
 677         DRM_DEBUG("IH: JPEG TRAP\n");
 678
 679         switch (entry->src_id) {
 680         case VCN_4_0__SRCID__JPEG_DECODE:
 681                 amdgpu_fence_process(&adev->jpeg.inst->ring_dec);
 682                 break;
 683         case VCN_4_0__SRCID_DJPEG0_POISON:
 684         case VCN_4_0__SRCID_EJPEG0_POISON:
 685                 amdgpu_jpeg_process_poison_irq(adev, source, entry);
 686                 break;
 687         default:
 688                 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
 689                           entry->src_id, entry->src_data[0]);
 690                 break;
 691         }
 692
 693         return 0;
 694 }
 695
 696 static const struct amd_ip_funcs jpeg_v4_0_ip_funcs = {
 697         .name = "jpeg_v4_0",
 698         .early_init = jpeg_v4_0_early_init,
 699         .late_init = NULL,
 700         .sw_init = jpeg_v4_0_sw_init,
 701         .sw_fini = jpeg_v4_0_sw_fini,
 702         .hw_init = jpeg_v4_0_hw_init,
 703         .hw_fini = jpeg_v4_0_hw_fini,
 704         .suspend = jpeg_v4_0_suspend,
 705         .resume = jpeg_v4_0_resume,
 706         .is_idle = jpeg_v4_0_is_idle,
 707         .wait_for_idle = jpeg_v4_0_wait_for_idle,
 708         .check_soft_reset = NULL,
 709         .pre_soft_reset = NULL,
 710         .soft_reset = NULL,
 711         .post_soft_reset = NULL,
 712         .set_clockgating_state = jpeg_v4_0_set_clockgating_state,
 713         .set_powergating_state = jpeg_v4_0_set_powergating_state,
 714 };
 715
 716 static const struct amdgpu_ring_funcs jpeg_v4_0_dec_ring_vm_funcs = {
 717         .type = AMDGPU_RING_TYPE_VCN_JPEG,
 718         .align_mask = 0xf,
 719         .get_rptr = jpeg_v4_0_dec_ring_get_rptr,
 720         .get_wptr = jpeg_v4_0_dec_ring_get_wptr,
 721         .set_wptr = jpeg_v4_0_dec_ring_set_wptr,
 722         .emit_frame_size =
 723                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
 724                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
 725                 8 + /* jpeg_v4_0_dec_ring_emit_vm_flush */
 726                 18 + 18 + /* jpeg_v4_0_dec_ring_emit_fence x2 vm fence */
 727                 8 + 16,
 728         .emit_ib_size = 22, /* jpeg_v4_0_dec_ring_emit_ib */
 729         .emit_ib = jpeg_v2_0_dec_ring_emit_ib,
 730         .emit_fence = jpeg_v2_0_dec_ring_emit_fence,
 731         .emit_vm_flush = jpeg_v2_0_dec_ring_emit_vm_flush,
 732         .test_ring = amdgpu_jpeg_dec_ring_test_ring,
 733         .test_ib = amdgpu_jpeg_dec_ring_test_ib,
 734         .insert_nop = jpeg_v2_0_dec_ring_nop,
 735         .insert_start = jpeg_v2_0_dec_ring_insert_start,
 736         .insert_end = jpeg_v2_0_dec_ring_insert_end,
 737         .pad_ib = amdgpu_ring_generic_pad_ib,
 738         .begin_use = amdgpu_jpeg_ring_begin_use,
 739         .end_use = amdgpu_jpeg_ring_end_use,
 740         .emit_wreg = jpeg_v2_0_dec_ring_emit_wreg,
 741         .emit_reg_wait = jpeg_v2_0_dec_ring_emit_reg_wait,
 742         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
 743 };
 744
 745 static void jpeg_v4_0_set_dec_ring_funcs(struct amdgpu_device *adev)
 746 {
 747         adev->jpeg.inst->ring_dec.funcs = &jpeg_v4_0_dec_ring_vm_funcs;
 748         DRM_DEV_INFO(adev->dev, "JPEG decode is enabled in VM mode\n");
 749 }
 750
 751 static const struct amdgpu_irq_src_funcs jpeg_v4_0_irq_funcs = {
 752         .set = jpeg_v4_0_set_interrupt_state,
 753         .process = jpeg_v4_0_process_interrupt,
 754 };
 755
 756 static void jpeg_v4_0_set_irq_funcs(struct amdgpu_device *adev)
 757 {
 758         adev->jpeg.inst->irq.num_types = 1;
 759         adev->jpeg.inst->irq.funcs = &jpeg_v4_0_irq_funcs;
 760 }
 761
 762 const struct amdgpu_ip_block_version jpeg_v4_0_ip_block = {
 763         .type = AMD_IP_BLOCK_TYPE_JPEG,
 764         .major = 4,
 765         .minor = 0,
 766         .rev = 0,
 767         .funcs = &jpeg_v4_0_ip_funcs,
 768 };
 769
 770 static uint32_t jpeg_v4_0_query_poison_by_instance(struct amdgpu_device *adev,
 771                 uint32_t instance, uint32_t sub_block)
 772 {
 773         uint32_t poison_stat = 0, reg_value = 0;
 774
 775         switch (sub_block) {
 776         case AMDGPU_JPEG_V4_0_JPEG0:
 777                 reg_value = RREG32_SOC15(JPEG, instance, regUVD_RAS_JPEG0_STATUS);
 778                 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_JPEG0_STATUS, POISONED_PF);
 779                 break;
 780         case AMDGPU_JPEG_V4_0_JPEG1:
 781                 reg_value = RREG32_SOC15(JPEG, instance, regUVD_RAS_JPEG1_STATUS);
 782                 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_JPEG1_STATUS, POISONED_PF);
 783                 break;
 784         default:
 785                 break;
 786         }
 787
 788         if (poison_stat)
 789                 dev_info(adev->dev, "Poison detected in JPEG%d sub_block%d\n",
 790                         instance, sub_block);
 791
 792         return poison_stat;
 793 }
 794
 795 static bool jpeg_v4_0_query_ras_poison_status(struct amdgpu_device *adev)
 796 {
 797         uint32_t inst = 0, sub = 0, poison_stat = 0;
 798
 799         for (inst = 0; inst < adev->jpeg.num_jpeg_inst; inst++)
 800                 for (sub = 0; sub < AMDGPU_JPEG_V4_0_MAX_SUB_BLOCK; sub++)
 801                         poison_stat +=
 802                                 jpeg_v4_0_query_poison_by_instance(adev, inst, sub);
 803
 804         return !!poison_stat;
 805 }
 806
 807 const struct amdgpu_ras_block_hw_ops jpeg_v4_0_ras_hw_ops = {
 808         .query_poison_status = jpeg_v4_0_query_ras_poison_status,
 809 };
 810
 811 static struct amdgpu_jpeg_ras jpeg_v4_0_ras = {
 812         .ras_block = {
 813                 .hw_ops = &jpeg_v4_0_ras_hw_ops,
 814         },
 815 };
 816
 817 static void jpeg_v4_0_set_ras_funcs(struct amdgpu_device *adev)
 818 {
 819         switch (adev->ip_versions[JPEG_HWIP][0]) {
 820         case IP_VERSION(4, 0, 0):
 821                 adev->jpeg.ras = &jpeg_v4_0_ras;
 822                 break;
 823         default:
 824                 break;
 825         }
 826 }