2 * Copyright 2019 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/firmware.h>
27 #include "amdgpu_vcn.h"
28 #include "amdgpu_pm.h"
32 #include "mmsch_v1_0.h"
34 #include "vcn/vcn_2_5_offset.h"
35 #include "vcn/vcn_2_5_sh_mask.h"
36 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
38 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x27
39 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x0f
40 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x10
41 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x11
42 #define mmUVD_NO_OP_INTERNAL_OFFSET 0x29
43 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x66
44 #define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d
46 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x431
47 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x3b4
48 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x3b5
49 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x25c
51 #define VCN25_MAX_HW_INSTANCES_ARCTURUS 2
53 static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev);
54 static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev);
55 static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev);
56 static int vcn_v2_5_set_powergating_state(void *handle,
57 enum amd_powergating_state state);
58 static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev,
59 int inst_idx, struct dpg_pause_state *new_state);
60 static int vcn_v2_5_sriov_start(struct amdgpu_device *adev);
62 static int amdgpu_ih_clientid_vcns[] = {
63 SOC15_IH_CLIENTID_VCN,
64 SOC15_IH_CLIENTID_VCN1
68 * vcn_v2_5_early_init - set function pointers
70 * @handle: amdgpu_device pointer
72 * Set ring and irq function pointers
74 static int vcn_v2_5_early_init(void *handle)
76 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
78 if (amdgpu_sriov_vf(adev)) {
79 adev->vcn.num_vcn_inst = 2;
80 adev->vcn.harvest_config = 0;
81 adev->vcn.num_enc_rings = 1;
83 if (adev->asic_type == CHIP_ARCTURUS) {
87 adev->vcn.num_vcn_inst = VCN25_MAX_HW_INSTANCES_ARCTURUS;
88 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
89 harvest = RREG32_SOC15(VCN, i, mmCC_UVD_HARVESTING);
90 if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK)
91 adev->vcn.harvest_config |= 1 << i;
94 if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
95 AMDGPU_VCN_HARVEST_VCN1))
96 /* both instances are harvested, disable the block */
99 adev->vcn.num_vcn_inst = 1;
101 adev->vcn.num_enc_rings = 2;
104 vcn_v2_5_set_dec_ring_funcs(adev);
105 vcn_v2_5_set_enc_ring_funcs(adev);
106 vcn_v2_5_set_irq_funcs(adev);
112 * vcn_v2_5_sw_init - sw init for VCN block
114 * @handle: amdgpu_device pointer
116 * Load firmware and sw initialization
118 static int vcn_v2_5_sw_init(void *handle)
120 struct amdgpu_ring *ring;
122 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
124 for (j = 0; j < adev->vcn.num_vcn_inst; j++) {
125 if (adev->vcn.harvest_config & (1 << j))
128 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
129 VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[j].irq);
134 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
135 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
136 i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[j].irq);
142 r = amdgpu_vcn_sw_init(adev);
146 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
147 const struct common_firmware_header *hdr;
148 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
149 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN;
150 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw;
151 adev->firmware.fw_size +=
152 ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
154 if (adev->vcn.num_vcn_inst == VCN25_MAX_HW_INSTANCES_ARCTURUS) {
155 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN1].ucode_id = AMDGPU_UCODE_ID_VCN1;
156 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN1].fw = adev->vcn.fw;
157 adev->firmware.fw_size +=
158 ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
160 DRM_INFO("PSP loading VCN firmware\n");
163 r = amdgpu_vcn_resume(adev);
167 for (j = 0; j < adev->vcn.num_vcn_inst; j++) {
168 volatile struct amdgpu_fw_shared *fw_shared;
170 if (adev->vcn.harvest_config & (1 << j))
172 adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
173 adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
174 adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
175 adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
176 adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
177 adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
179 adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
180 adev->vcn.inst[j].external.scratch9 = SOC15_REG_OFFSET(VCN, j, mmUVD_SCRATCH9);
181 adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
182 adev->vcn.inst[j].external.data0 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA0);
183 adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
184 adev->vcn.inst[j].external.data1 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA1);
185 adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
186 adev->vcn.inst[j].external.cmd = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_CMD);
187 adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
188 adev->vcn.inst[j].external.nop = SOC15_REG_OFFSET(VCN, j, mmUVD_NO_OP);
190 ring = &adev->vcn.inst[j].ring_dec;
191 ring->use_doorbell = true;
193 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
194 (amdgpu_sriov_vf(adev) ? 2*j : 8*j);
195 sprintf(ring->name, "vcn_dec_%d", j);
196 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[j].irq,
197 0, AMDGPU_RING_PRIO_DEFAULT);
201 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
202 ring = &adev->vcn.inst[j].ring_enc[i];
203 ring->use_doorbell = true;
205 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
206 (amdgpu_sriov_vf(adev) ? (1 + i + 2*j) : (2 + i + 8*j));
208 sprintf(ring->name, "vcn_enc_%d.%d", j, i);
209 r = amdgpu_ring_init(adev, ring, 512,
210 &adev->vcn.inst[j].irq, 0,
211 AMDGPU_RING_PRIO_DEFAULT);
216 fw_shared = adev->vcn.inst[j].fw_shared_cpu_addr;
217 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG);
220 if (amdgpu_sriov_vf(adev)) {
221 r = amdgpu_virt_alloc_mm_table(adev);
226 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
227 adev->vcn.pause_dpg_mode = vcn_v2_5_pause_dpg_mode;
233 * vcn_v2_5_sw_fini - sw fini for VCN block
235 * @handle: amdgpu_device pointer
237 * VCN suspend and free up sw allocation
239 static int vcn_v2_5_sw_fini(void *handle)
242 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
243 volatile struct amdgpu_fw_shared *fw_shared;
245 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
246 if (adev->vcn.harvest_config & (1 << i))
248 fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
249 fw_shared->present_flag_0 = 0;
252 if (amdgpu_sriov_vf(adev))
253 amdgpu_virt_free_mm_table(adev);
255 r = amdgpu_vcn_suspend(adev);
259 r = amdgpu_vcn_sw_fini(adev);
265 * vcn_v2_5_hw_init - start and test VCN block
267 * @handle: amdgpu_device pointer
269 * Initialize the hardware, boot up the VCPU and do some testing
271 static int vcn_v2_5_hw_init(void *handle)
273 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
274 struct amdgpu_ring *ring;
277 if (amdgpu_sriov_vf(adev))
278 r = vcn_v2_5_sriov_start(adev);
280 for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {
281 if (adev->vcn.harvest_config & (1 << j))
284 if (amdgpu_sriov_vf(adev)) {
285 adev->vcn.inst[j].ring_enc[0].sched.ready = true;
286 adev->vcn.inst[j].ring_enc[1].sched.ready = false;
287 adev->vcn.inst[j].ring_enc[2].sched.ready = false;
288 adev->vcn.inst[j].ring_dec.sched.ready = true;
291 ring = &adev->vcn.inst[j].ring_dec;
293 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
294 ring->doorbell_index, j);
296 r = amdgpu_ring_test_helper(ring);
300 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
301 ring = &adev->vcn.inst[j].ring_enc[i];
302 r = amdgpu_ring_test_helper(ring);
311 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
312 (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
318 * vcn_v2_5_hw_fini - stop the hardware block
320 * @handle: amdgpu_device pointer
322 * Stop the VCN block, mark ring as not ready any more
324 static int vcn_v2_5_hw_fini(void *handle)
326 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
329 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
330 if (adev->vcn.harvest_config & (1 << i))
333 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
334 (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
335 RREG32_SOC15(VCN, i, mmUVD_STATUS)))
336 vcn_v2_5_set_powergating_state(adev, AMD_PG_STATE_GATE);
343 * vcn_v2_5_suspend - suspend VCN block
345 * @handle: amdgpu_device pointer
347 * HW fini and suspend VCN block
349 static int vcn_v2_5_suspend(void *handle)
352 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
354 r = vcn_v2_5_hw_fini(adev);
358 r = amdgpu_vcn_suspend(adev);
364 * vcn_v2_5_resume - resume VCN block
366 * @handle: amdgpu_device pointer
368 * Resume firmware and hw init VCN block
370 static int vcn_v2_5_resume(void *handle)
373 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
375 r = amdgpu_vcn_resume(adev);
379 r = vcn_v2_5_hw_init(adev);
385 * vcn_v2_5_mc_resume - memory controller programming
387 * @adev: amdgpu_device pointer
389 * Let the VCN memory controller know it's offsets
391 static void vcn_v2_5_mc_resume(struct amdgpu_device *adev)
393 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
397 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
398 if (adev->vcn.harvest_config & (1 << i))
400 /* cache window 0: fw */
401 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
402 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
403 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo));
404 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
405 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi));
406 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0, 0);
409 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
410 lower_32_bits(adev->vcn.inst[i].gpu_addr));
411 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
412 upper_32_bits(adev->vcn.inst[i].gpu_addr));
414 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0,
415 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
417 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE0, size);
419 /* cache window 1: stack */
420 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
421 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset));
422 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
423 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset));
424 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET1, 0);
425 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
427 /* cache window 2: context */
428 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
429 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
430 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
431 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
432 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET2, 0);
433 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
435 /* non-cache window */
436 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
437 lower_32_bits(adev->vcn.inst[i].fw_shared_gpu_addr));
438 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
439 upper_32_bits(adev->vcn.inst[i].fw_shared_gpu_addr));
440 WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
441 WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_SIZE0,
442 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
446 static void vcn_v2_5_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
448 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
451 /* cache window 0: fw */
452 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
454 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
455 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
456 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
457 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
458 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
459 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
460 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
461 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
463 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
464 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
465 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
466 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
467 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
468 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
472 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
473 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
474 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
475 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
476 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
477 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
479 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
480 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0),
481 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
485 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
486 VCN, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
488 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
489 VCN, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
491 /* cache window 1: stack */
493 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
494 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
495 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
496 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
497 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
498 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
499 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
500 VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
502 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
503 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
504 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
505 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
506 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
507 VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
509 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
510 VCN, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
512 /* cache window 2: context */
513 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
514 VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
515 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
516 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
517 VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
518 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
519 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
520 VCN, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
521 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
522 VCN, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
524 /* non-cache window */
525 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
526 VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
527 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared_gpu_addr), 0, indirect);
528 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
529 VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
530 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared_gpu_addr), 0, indirect);
531 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
532 VCN, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
533 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
534 VCN, 0, mmUVD_VCPU_NONCACHE_SIZE0),
535 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
537 /* VCN global tiling registers */
538 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
539 VCN, 0, mmUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
543 * vcn_v2_5_disable_clock_gating - disable VCN clock gating
545 * @adev: amdgpu_device pointer
547 * Disable clock gating for VCN block
549 static void vcn_v2_5_disable_clock_gating(struct amdgpu_device *adev)
555 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
556 if (adev->vcn.harvest_config & (1 << i))
558 /* UVD disable CGC */
559 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
560 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
561 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
563 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
564 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
565 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
566 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
568 data = RREG32_SOC15(VCN, i, mmUVD_CGC_GATE);
569 data &= ~(UVD_CGC_GATE__SYS_MASK
570 | UVD_CGC_GATE__UDEC_MASK
571 | UVD_CGC_GATE__MPEG2_MASK
572 | UVD_CGC_GATE__REGS_MASK
573 | UVD_CGC_GATE__RBC_MASK
574 | UVD_CGC_GATE__LMI_MC_MASK
575 | UVD_CGC_GATE__LMI_UMC_MASK
576 | UVD_CGC_GATE__IDCT_MASK
577 | UVD_CGC_GATE__MPRD_MASK
578 | UVD_CGC_GATE__MPC_MASK
579 | UVD_CGC_GATE__LBSI_MASK
580 | UVD_CGC_GATE__LRBBM_MASK
581 | UVD_CGC_GATE__UDEC_RE_MASK
582 | UVD_CGC_GATE__UDEC_CM_MASK
583 | UVD_CGC_GATE__UDEC_IT_MASK
584 | UVD_CGC_GATE__UDEC_DB_MASK
585 | UVD_CGC_GATE__UDEC_MP_MASK
586 | UVD_CGC_GATE__WCB_MASK
587 | UVD_CGC_GATE__VCPU_MASK
588 | UVD_CGC_GATE__MMSCH_MASK);
590 WREG32_SOC15(VCN, i, mmUVD_CGC_GATE, data);
592 SOC15_WAIT_ON_RREG(VCN, i, mmUVD_CGC_GATE, 0, 0xFFFFFFFF, ret);
594 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
595 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
596 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
597 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
598 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
599 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
600 | UVD_CGC_CTRL__SYS_MODE_MASK
601 | UVD_CGC_CTRL__UDEC_MODE_MASK
602 | UVD_CGC_CTRL__MPEG2_MODE_MASK
603 | UVD_CGC_CTRL__REGS_MODE_MASK
604 | UVD_CGC_CTRL__RBC_MODE_MASK
605 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
606 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
607 | UVD_CGC_CTRL__IDCT_MODE_MASK
608 | UVD_CGC_CTRL__MPRD_MODE_MASK
609 | UVD_CGC_CTRL__MPC_MODE_MASK
610 | UVD_CGC_CTRL__LBSI_MODE_MASK
611 | UVD_CGC_CTRL__LRBBM_MODE_MASK
612 | UVD_CGC_CTRL__WCB_MODE_MASK
613 | UVD_CGC_CTRL__VCPU_MODE_MASK
614 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
615 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
618 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE);
619 data |= (UVD_SUVD_CGC_GATE__SRE_MASK
620 | UVD_SUVD_CGC_GATE__SIT_MASK
621 | UVD_SUVD_CGC_GATE__SMP_MASK
622 | UVD_SUVD_CGC_GATE__SCM_MASK
623 | UVD_SUVD_CGC_GATE__SDB_MASK
624 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
625 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
626 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
627 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
628 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
629 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
630 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
631 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
632 | UVD_SUVD_CGC_GATE__SCLR_MASK
633 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
634 | UVD_SUVD_CGC_GATE__ENT_MASK
635 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
636 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
637 | UVD_SUVD_CGC_GATE__SITE_MASK
638 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
639 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
640 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
641 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
642 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
643 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE, data);
645 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL);
646 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
647 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
648 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
649 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
650 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
651 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
652 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
653 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
654 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
655 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
656 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data);
660 static void vcn_v2_5_clock_gating_dpg_mode(struct amdgpu_device *adev,
661 uint8_t sram_sel, int inst_idx, uint8_t indirect)
663 uint32_t reg_data = 0;
665 /* enable sw clock gating control */
666 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
667 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
669 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
670 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
671 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
672 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
673 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
674 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
675 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
676 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
677 UVD_CGC_CTRL__SYS_MODE_MASK |
678 UVD_CGC_CTRL__UDEC_MODE_MASK |
679 UVD_CGC_CTRL__MPEG2_MODE_MASK |
680 UVD_CGC_CTRL__REGS_MODE_MASK |
681 UVD_CGC_CTRL__RBC_MODE_MASK |
682 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
683 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
684 UVD_CGC_CTRL__IDCT_MODE_MASK |
685 UVD_CGC_CTRL__MPRD_MODE_MASK |
686 UVD_CGC_CTRL__MPC_MODE_MASK |
687 UVD_CGC_CTRL__LBSI_MODE_MASK |
688 UVD_CGC_CTRL__LRBBM_MODE_MASK |
689 UVD_CGC_CTRL__WCB_MODE_MASK |
690 UVD_CGC_CTRL__VCPU_MODE_MASK |
691 UVD_CGC_CTRL__MMSCH_MODE_MASK);
692 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
693 VCN, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
695 /* turn off clock gating */
696 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
697 VCN, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect);
699 /* turn on SUVD clock gating */
700 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
701 VCN, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
703 /* turn on sw mode in UVD_SUVD_CGC_CTRL */
704 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
705 VCN, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
709 * vcn_v2_5_enable_clock_gating - enable VCN clock gating
711 * @adev: amdgpu_device pointer
713 * Enable clock gating for VCN block
715 static void vcn_v2_5_enable_clock_gating(struct amdgpu_device *adev)
720 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
721 if (adev->vcn.harvest_config & (1 << i))
724 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
725 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
726 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
728 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
729 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
730 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
731 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
733 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
734 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
735 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
736 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
737 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
738 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
739 | UVD_CGC_CTRL__SYS_MODE_MASK
740 | UVD_CGC_CTRL__UDEC_MODE_MASK
741 | UVD_CGC_CTRL__MPEG2_MODE_MASK
742 | UVD_CGC_CTRL__REGS_MODE_MASK
743 | UVD_CGC_CTRL__RBC_MODE_MASK
744 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
745 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
746 | UVD_CGC_CTRL__IDCT_MODE_MASK
747 | UVD_CGC_CTRL__MPRD_MODE_MASK
748 | UVD_CGC_CTRL__MPC_MODE_MASK
749 | UVD_CGC_CTRL__LBSI_MODE_MASK
750 | UVD_CGC_CTRL__LRBBM_MODE_MASK
751 | UVD_CGC_CTRL__WCB_MODE_MASK
752 | UVD_CGC_CTRL__VCPU_MODE_MASK);
753 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
755 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL);
756 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
757 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
758 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
759 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
760 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
761 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
762 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
763 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
764 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
765 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
766 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data);
770 static int vcn_v2_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
772 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared_cpu_addr;
773 struct amdgpu_ring *ring;
774 uint32_t rb_bufsz, tmp;
776 /* disable register anti-hang mechanism */
777 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
778 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
779 /* enable dynamic power gating mode */
780 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
781 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
782 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
783 WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
786 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t*)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
788 /* enable clock gating */
789 vcn_v2_5_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
791 /* enable VCPU clock */
792 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
793 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
794 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
795 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
796 VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
798 /* disable master interupt */
799 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
800 VCN, 0, mmUVD_MASTINT_EN), 0, 0, indirect);
802 /* setup mmUVD_LMI_CTRL */
803 tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
804 UVD_LMI_CTRL__REQ_MODE_MASK |
805 UVD_LMI_CTRL__CRC_RESET_MASK |
806 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
807 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
808 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
809 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
811 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
812 VCN, 0, mmUVD_LMI_CTRL), tmp, 0, indirect);
814 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
815 VCN, 0, mmUVD_MPC_CNTL),
816 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
818 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
819 VCN, 0, mmUVD_MPC_SET_MUXA0),
820 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
821 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
822 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
823 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
825 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
826 VCN, 0, mmUVD_MPC_SET_MUXB0),
827 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
828 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
829 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
830 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
832 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
833 VCN, 0, mmUVD_MPC_SET_MUX),
834 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
835 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
836 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
838 vcn_v2_5_mc_resume_dpg_mode(adev, inst_idx, indirect);
840 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
841 VCN, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
842 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
843 VCN, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
845 /* enable LMI MC and UMC channels */
846 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
847 VCN, 0, mmUVD_LMI_CTRL2), 0, 0, indirect);
849 /* unblock VCPU register access */
850 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
851 VCN, 0, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
853 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
854 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
855 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
856 VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
858 /* enable master interrupt */
859 WREG32_SOC15_DPG_MODE_2_0(inst_idx, SOC15_DPG_MODE_OFFSET_2_0(
860 VCN, 0, mmUVD_MASTINT_EN),
861 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
864 psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
865 (uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
866 (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr));
868 ring = &adev->vcn.inst[inst_idx].ring_dec;
869 /* force RBC into idle state */
870 rb_bufsz = order_base_2(ring->ring_size);
871 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
872 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
873 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
874 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
875 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
876 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
878 /* Stall DPG before WPTR/RPTR reset */
879 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
880 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
881 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
882 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
884 /* set the write pointer delay */
885 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
887 /* set the wb address */
888 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
889 (upper_32_bits(ring->gpu_addr) >> 2));
891 /* programm the RB_BASE for ring buffer */
892 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
893 lower_32_bits(ring->gpu_addr));
894 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
895 upper_32_bits(ring->gpu_addr));
897 /* Initialize the ring buffer's read and write pointers */
898 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
900 WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
902 ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
903 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
904 lower_32_bits(ring->wptr));
906 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
908 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
909 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
914 static int vcn_v2_5_start(struct amdgpu_device *adev)
916 struct amdgpu_ring *ring;
917 uint32_t rb_bufsz, tmp;
920 if (adev->pm.dpm_enabled)
921 amdgpu_dpm_enable_uvd(adev, true);
923 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
924 if (adev->vcn.harvest_config & (1 << i))
926 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
927 r = vcn_v2_5_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
931 /* disable register anti-hang mechanism */
932 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS), 0,
933 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
935 /* set uvd status busy */
936 tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
937 WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
940 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
944 vcn_v2_5_disable_clock_gating(adev);
946 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
947 if (adev->vcn.harvest_config & (1 << i))
949 /* enable VCPU clock */
950 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
951 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
953 /* disable master interrupt */
954 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
955 ~UVD_MASTINT_EN__VCPU_EN_MASK);
957 /* setup mmUVD_LMI_CTRL */
958 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
960 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp | 0x8|
961 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
962 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
963 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
964 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
966 /* setup mmUVD_MPC_CNTL */
967 tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
968 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
969 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
970 WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
972 /* setup UVD_MPC_SET_MUXA0 */
973 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
974 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
975 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
976 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
977 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
979 /* setup UVD_MPC_SET_MUXB0 */
980 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
981 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
982 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
983 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
984 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
986 /* setup mmUVD_MPC_SET_MUX */
987 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
988 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
989 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
990 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
993 vcn_v2_5_mc_resume(adev);
995 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
996 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[i].fw_shared_cpu_addr;
997 if (adev->vcn.harvest_config & (1 << i))
999 /* VCN global tiling registers */
1000 WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG,
1001 adev->gfx.config.gb_addr_config);
1002 WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG,
1003 adev->gfx.config.gb_addr_config);
1005 /* enable LMI MC and UMC channels */
1006 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1007 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1009 /* unblock VCPU register access */
1010 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1011 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1013 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1014 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1016 for (k = 0; k < 10; ++k) {
1019 for (j = 0; j < 100; ++j) {
1020 status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1023 if (amdgpu_emu_mode == 1)
1032 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
1033 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1034 UVD_VCPU_CNTL__BLK_RST_MASK,
1035 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1037 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1038 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1045 DRM_ERROR("VCN decode not responding, giving up!!!\n");
1049 /* enable master interrupt */
1050 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1051 UVD_MASTINT_EN__VCPU_EN_MASK,
1052 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1054 /* clear the busy bit of VCN_STATUS */
1055 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1056 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1058 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1060 ring = &adev->vcn.inst[i].ring_dec;
1061 /* force RBC into idle state */
1062 rb_bufsz = order_base_2(ring->ring_size);
1063 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1064 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1065 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1066 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1067 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1068 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1070 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
1071 /* programm the RB_BASE for ring buffer */
1072 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1073 lower_32_bits(ring->gpu_addr));
1074 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1075 upper_32_bits(ring->gpu_addr));
1077 /* Initialize the ring buffer's read and write pointers */
1078 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1080 ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1081 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1082 lower_32_bits(ring->wptr));
1083 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
1085 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1086 ring = &adev->vcn.inst[i].ring_enc[0];
1087 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1088 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1089 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1090 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1091 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1092 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1094 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1095 ring = &adev->vcn.inst[i].ring_enc[1];
1096 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1097 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1098 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1099 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1100 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1101 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1107 static int vcn_v2_5_mmsch_start(struct amdgpu_device *adev,
1108 struct amdgpu_mm_table *table)
1110 uint32_t data = 0, loop = 0, size = 0;
1111 uint64_t addr = table->gpu_addr;
1112 struct mmsch_v1_1_init_header *header = NULL;;
1114 header = (struct mmsch_v1_1_init_header *)table->cpu_addr;
1115 size = header->total_size;
1118 * 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of
1119 * memory descriptor location
1121 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr));
1122 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr));
1124 /* 2, update vmid of descriptor */
1125 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1126 data &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1127 /* use domain0 for MM scheduler */
1128 data |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1129 WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, data);
1131 /* 3, notify mmsch about the size of this descriptor */
1132 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1134 /* 4, set resp to zero */
1135 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1138 * 5, kick off the initialization and wait until
1139 * VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero
1141 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, 0x10000001);
1143 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1145 while ((data & 0x10000002) != 0x10000002) {
1147 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1155 "failed to init MMSCH, mmMMSCH_VF_MAILBOX_RESP = %x\n",
1163 static int vcn_v2_5_sriov_start(struct amdgpu_device *adev)
1165 struct amdgpu_ring *ring;
1166 uint32_t offset, size, tmp, i, rb_bufsz;
1167 uint32_t table_size = 0;
1168 struct mmsch_v1_0_cmd_direct_write direct_wt = { { 0 } };
1169 struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { { 0 } };
1170 struct mmsch_v1_0_cmd_end end = { { 0 } };
1171 uint32_t *init_table = adev->virt.mm_table.cpu_addr;
1172 struct mmsch_v1_1_init_header *header = (struct mmsch_v1_1_init_header *)init_table;
1174 direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
1175 direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1176 end.cmd_header.command_type = MMSCH_COMMAND__END;
1178 header->version = MMSCH_VERSION;
1179 header->total_size = sizeof(struct mmsch_v1_1_init_header) >> 2;
1180 init_table += header->total_size;
1182 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1183 header->eng[i].table_offset = header->total_size;
1184 header->eng[i].init_status = 0;
1185 header->eng[i].table_size = 0;
1189 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(
1190 SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS),
1191 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1193 size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1195 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1196 MMSCH_V1_0_INSERT_DIRECT_WT(
1197 SOC15_REG_OFFSET(VCN, i,
1198 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1199 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1200 MMSCH_V1_0_INSERT_DIRECT_WT(
1201 SOC15_REG_OFFSET(VCN, i,
1202 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1203 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1205 MMSCH_V1_0_INSERT_DIRECT_WT(
1206 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0), 0);
1208 MMSCH_V1_0_INSERT_DIRECT_WT(
1209 SOC15_REG_OFFSET(VCN, i,
1210 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1211 lower_32_bits(adev->vcn.inst[i].gpu_addr));
1212 MMSCH_V1_0_INSERT_DIRECT_WT(
1213 SOC15_REG_OFFSET(VCN, i,
1214 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1215 upper_32_bits(adev->vcn.inst[i].gpu_addr));
1217 MMSCH_V1_0_INSERT_DIRECT_WT(
1218 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0),
1219 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1222 MMSCH_V1_0_INSERT_DIRECT_WT(
1223 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE0),
1225 MMSCH_V1_0_INSERT_DIRECT_WT(
1226 SOC15_REG_OFFSET(VCN, i,
1227 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1228 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset));
1229 MMSCH_V1_0_INSERT_DIRECT_WT(
1230 SOC15_REG_OFFSET(VCN, i,
1231 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1232 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset));
1233 MMSCH_V1_0_INSERT_DIRECT_WT(
1234 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET1),
1236 MMSCH_V1_0_INSERT_DIRECT_WT(
1237 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE1),
1238 AMDGPU_VCN_STACK_SIZE);
1239 MMSCH_V1_0_INSERT_DIRECT_WT(
1240 SOC15_REG_OFFSET(VCN, i,
1241 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1242 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset +
1243 AMDGPU_VCN_STACK_SIZE));
1244 MMSCH_V1_0_INSERT_DIRECT_WT(
1245 SOC15_REG_OFFSET(VCN, i,
1246 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1247 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset +
1248 AMDGPU_VCN_STACK_SIZE));
1249 MMSCH_V1_0_INSERT_DIRECT_WT(
1250 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET2),
1252 MMSCH_V1_0_INSERT_DIRECT_WT(
1253 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE2),
1254 AMDGPU_VCN_CONTEXT_SIZE);
1256 ring = &adev->vcn.inst[i].ring_enc[0];
1259 MMSCH_V1_0_INSERT_DIRECT_WT(
1260 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_LO),
1261 lower_32_bits(ring->gpu_addr));
1262 MMSCH_V1_0_INSERT_DIRECT_WT(
1263 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_HI),
1264 upper_32_bits(ring->gpu_addr));
1265 MMSCH_V1_0_INSERT_DIRECT_WT(
1266 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_SIZE),
1267 ring->ring_size / 4);
1269 ring = &adev->vcn.inst[i].ring_dec;
1271 MMSCH_V1_0_INSERT_DIRECT_WT(
1272 SOC15_REG_OFFSET(VCN, i,
1273 mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1274 lower_32_bits(ring->gpu_addr));
1275 MMSCH_V1_0_INSERT_DIRECT_WT(
1276 SOC15_REG_OFFSET(VCN, i,
1277 mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1278 upper_32_bits(ring->gpu_addr));
1280 /* force RBC into idle state */
1281 rb_bufsz = order_base_2(ring->ring_size);
1282 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1283 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1284 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1285 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1286 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1287 MMSCH_V1_0_INSERT_DIRECT_WT(
1288 SOC15_REG_OFFSET(VCN, i, mmUVD_RBC_RB_CNTL), tmp);
1290 /* add end packet */
1291 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
1292 table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
1293 init_table += sizeof(struct mmsch_v1_0_cmd_end) / 4;
1296 header->eng[i].table_size = table_size;
1297 header->total_size += table_size;
1300 return vcn_v2_5_mmsch_start(adev, &adev->virt.mm_table);
1303 static int vcn_v2_5_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1308 /* Wait for power status to be 1 */
1309 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1310 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1312 /* wait for read ptr to be equal to write ptr */
1313 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1314 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
1316 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1317 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF, ret_code);
1319 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1320 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
1322 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1323 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1325 /* disable dynamic power gating mode */
1326 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1327 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1332 static int vcn_v2_5_stop(struct amdgpu_device *adev)
1337 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1338 if (adev->vcn.harvest_config & (1 << i))
1340 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1341 r = vcn_v2_5_stop_dpg_mode(adev, i);
1345 /* wait for vcn idle */
1346 SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7, r);
1350 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1351 UVD_LMI_STATUS__READ_CLEAN_MASK |
1352 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1353 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1354 SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp, r);
1358 /* block LMI UMC channel */
1359 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1360 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1361 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1363 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1364 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1365 SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp, r);
1369 /* block VCPU register access */
1370 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1371 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1372 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1375 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1376 UVD_VCPU_CNTL__BLK_RST_MASK,
1377 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1379 /* disable VCPU clock */
1380 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1381 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1384 WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1386 vcn_v2_5_enable_clock_gating(adev);
1388 /* enable register anti-hang mechanism */
1389 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS),
1390 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK,
1391 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1394 if (adev->pm.dpm_enabled)
1395 amdgpu_dpm_enable_uvd(adev, false);
1400 static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev,
1401 int inst_idx, struct dpg_pause_state *new_state)
1403 struct amdgpu_ring *ring;
1404 uint32_t reg_data = 0;
1407 /* pause/unpause if state is changed */
1408 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1409 DRM_DEBUG("dpg pause state changed %d -> %d",
1410 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1411 reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1412 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1414 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1415 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1416 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1419 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared_cpu_addr;
1422 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1423 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1426 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1427 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1428 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK, ret_code);
1430 /* Stall DPG before WPTR/RPTR reset */
1431 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1432 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1433 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1436 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1437 ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1439 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1440 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1441 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1442 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1443 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1444 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1446 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1447 ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1449 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1450 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1451 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1452 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1453 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1454 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1457 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1458 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1460 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1461 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1464 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1465 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1466 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1467 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1469 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1476 * vcn_v2_5_dec_ring_get_rptr - get read pointer
1478 * @ring: amdgpu_ring pointer
1480 * Returns the current hardware read pointer
1482 static uint64_t vcn_v2_5_dec_ring_get_rptr(struct amdgpu_ring *ring)
1484 struct amdgpu_device *adev = ring->adev;
1486 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1490 * vcn_v2_5_dec_ring_get_wptr - get write pointer
1492 * @ring: amdgpu_ring pointer
1494 * Returns the current hardware write pointer
1496 static uint64_t vcn_v2_5_dec_ring_get_wptr(struct amdgpu_ring *ring)
1498 struct amdgpu_device *adev = ring->adev;
1500 if (ring->use_doorbell)
1501 return adev->wb.wb[ring->wptr_offs];
1503 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1507 * vcn_v2_5_dec_ring_set_wptr - set write pointer
1509 * @ring: amdgpu_ring pointer
1511 * Commits the write pointer to the hardware
1513 static void vcn_v2_5_dec_ring_set_wptr(struct amdgpu_ring *ring)
1515 struct amdgpu_device *adev = ring->adev;
1517 if (ring->use_doorbell) {
1518 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1519 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1521 WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1525 static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = {
1526 .type = AMDGPU_RING_TYPE_VCN_DEC,
1528 .vmhub = AMDGPU_MMHUB_1,
1529 .get_rptr = vcn_v2_5_dec_ring_get_rptr,
1530 .get_wptr = vcn_v2_5_dec_ring_get_wptr,
1531 .set_wptr = vcn_v2_5_dec_ring_set_wptr,
1533 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1534 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1535 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1536 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1538 .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1539 .emit_ib = vcn_v2_0_dec_ring_emit_ib,
1540 .emit_fence = vcn_v2_0_dec_ring_emit_fence,
1541 .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1542 .test_ring = vcn_v2_0_dec_ring_test_ring,
1543 .test_ib = amdgpu_vcn_dec_ring_test_ib,
1544 .insert_nop = vcn_v2_0_dec_ring_insert_nop,
1545 .insert_start = vcn_v2_0_dec_ring_insert_start,
1546 .insert_end = vcn_v2_0_dec_ring_insert_end,
1547 .pad_ib = amdgpu_ring_generic_pad_ib,
1548 .begin_use = amdgpu_vcn_ring_begin_use,
1549 .end_use = amdgpu_vcn_ring_end_use,
1550 .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1551 .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1552 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1556 * vcn_v2_5_enc_ring_get_rptr - get enc read pointer
1558 * @ring: amdgpu_ring pointer
1560 * Returns the current hardware enc read pointer
1562 static uint64_t vcn_v2_5_enc_ring_get_rptr(struct amdgpu_ring *ring)
1564 struct amdgpu_device *adev = ring->adev;
1566 if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1567 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1569 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1573 * vcn_v2_5_enc_ring_get_wptr - get enc write pointer
1575 * @ring: amdgpu_ring pointer
1577 * Returns the current hardware enc write pointer
1579 static uint64_t vcn_v2_5_enc_ring_get_wptr(struct amdgpu_ring *ring)
1581 struct amdgpu_device *adev = ring->adev;
1583 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1584 if (ring->use_doorbell)
1585 return adev->wb.wb[ring->wptr_offs];
1587 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1589 if (ring->use_doorbell)
1590 return adev->wb.wb[ring->wptr_offs];
1592 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
1597 * vcn_v2_5_enc_ring_set_wptr - set enc write pointer
1599 * @ring: amdgpu_ring pointer
1601 * Commits the enc write pointer to the hardware
1603 static void vcn_v2_5_enc_ring_set_wptr(struct amdgpu_ring *ring)
1605 struct amdgpu_device *adev = ring->adev;
1607 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1608 if (ring->use_doorbell) {
1609 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1610 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1612 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1615 if (ring->use_doorbell) {
1616 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
1617 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1619 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1624 static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = {
1625 .type = AMDGPU_RING_TYPE_VCN_ENC,
1627 .nop = VCN_ENC_CMD_NO_OP,
1628 .vmhub = AMDGPU_MMHUB_1,
1629 .get_rptr = vcn_v2_5_enc_ring_get_rptr,
1630 .get_wptr = vcn_v2_5_enc_ring_get_wptr,
1631 .set_wptr = vcn_v2_5_enc_ring_set_wptr,
1633 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1634 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1635 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1636 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1637 1, /* vcn_v2_0_enc_ring_insert_end */
1638 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1639 .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1640 .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1641 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1642 .test_ring = amdgpu_vcn_enc_ring_test_ring,
1643 .test_ib = amdgpu_vcn_enc_ring_test_ib,
1644 .insert_nop = amdgpu_ring_insert_nop,
1645 .insert_end = vcn_v2_0_enc_ring_insert_end,
1646 .pad_ib = amdgpu_ring_generic_pad_ib,
1647 .begin_use = amdgpu_vcn_ring_begin_use,
1648 .end_use = amdgpu_vcn_ring_end_use,
1649 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1650 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1651 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1654 static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev)
1658 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1659 if (adev->vcn.harvest_config & (1 << i))
1661 adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_5_dec_ring_vm_funcs;
1662 adev->vcn.inst[i].ring_dec.me = i;
1663 DRM_INFO("VCN(%d) decode is enabled in VM mode\n", i);
1667 static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev)
1671 for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {
1672 if (adev->vcn.harvest_config & (1 << j))
1674 for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
1675 adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_5_enc_ring_vm_funcs;
1676 adev->vcn.inst[j].ring_enc[i].me = j;
1678 DRM_INFO("VCN(%d) encode is enabled in VM mode\n", j);
1682 static bool vcn_v2_5_is_idle(void *handle)
1684 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1687 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1688 if (adev->vcn.harvest_config & (1 << i))
1690 ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
1696 static int vcn_v2_5_wait_for_idle(void *handle)
1698 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1701 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1702 if (adev->vcn.harvest_config & (1 << i))
1704 SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
1705 UVD_STATUS__IDLE, ret);
1713 static int vcn_v2_5_set_clockgating_state(void *handle,
1714 enum amd_clockgating_state state)
1716 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1717 bool enable = (state == AMD_CG_STATE_GATE);
1719 if (amdgpu_sriov_vf(adev))
1723 if (!vcn_v2_5_is_idle(handle))
1725 vcn_v2_5_enable_clock_gating(adev);
1727 vcn_v2_5_disable_clock_gating(adev);
1733 static int vcn_v2_5_set_powergating_state(void *handle,
1734 enum amd_powergating_state state)
1736 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1739 if (amdgpu_sriov_vf(adev))
1742 if(state == adev->vcn.cur_state)
1745 if (state == AMD_PG_STATE_GATE)
1746 ret = vcn_v2_5_stop(adev);
1748 ret = vcn_v2_5_start(adev);
1751 adev->vcn.cur_state = state;
1756 static int vcn_v2_5_set_interrupt_state(struct amdgpu_device *adev,
1757 struct amdgpu_irq_src *source,
1759 enum amdgpu_interrupt_state state)
1764 static int vcn_v2_5_process_interrupt(struct amdgpu_device *adev,
1765 struct amdgpu_irq_src *source,
1766 struct amdgpu_iv_entry *entry)
1768 uint32_t ip_instance;
1770 switch (entry->client_id) {
1771 case SOC15_IH_CLIENTID_VCN:
1774 case SOC15_IH_CLIENTID_VCN1:
1778 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
1782 DRM_DEBUG("IH: VCN TRAP\n");
1784 switch (entry->src_id) {
1785 case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
1786 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
1788 case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1789 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
1791 case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
1792 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
1795 DRM_ERROR("Unhandled interrupt: %d %d\n",
1796 entry->src_id, entry->src_data[0]);
1803 static const struct amdgpu_irq_src_funcs vcn_v2_5_irq_funcs = {
1804 .set = vcn_v2_5_set_interrupt_state,
1805 .process = vcn_v2_5_process_interrupt,
1808 static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev)
1812 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1813 if (adev->vcn.harvest_config & (1 << i))
1815 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
1816 adev->vcn.inst[i].irq.funcs = &vcn_v2_5_irq_funcs;
1820 static const struct amd_ip_funcs vcn_v2_5_ip_funcs = {
1822 .early_init = vcn_v2_5_early_init,
1824 .sw_init = vcn_v2_5_sw_init,
1825 .sw_fini = vcn_v2_5_sw_fini,
1826 .hw_init = vcn_v2_5_hw_init,
1827 .hw_fini = vcn_v2_5_hw_fini,
1828 .suspend = vcn_v2_5_suspend,
1829 .resume = vcn_v2_5_resume,
1830 .is_idle = vcn_v2_5_is_idle,
1831 .wait_for_idle = vcn_v2_5_wait_for_idle,
1832 .check_soft_reset = NULL,
1833 .pre_soft_reset = NULL,
1835 .post_soft_reset = NULL,
1836 .set_clockgating_state = vcn_v2_5_set_clockgating_state,
1837 .set_powergating_state = vcn_v2_5_set_powergating_state,
1840 const struct amdgpu_ip_block_version vcn_v2_5_ip_block =
1842 .type = AMD_IP_BLOCK_TYPE_VCN,
1846 .funcs = &vcn_v2_5_ip_funcs,