2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
28 #include <linux/kthread.h>
29 #include <linux/console.h>
30 #include <linux/slab.h>
31 #include <linux/debugfs.h>
33 #include <drm/drm_crtc_helper.h>
34 #include <drm/amdgpu_drm.h>
35 #include <linux/vgaarb.h>
36 #include <linux/vga_switcheroo.h>
37 #include <linux/efi.h>
39 #include "amdgpu_trace.h"
40 #include "amdgpu_i2c.h"
42 #include "amdgpu_atombios.h"
43 #include "amdgpu_atomfirmware.h"
45 #ifdef CONFIG_DRM_AMDGPU_SI
48 #ifdef CONFIG_DRM_AMDGPU_CIK
53 #include "bif/bif_4_1_d.h"
54 #include <linux/pci.h>
55 #include <linux/firmware.h>
56 #include "amdgpu_vf_error.h"
58 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
59 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
61 #define AMDGPU_RESUME_MS 2000
63 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
64 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
65 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev);
67 static const char *amdgpu_asic_name[] = {
91 bool amdgpu_device_is_px(struct drm_device *dev)
93 struct amdgpu_device *adev = dev->dev_private;
95 if (adev->flags & AMD_IS_PX)
101 * MMIO register access helper functions.
103 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
108 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
109 BUG_ON(in_interrupt());
110 return amdgpu_virt_kiq_rreg(adev, reg);
113 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
114 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
118 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
119 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
120 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
121 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
123 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
127 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
130 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
132 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
133 adev->last_mm_index = v;
136 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
137 BUG_ON(in_interrupt());
138 return amdgpu_virt_kiq_wreg(adev, reg, v);
141 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
142 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
146 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
147 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
148 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
149 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
152 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
157 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
159 if ((reg * 4) < adev->rio_mem_size)
160 return ioread32(adev->rio_mem + (reg * 4));
162 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
163 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
167 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
169 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
170 adev->last_mm_index = v;
173 if ((reg * 4) < adev->rio_mem_size)
174 iowrite32(v, adev->rio_mem + (reg * 4));
176 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
177 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
180 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
186 * amdgpu_mm_rdoorbell - read a doorbell dword
188 * @adev: amdgpu_device pointer
189 * @index: doorbell index
191 * Returns the value in the doorbell aperture at the
192 * requested doorbell index (CIK).
194 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
196 if (index < adev->doorbell.num_doorbells) {
197 return readl(adev->doorbell.ptr + index);
199 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
205 * amdgpu_mm_wdoorbell - write a doorbell dword
207 * @adev: amdgpu_device pointer
208 * @index: doorbell index
211 * Writes @v to the doorbell aperture at the
212 * requested doorbell index (CIK).
214 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
216 if (index < adev->doorbell.num_doorbells) {
217 writel(v, adev->doorbell.ptr + index);
219 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
224 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
226 * @adev: amdgpu_device pointer
227 * @index: doorbell index
229 * Returns the value in the doorbell aperture at the
230 * requested doorbell index (VEGA10+).
232 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
234 if (index < adev->doorbell.num_doorbells) {
235 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
237 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
243 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
245 * @adev: amdgpu_device pointer
246 * @index: doorbell index
249 * Writes @v to the doorbell aperture at the
250 * requested doorbell index (VEGA10+).
252 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
254 if (index < adev->doorbell.num_doorbells) {
255 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
257 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
262 * amdgpu_invalid_rreg - dummy reg read function
264 * @adev: amdgpu device pointer
265 * @reg: offset of register
267 * Dummy register read function. Used for register blocks
268 * that certain asics don't have (all asics).
269 * Returns the value in the register.
271 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
273 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
279 * amdgpu_invalid_wreg - dummy reg write function
281 * @adev: amdgpu device pointer
282 * @reg: offset of register
283 * @v: value to write to the register
285 * Dummy register read function. Used for register blocks
286 * that certain asics don't have (all asics).
288 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
290 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
296 * amdgpu_block_invalid_rreg - dummy reg read function
298 * @adev: amdgpu device pointer
299 * @block: offset of instance
300 * @reg: offset of register
302 * Dummy register read function. Used for register blocks
303 * that certain asics don't have (all asics).
304 * Returns the value in the register.
306 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
307 uint32_t block, uint32_t reg)
309 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
316 * amdgpu_block_invalid_wreg - dummy reg write function
318 * @adev: amdgpu device pointer
319 * @block: offset of instance
320 * @reg: offset of register
321 * @v: value to write to the register
323 * Dummy register read function. Used for register blocks
324 * that certain asics don't have (all asics).
326 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
328 uint32_t reg, uint32_t v)
330 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
335 static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
339 if (adev->vram_scratch.robj == NULL) {
340 r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
341 PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
342 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
343 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
344 NULL, NULL, &adev->vram_scratch.robj);
350 r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
351 if (unlikely(r != 0))
353 r = amdgpu_bo_pin(adev->vram_scratch.robj,
354 AMDGPU_GEM_DOMAIN_VRAM, &adev->vram_scratch.gpu_addr);
356 amdgpu_bo_unreserve(adev->vram_scratch.robj);
359 r = amdgpu_bo_kmap(adev->vram_scratch.robj,
360 (void **)&adev->vram_scratch.ptr);
362 amdgpu_bo_unpin(adev->vram_scratch.robj);
363 amdgpu_bo_unreserve(adev->vram_scratch.robj);
368 static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
372 if (adev->vram_scratch.robj == NULL) {
375 r = amdgpu_bo_reserve(adev->vram_scratch.robj, true);
376 if (likely(r == 0)) {
377 amdgpu_bo_kunmap(adev->vram_scratch.robj);
378 amdgpu_bo_unpin(adev->vram_scratch.robj);
379 amdgpu_bo_unreserve(adev->vram_scratch.robj);
381 amdgpu_bo_unref(&adev->vram_scratch.robj);
385 * amdgpu_program_register_sequence - program an array of registers.
387 * @adev: amdgpu_device pointer
388 * @registers: pointer to the register array
389 * @array_size: size of the register array
391 * Programs an array or registers with and and or masks.
392 * This is a helper for setting golden registers.
394 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
395 const u32 *registers,
396 const u32 array_size)
398 u32 tmp, reg, and_mask, or_mask;
404 for (i = 0; i < array_size; i +=3) {
405 reg = registers[i + 0];
406 and_mask = registers[i + 1];
407 or_mask = registers[i + 2];
409 if (and_mask == 0xffffffff) {
420 void amdgpu_pci_config_reset(struct amdgpu_device *adev)
422 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
426 * GPU doorbell aperture helpers function.
429 * amdgpu_doorbell_init - Init doorbell driver information.
431 * @adev: amdgpu_device pointer
433 * Init doorbell driver information (CIK)
434 * Returns 0 on success, error on failure.
436 static int amdgpu_doorbell_init(struct amdgpu_device *adev)
438 /* doorbell bar mapping */
439 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
440 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
442 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
443 AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
444 if (adev->doorbell.num_doorbells == 0)
447 adev->doorbell.ptr = ioremap(adev->doorbell.base,
448 adev->doorbell.num_doorbells *
450 if (adev->doorbell.ptr == NULL)
457 * amdgpu_doorbell_fini - Tear down doorbell driver information.
459 * @adev: amdgpu_device pointer
461 * Tear down doorbell driver information (CIK)
463 static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
465 iounmap(adev->doorbell.ptr);
466 adev->doorbell.ptr = NULL;
470 * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
473 * @adev: amdgpu_device pointer
474 * @aperture_base: output returning doorbell aperture base physical address
475 * @aperture_size: output returning doorbell aperture size in bytes
476 * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
478 * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
479 * takes doorbells required for its own rings and reports the setup to amdkfd.
480 * amdgpu reserved doorbells are at the start of the doorbell aperture.
482 void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
483 phys_addr_t *aperture_base,
484 size_t *aperture_size,
485 size_t *start_offset)
488 * The first num_doorbells are used by amdgpu.
489 * amdkfd takes whatever's left in the aperture.
491 if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
492 *aperture_base = adev->doorbell.base;
493 *aperture_size = adev->doorbell.size;
494 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
504 * Writeback is the method by which the GPU updates special pages in memory
505 * with the status of certain GPU events (fences, ring pointers,etc.).
509 * amdgpu_wb_fini - Disable Writeback and free memory
511 * @adev: amdgpu_device pointer
513 * Disables Writeback and frees the Writeback memory (all asics).
514 * Used at driver shutdown.
516 static void amdgpu_wb_fini(struct amdgpu_device *adev)
518 if (adev->wb.wb_obj) {
519 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
521 (void **)&adev->wb.wb);
522 adev->wb.wb_obj = NULL;
527 * amdgpu_wb_init- Init Writeback driver info and allocate memory
529 * @adev: amdgpu_device pointer
531 * Initializes writeback and allocates writeback memory (all asics).
532 * Used at driver startup.
533 * Returns 0 on success or an -error on failure.
535 static int amdgpu_wb_init(struct amdgpu_device *adev)
539 if (adev->wb.wb_obj == NULL) {
540 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t),
541 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
542 &adev->wb.wb_obj, &adev->wb.gpu_addr,
543 (void **)&adev->wb.wb);
545 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
549 adev->wb.num_wb = AMDGPU_MAX_WB;
550 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
552 /* clear wb memory */
553 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t));
560 * amdgpu_wb_get - Allocate a wb entry
562 * @adev: amdgpu_device pointer
565 * Allocate a wb slot for use by the driver (all asics).
566 * Returns 0 on success or -EINVAL on failure.
568 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
570 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
571 if (offset < adev->wb.num_wb) {
572 __set_bit(offset, adev->wb.used);
581 * amdgpu_wb_get_64bit - Allocate a wb entry
583 * @adev: amdgpu_device pointer
586 * Allocate a wb slot for use by the driver (all asics).
587 * Returns 0 on success or -EINVAL on failure.
589 int amdgpu_wb_get_64bit(struct amdgpu_device *adev, u32 *wb)
591 unsigned long offset = bitmap_find_next_zero_area_off(adev->wb.used,
592 adev->wb.num_wb, 0, 2, 7, 0);
593 if ((offset + 1) < adev->wb.num_wb) {
594 __set_bit(offset, adev->wb.used);
595 __set_bit(offset + 1, adev->wb.used);
604 * amdgpu_wb_free - Free a wb entry
606 * @adev: amdgpu_device pointer
609 * Free a wb slot allocated for use by the driver (all asics)
611 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
613 if (wb < adev->wb.num_wb)
614 __clear_bit(wb, adev->wb.used);
618 * amdgpu_wb_free_64bit - Free a wb entry
620 * @adev: amdgpu_device pointer
623 * Free a wb slot allocated for use by the driver (all asics)
625 void amdgpu_wb_free_64bit(struct amdgpu_device *adev, u32 wb)
627 if ((wb + 1) < adev->wb.num_wb) {
628 __clear_bit(wb, adev->wb.used);
629 __clear_bit(wb + 1, adev->wb.used);
634 * amdgpu_vram_location - try to find VRAM location
635 * @adev: amdgpu device structure holding all necessary informations
636 * @mc: memory controller structure holding memory informations
637 * @base: base address at which to put VRAM
639 * Function will try to place VRAM at base address provided
640 * as parameter (which is so far either PCI aperture address or
641 * for IGP TOM base address).
643 * If there is not enough space to fit the unvisible VRAM in the 32bits
644 * address space then we limit the VRAM size to the aperture.
646 * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
647 * this shouldn't be a problem as we are using the PCI aperture as a reference.
648 * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
651 * Note: we use mc_vram_size as on some board we need to program the mc to
652 * cover the whole aperture even if VRAM size is inferior to aperture size
653 * Novell bug 204882 + along with lots of ubuntu ones
655 * Note: when limiting vram it's safe to overwritte real_vram_size because
656 * we are not in case where real_vram_size is inferior to mc_vram_size (ie
657 * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
660 * Note: IGP TOM addr should be the same as the aperture addr, we don't
661 * explicitly check for that though.
663 * FIXME: when reducing VRAM size align new size on power of 2.
665 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
667 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
669 mc->vram_start = base;
670 if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
671 dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
672 mc->real_vram_size = mc->aper_size;
673 mc->mc_vram_size = mc->aper_size;
675 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
676 if (limit && limit < mc->real_vram_size)
677 mc->real_vram_size = limit;
678 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
679 mc->mc_vram_size >> 20, mc->vram_start,
680 mc->vram_end, mc->real_vram_size >> 20);
684 * amdgpu_gart_location - try to find GTT location
685 * @adev: amdgpu device structure holding all necessary informations
686 * @mc: memory controller structure holding memory informations
688 * Function will place try to place GTT before or after VRAM.
690 * If GTT size is bigger than space left then we ajust GTT size.
691 * Thus function will never fails.
693 * FIXME: when reducing GTT size align new size on power of 2.
695 void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
697 u64 size_af, size_bf;
699 size_af = adev->mc.mc_mask - mc->vram_end;
700 size_bf = mc->vram_start;
701 if (size_bf > size_af) {
702 if (mc->gart_size > size_bf) {
703 dev_warn(adev->dev, "limiting GTT\n");
704 mc->gart_size = size_bf;
708 if (mc->gart_size > size_af) {
709 dev_warn(adev->dev, "limiting GTT\n");
710 mc->gart_size = size_af;
712 mc->gart_start = mc->vram_end + 1;
714 mc->gart_end = mc->gart_start + mc->gart_size - 1;
715 dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
716 mc->gart_size >> 20, mc->gart_start, mc->gart_end);
720 * GPU helpers function.
723 * amdgpu_need_post - check if the hw need post or not
725 * @adev: amdgpu_device pointer
727 * Check if the asic has been initialized (all asics) at driver startup
728 * or post is needed if hw reset is performed.
729 * Returns true if need or false if not.
731 bool amdgpu_need_post(struct amdgpu_device *adev)
735 if (adev->has_hw_reset) {
736 adev->has_hw_reset = false;
740 /* bios scratch used on CIK+ */
741 if (adev->asic_type >= CHIP_BONAIRE)
742 return amdgpu_atombios_scratch_need_asic_init(adev);
744 /* check MEM_SIZE for older asics */
745 reg = amdgpu_asic_get_config_memsize(adev);
747 if ((reg != 0) && (reg != 0xffffffff))
754 static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
756 if (amdgpu_sriov_vf(adev))
759 if (amdgpu_passthrough(adev)) {
760 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
761 * some old smc fw still need driver do vPost otherwise gpu hang, while
762 * those smc fw version above 22.15 doesn't have this flaw, so we force
763 * vpost executed for smc version below 22.15
765 if (adev->asic_type == CHIP_FIJI) {
768 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
769 /* force vPost if error occured */
773 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
774 if (fw_ver < 0x00160e00)
778 return amdgpu_need_post(adev);
782 * amdgpu_dummy_page_init - init dummy page used by the driver
784 * @adev: amdgpu_device pointer
786 * Allocate the dummy page used by the driver (all asics).
787 * This dummy page is used by the driver as a filler for gart entries
788 * when pages are taken out of the GART
789 * Returns 0 on sucess, -ENOMEM on failure.
791 int amdgpu_dummy_page_init(struct amdgpu_device *adev)
793 if (adev->dummy_page.page)
795 adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
796 if (adev->dummy_page.page == NULL)
798 adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
799 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
800 if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
801 dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
802 __free_page(adev->dummy_page.page);
803 adev->dummy_page.page = NULL;
810 * amdgpu_dummy_page_fini - free dummy page used by the driver
812 * @adev: amdgpu_device pointer
814 * Frees the dummy page used by the driver (all asics).
816 void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
818 if (adev->dummy_page.page == NULL)
820 pci_unmap_page(adev->pdev, adev->dummy_page.addr,
821 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
822 __free_page(adev->dummy_page.page);
823 adev->dummy_page.page = NULL;
827 /* ATOM accessor methods */
829 * ATOM is an interpreted byte code stored in tables in the vbios. The
830 * driver registers callbacks to access registers and the interpreter
831 * in the driver parses the tables and executes then to program specific
832 * actions (set display modes, asic init, etc.). See amdgpu_atombios.c,
833 * atombios.h, and atom.c
837 * cail_pll_read - read PLL register
839 * @info: atom card_info pointer
840 * @reg: PLL register offset
842 * Provides a PLL register accessor for the atom interpreter (r4xx+).
843 * Returns the value of the PLL register.
845 static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
851 * cail_pll_write - write PLL register
853 * @info: atom card_info pointer
854 * @reg: PLL register offset
855 * @val: value to write to the pll register
857 * Provides a PLL register accessor for the atom interpreter (r4xx+).
859 static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
865 * cail_mc_read - read MC (Memory Controller) register
867 * @info: atom card_info pointer
868 * @reg: MC register offset
870 * Provides an MC register accessor for the atom interpreter (r4xx+).
871 * Returns the value of the MC register.
873 static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
879 * cail_mc_write - write MC (Memory Controller) register
881 * @info: atom card_info pointer
882 * @reg: MC register offset
883 * @val: value to write to the pll register
885 * Provides a MC register accessor for the atom interpreter (r4xx+).
887 static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
893 * cail_reg_write - write MMIO register
895 * @info: atom card_info pointer
896 * @reg: MMIO register offset
897 * @val: value to write to the pll register
899 * Provides a MMIO register accessor for the atom interpreter (r4xx+).
901 static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
903 struct amdgpu_device *adev = info->dev->dev_private;
909 * cail_reg_read - read MMIO register
911 * @info: atom card_info pointer
912 * @reg: MMIO register offset
914 * Provides an MMIO register accessor for the atom interpreter (r4xx+).
915 * Returns the value of the MMIO register.
917 static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
919 struct amdgpu_device *adev = info->dev->dev_private;
927 * cail_ioreg_write - write IO register
929 * @info: atom card_info pointer
930 * @reg: IO register offset
931 * @val: value to write to the pll register
933 * Provides a IO register accessor for the atom interpreter (r4xx+).
935 static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
937 struct amdgpu_device *adev = info->dev->dev_private;
943 * cail_ioreg_read - read IO register
945 * @info: atom card_info pointer
946 * @reg: IO register offset
948 * Provides an IO register accessor for the atom interpreter (r4xx+).
949 * Returns the value of the IO register.
951 static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
953 struct amdgpu_device *adev = info->dev->dev_private;
961 * amdgpu_atombios_fini - free the driver info and callbacks for atombios
963 * @adev: amdgpu_device pointer
965 * Frees the driver info and register access callbacks for the ATOM
966 * interpreter (r4xx+).
967 * Called at driver shutdown.
969 static void amdgpu_atombios_fini(struct amdgpu_device *adev)
971 if (adev->mode_info.atom_context) {
972 kfree(adev->mode_info.atom_context->scratch);
973 kfree(adev->mode_info.atom_context->iio);
975 kfree(adev->mode_info.atom_context);
976 adev->mode_info.atom_context = NULL;
977 kfree(adev->mode_info.atom_card_info);
978 adev->mode_info.atom_card_info = NULL;
982 * amdgpu_atombios_init - init the driver info and callbacks for atombios
984 * @adev: amdgpu_device pointer
986 * Initializes the driver info and register access callbacks for the
987 * ATOM interpreter (r4xx+).
988 * Returns 0 on sucess, -ENOMEM on failure.
989 * Called at driver startup.
991 static int amdgpu_atombios_init(struct amdgpu_device *adev)
993 struct card_info *atom_card_info =
994 kzalloc(sizeof(struct card_info), GFP_KERNEL);
999 adev->mode_info.atom_card_info = atom_card_info;
1000 atom_card_info->dev = adev->ddev;
1001 atom_card_info->reg_read = cail_reg_read;
1002 atom_card_info->reg_write = cail_reg_write;
1003 /* needed for iio ops */
1004 if (adev->rio_mem) {
1005 atom_card_info->ioreg_read = cail_ioreg_read;
1006 atom_card_info->ioreg_write = cail_ioreg_write;
1008 DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
1009 atom_card_info->ioreg_read = cail_reg_read;
1010 atom_card_info->ioreg_write = cail_reg_write;
1012 atom_card_info->mc_read = cail_mc_read;
1013 atom_card_info->mc_write = cail_mc_write;
1014 atom_card_info->pll_read = cail_pll_read;
1015 atom_card_info->pll_write = cail_pll_write;
1017 adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
1018 if (!adev->mode_info.atom_context) {
1019 amdgpu_atombios_fini(adev);
1023 mutex_init(&adev->mode_info.atom_context->mutex);
1024 if (adev->is_atom_fw) {
1025 amdgpu_atomfirmware_scratch_regs_init(adev);
1026 amdgpu_atomfirmware_allocate_fb_scratch(adev);
1028 amdgpu_atombios_scratch_regs_init(adev);
1029 amdgpu_atombios_allocate_fb_scratch(adev);
1034 /* if we get transitioned to only one device, take VGA back */
1036 * amdgpu_vga_set_decode - enable/disable vga decode
1038 * @cookie: amdgpu_device pointer
1039 * @state: enable/disable vga decode
1041 * Enable/disable vga decode (all asics).
1042 * Returns VGA resource flags.
1044 static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
1046 struct amdgpu_device *adev = cookie;
1047 amdgpu_asic_set_vga_state(adev, state);
1049 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1050 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1052 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1055 static void amdgpu_check_block_size(struct amdgpu_device *adev)
1057 /* defines number of bits in page table versus page directory,
1058 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1059 * page table and the remaining bits are in the page directory */
1060 if (amdgpu_vm_block_size == -1)
1063 if (amdgpu_vm_block_size < 9) {
1064 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1065 amdgpu_vm_block_size);
1069 if (amdgpu_vm_block_size > 24 ||
1070 (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
1071 dev_warn(adev->dev, "VM page table size (%d) too large\n",
1072 amdgpu_vm_block_size);
1079 amdgpu_vm_block_size = -1;
1082 static void amdgpu_check_vm_size(struct amdgpu_device *adev)
1084 /* no need to check the default value */
1085 if (amdgpu_vm_size == -1)
1088 if (!is_power_of_2(amdgpu_vm_size)) {
1089 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
1094 if (amdgpu_vm_size < 1) {
1095 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1101 * Max GPUVM size for Cayman, SI, CI VI are 40 bits.
1103 if (amdgpu_vm_size > 1024) {
1104 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
1112 amdgpu_vm_size = -1;
1116 * amdgpu_check_arguments - validate module params
1118 * @adev: amdgpu_device pointer
1120 * Validates certain module parameters and updates
1121 * the associated values used by the driver (all asics).
1123 static void amdgpu_check_arguments(struct amdgpu_device *adev)
1125 if (amdgpu_sched_jobs < 4) {
1126 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1128 amdgpu_sched_jobs = 4;
1129 } else if (!is_power_of_2(amdgpu_sched_jobs)){
1130 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1132 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1135 if (amdgpu_gart_size < 32) {
1136 /* gart size must be greater or equal to 32M */
1137 dev_warn(adev->dev, "gart size (%d) too small\n",
1139 amdgpu_gart_size = 32;
1142 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1143 /* gtt size must be greater or equal to 32M */
1144 dev_warn(adev->dev, "gtt size (%d) too small\n",
1146 amdgpu_gtt_size = -1;
1149 amdgpu_check_vm_size(adev);
1151 amdgpu_check_block_size(adev);
1153 if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
1154 !is_power_of_2(amdgpu_vram_page_split))) {
1155 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
1156 amdgpu_vram_page_split);
1157 amdgpu_vram_page_split = 1024;
1162 * amdgpu_switcheroo_set_state - set switcheroo state
1164 * @pdev: pci dev pointer
1165 * @state: vga_switcheroo state
1167 * Callback for the switcheroo driver. Suspends or resumes the
1168 * the asics before or after it is powered up using ACPI methods.
1170 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1172 struct drm_device *dev = pci_get_drvdata(pdev);
1174 if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1177 if (state == VGA_SWITCHEROO_ON) {
1178 unsigned d3_delay = dev->pdev->d3_delay;
1180 pr_info("amdgpu: switched on\n");
1181 /* don't suspend or resume card normally */
1182 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1184 amdgpu_device_resume(dev, true, true);
1186 dev->pdev->d3_delay = d3_delay;
1188 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1189 drm_kms_helper_poll_enable(dev);
1191 pr_info("amdgpu: switched off\n");
1192 drm_kms_helper_poll_disable(dev);
1193 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1194 amdgpu_device_suspend(dev, true, true);
1195 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1200 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1202 * @pdev: pci dev pointer
1204 * Callback for the switcheroo driver. Check of the switcheroo
1205 * state can be changed.
1206 * Returns true if the state can be changed, false if not.
1208 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1210 struct drm_device *dev = pci_get_drvdata(pdev);
1213 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1214 * locking inversion with the driver load path. And the access here is
1215 * completely racy anyway. So don't bother with locking for now.
1217 return dev->open_count == 0;
1220 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1221 .set_gpu_state = amdgpu_switcheroo_set_state,
1223 .can_switch = amdgpu_switcheroo_can_switch,
1226 int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
1227 enum amd_ip_block_type block_type,
1228 enum amd_clockgating_state state)
1232 for (i = 0; i < adev->num_ip_blocks; i++) {
1233 if (!adev->ip_blocks[i].status.valid)
1235 if (adev->ip_blocks[i].version->type != block_type)
1237 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1239 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1240 (void *)adev, state);
1242 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1243 adev->ip_blocks[i].version->funcs->name, r);
1248 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
1249 enum amd_ip_block_type block_type,
1250 enum amd_powergating_state state)
1254 for (i = 0; i < adev->num_ip_blocks; i++) {
1255 if (!adev->ip_blocks[i].status.valid)
1257 if (adev->ip_blocks[i].version->type != block_type)
1259 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1261 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1262 (void *)adev, state);
1264 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1265 adev->ip_blocks[i].version->funcs->name, r);
1270 void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
1274 for (i = 0; i < adev->num_ip_blocks; i++) {
1275 if (!adev->ip_blocks[i].status.valid)
1277 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1278 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1282 int amdgpu_wait_for_idle(struct amdgpu_device *adev,
1283 enum amd_ip_block_type block_type)
1287 for (i = 0; i < adev->num_ip_blocks; i++) {
1288 if (!adev->ip_blocks[i].status.valid)
1290 if (adev->ip_blocks[i].version->type == block_type) {
1291 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1301 bool amdgpu_is_idle(struct amdgpu_device *adev,
1302 enum amd_ip_block_type block_type)
1306 for (i = 0; i < adev->num_ip_blocks; i++) {
1307 if (!adev->ip_blocks[i].status.valid)
1309 if (adev->ip_blocks[i].version->type == block_type)
1310 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1316 struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
1317 enum amd_ip_block_type type)
1321 for (i = 0; i < adev->num_ip_blocks; i++)
1322 if (adev->ip_blocks[i].version->type == type)
1323 return &adev->ip_blocks[i];
1329 * amdgpu_ip_block_version_cmp
1331 * @adev: amdgpu_device pointer
1332 * @type: enum amd_ip_block_type
1333 * @major: major version
1334 * @minor: minor version
1336 * return 0 if equal or greater
1337 * return 1 if smaller or the ip_block doesn't exist
1339 int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
1340 enum amd_ip_block_type type,
1341 u32 major, u32 minor)
1343 struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
1345 if (ip_block && ((ip_block->version->major > major) ||
1346 ((ip_block->version->major == major) &&
1347 (ip_block->version->minor >= minor))))
1354 * amdgpu_ip_block_add
1356 * @adev: amdgpu_device pointer
1357 * @ip_block_version: pointer to the IP to add
1359 * Adds the IP block driver information to the collection of IPs
1362 int amdgpu_ip_block_add(struct amdgpu_device *adev,
1363 const struct amdgpu_ip_block_version *ip_block_version)
1365 if (!ip_block_version)
1368 DRM_DEBUG("add ip block number %d <%s>\n", adev->num_ip_blocks,
1369 ip_block_version->funcs->name);
1371 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1376 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1378 adev->enable_virtual_display = false;
1380 if (amdgpu_virtual_display) {
1381 struct drm_device *ddev = adev->ddev;
1382 const char *pci_address_name = pci_name(ddev->pdev);
1383 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1385 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1386 pciaddstr_tmp = pciaddstr;
1387 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1388 pciaddname = strsep(&pciaddname_tmp, ",");
1389 if (!strcmp("all", pciaddname)
1390 || !strcmp(pci_address_name, pciaddname)) {
1394 adev->enable_virtual_display = true;
1397 res = kstrtol(pciaddname_tmp, 10,
1405 adev->mode_info.num_crtc = num_crtc;
1407 adev->mode_info.num_crtc = 1;
1413 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1414 amdgpu_virtual_display, pci_address_name,
1415 adev->enable_virtual_display, adev->mode_info.num_crtc);
1421 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1423 const char *chip_name;
1426 const struct gpu_info_firmware_header_v1_0 *hdr;
1428 adev->firmware.gpu_info_fw = NULL;
1430 switch (adev->asic_type) {
1434 case CHIP_POLARIS11:
1435 case CHIP_POLARIS10:
1436 case CHIP_POLARIS12:
1439 #ifdef CONFIG_DRM_AMDGPU_SI
1446 #ifdef CONFIG_DRM_AMDGPU_CIK
1456 chip_name = "vega10";
1459 chip_name = "raven";
1463 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1464 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1467 "Failed to load gpu_info firmware \"%s\"\n",
1471 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1474 "Failed to validate gpu_info firmware \"%s\"\n",
1479 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1480 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1482 switch (hdr->version_major) {
1485 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1486 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1487 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1489 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1490 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1491 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1492 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1493 adev->gfx.config.max_texture_channel_caches =
1494 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1495 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1496 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1497 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1498 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1499 adev->gfx.config.double_offchip_lds_buf =
1500 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1501 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1502 adev->gfx.cu_info.max_waves_per_simd =
1503 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1504 adev->gfx.cu_info.max_scratch_slots_per_cu =
1505 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1506 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1511 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1519 static int amdgpu_early_init(struct amdgpu_device *adev)
1523 amdgpu_device_enable_virtual_display(adev);
1525 switch (adev->asic_type) {
1529 case CHIP_POLARIS11:
1530 case CHIP_POLARIS10:
1531 case CHIP_POLARIS12:
1534 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1535 adev->family = AMDGPU_FAMILY_CZ;
1537 adev->family = AMDGPU_FAMILY_VI;
1539 r = vi_set_ip_blocks(adev);
1543 #ifdef CONFIG_DRM_AMDGPU_SI
1549 adev->family = AMDGPU_FAMILY_SI;
1550 r = si_set_ip_blocks(adev);
1555 #ifdef CONFIG_DRM_AMDGPU_CIK
1561 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1562 adev->family = AMDGPU_FAMILY_CI;
1564 adev->family = AMDGPU_FAMILY_KV;
1566 r = cik_set_ip_blocks(adev);
1573 if (adev->asic_type == CHIP_RAVEN)
1574 adev->family = AMDGPU_FAMILY_RV;
1576 adev->family = AMDGPU_FAMILY_AI;
1578 r = soc15_set_ip_blocks(adev);
1583 /* FIXME: not supported yet */
1587 r = amdgpu_device_parse_gpu_info_fw(adev);
1591 if (amdgpu_sriov_vf(adev)) {
1592 r = amdgpu_virt_request_full_gpu(adev, true);
1597 for (i = 0; i < adev->num_ip_blocks; i++) {
1598 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1599 DRM_ERROR("disabled ip block: %d <%s>\n",
1600 i, adev->ip_blocks[i].version->funcs->name);
1601 adev->ip_blocks[i].status.valid = false;
1603 if (adev->ip_blocks[i].version->funcs->early_init) {
1604 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1606 adev->ip_blocks[i].status.valid = false;
1608 DRM_ERROR("early_init of IP block <%s> failed %d\n",
1609 adev->ip_blocks[i].version->funcs->name, r);
1612 adev->ip_blocks[i].status.valid = true;
1615 adev->ip_blocks[i].status.valid = true;
1620 adev->cg_flags &= amdgpu_cg_mask;
1621 adev->pg_flags &= amdgpu_pg_mask;
1626 static int amdgpu_init(struct amdgpu_device *adev)
1630 for (i = 0; i < adev->num_ip_blocks; i++) {
1631 if (!adev->ip_blocks[i].status.valid)
1633 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1635 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1636 adev->ip_blocks[i].version->funcs->name, r);
1639 adev->ip_blocks[i].status.sw = true;
1640 /* need to do gmc hw init early so we can allocate gpu mem */
1641 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1642 r = amdgpu_vram_scratch_init(adev);
1644 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1647 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1649 DRM_ERROR("hw_init %d failed %d\n", i, r);
1652 r = amdgpu_wb_init(adev);
1654 DRM_ERROR("amdgpu_wb_init failed %d\n", r);
1657 adev->ip_blocks[i].status.hw = true;
1659 /* right after GMC hw init, we create CSA */
1660 if (amdgpu_sriov_vf(adev)) {
1661 r = amdgpu_allocate_static_csa(adev);
1663 DRM_ERROR("allocate CSA failed %d\n", r);
1670 for (i = 0; i < adev->num_ip_blocks; i++) {
1671 if (!adev->ip_blocks[i].status.sw)
1673 /* gmc hw init is done early */
1674 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
1676 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1678 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1679 adev->ip_blocks[i].version->funcs->name, r);
1682 adev->ip_blocks[i].status.hw = true;
1688 static void amdgpu_fill_reset_magic(struct amdgpu_device *adev)
1690 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1693 static bool amdgpu_check_vram_lost(struct amdgpu_device *adev)
1695 return !!memcmp(adev->gart.ptr, adev->reset_magic,
1696 AMDGPU_RESET_MAGIC_NUM);
1699 static int amdgpu_late_set_cg_state(struct amdgpu_device *adev)
1703 for (i = 0; i < adev->num_ip_blocks; i++) {
1704 if (!adev->ip_blocks[i].status.valid)
1706 /* skip CG for VCE/UVD, it's handled specially */
1707 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1708 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1709 /* enable clockgating to save power */
1710 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1713 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1714 adev->ip_blocks[i].version->funcs->name, r);
1722 static int amdgpu_late_init(struct amdgpu_device *adev)
1726 for (i = 0; i < adev->num_ip_blocks; i++) {
1727 if (!adev->ip_blocks[i].status.valid)
1729 if (adev->ip_blocks[i].version->funcs->late_init) {
1730 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1732 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1733 adev->ip_blocks[i].version->funcs->name, r);
1736 adev->ip_blocks[i].status.late_initialized = true;
1740 mod_delayed_work(system_wq, &adev->late_init_work,
1741 msecs_to_jiffies(AMDGPU_RESUME_MS));
1743 amdgpu_fill_reset_magic(adev);
1748 static int amdgpu_fini(struct amdgpu_device *adev)
1752 /* need to disable SMC first */
1753 for (i = 0; i < adev->num_ip_blocks; i++) {
1754 if (!adev->ip_blocks[i].status.hw)
1756 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
1757 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1758 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1759 AMD_CG_STATE_UNGATE);
1761 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1762 adev->ip_blocks[i].version->funcs->name, r);
1765 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1766 /* XXX handle errors */
1768 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1769 adev->ip_blocks[i].version->funcs->name, r);
1771 adev->ip_blocks[i].status.hw = false;
1776 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1777 if (!adev->ip_blocks[i].status.hw)
1779 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1780 amdgpu_wb_fini(adev);
1781 amdgpu_vram_scratch_fini(adev);
1784 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1785 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1786 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1787 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1788 AMD_CG_STATE_UNGATE);
1790 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1791 adev->ip_blocks[i].version->funcs->name, r);
1796 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1797 /* XXX handle errors */
1799 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1800 adev->ip_blocks[i].version->funcs->name, r);
1803 adev->ip_blocks[i].status.hw = false;
1806 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1807 if (!adev->ip_blocks[i].status.sw)
1809 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1810 /* XXX handle errors */
1812 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1813 adev->ip_blocks[i].version->funcs->name, r);
1815 adev->ip_blocks[i].status.sw = false;
1816 adev->ip_blocks[i].status.valid = false;
1819 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1820 if (!adev->ip_blocks[i].status.late_initialized)
1822 if (adev->ip_blocks[i].version->funcs->late_fini)
1823 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1824 adev->ip_blocks[i].status.late_initialized = false;
1827 if (amdgpu_sriov_vf(adev)) {
1828 amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
1829 amdgpu_virt_release_full_gpu(adev, false);
1835 static void amdgpu_late_init_func_handler(struct work_struct *work)
1837 struct amdgpu_device *adev =
1838 container_of(work, struct amdgpu_device, late_init_work.work);
1839 amdgpu_late_set_cg_state(adev);
1842 int amdgpu_suspend(struct amdgpu_device *adev)
1846 if (amdgpu_sriov_vf(adev))
1847 amdgpu_virt_request_full_gpu(adev, false);
1849 /* ungate SMC block first */
1850 r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1851 AMD_CG_STATE_UNGATE);
1853 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
1856 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1857 if (!adev->ip_blocks[i].status.valid)
1859 /* ungate blocks so that suspend can properly shut them down */
1860 if (i != AMD_IP_BLOCK_TYPE_SMC) {
1861 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1862 AMD_CG_STATE_UNGATE);
1864 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1865 adev->ip_blocks[i].version->funcs->name, r);
1868 /* XXX handle errors */
1869 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1870 /* XXX handle errors */
1872 DRM_ERROR("suspend of IP block <%s> failed %d\n",
1873 adev->ip_blocks[i].version->funcs->name, r);
1877 if (amdgpu_sriov_vf(adev))
1878 amdgpu_virt_release_full_gpu(adev, false);
1883 static int amdgpu_sriov_reinit_early(struct amdgpu_device *adev)
1887 static enum amd_ip_block_type ip_order[] = {
1888 AMD_IP_BLOCK_TYPE_GMC,
1889 AMD_IP_BLOCK_TYPE_COMMON,
1890 AMD_IP_BLOCK_TYPE_IH,
1893 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1895 struct amdgpu_ip_block *block;
1897 for (j = 0; j < adev->num_ip_blocks; j++) {
1898 block = &adev->ip_blocks[j];
1900 if (block->version->type != ip_order[i] ||
1901 !block->status.valid)
1904 r = block->version->funcs->hw_init(adev);
1905 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1912 static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
1916 static enum amd_ip_block_type ip_order[] = {
1917 AMD_IP_BLOCK_TYPE_SMC,
1918 AMD_IP_BLOCK_TYPE_DCE,
1919 AMD_IP_BLOCK_TYPE_GFX,
1920 AMD_IP_BLOCK_TYPE_SDMA,
1921 AMD_IP_BLOCK_TYPE_VCE,
1924 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1926 struct amdgpu_ip_block *block;
1928 for (j = 0; j < adev->num_ip_blocks; j++) {
1929 block = &adev->ip_blocks[j];
1931 if (block->version->type != ip_order[i] ||
1932 !block->status.valid)
1935 r = block->version->funcs->hw_init(adev);
1936 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1943 static int amdgpu_resume_phase1(struct amdgpu_device *adev)
1947 for (i = 0; i < adev->num_ip_blocks; i++) {
1948 if (!adev->ip_blocks[i].status.valid)
1950 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1951 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1952 adev->ip_blocks[i].version->type ==
1953 AMD_IP_BLOCK_TYPE_IH) {
1954 r = adev->ip_blocks[i].version->funcs->resume(adev);
1956 DRM_ERROR("resume of IP block <%s> failed %d\n",
1957 adev->ip_blocks[i].version->funcs->name, r);
1966 static int amdgpu_resume_phase2(struct amdgpu_device *adev)
1970 for (i = 0; i < adev->num_ip_blocks; i++) {
1971 if (!adev->ip_blocks[i].status.valid)
1973 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1974 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1975 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH )
1977 r = adev->ip_blocks[i].version->funcs->resume(adev);
1979 DRM_ERROR("resume of IP block <%s> failed %d\n",
1980 adev->ip_blocks[i].version->funcs->name, r);
1988 static int amdgpu_resume(struct amdgpu_device *adev)
1992 r = amdgpu_resume_phase1(adev);
1995 r = amdgpu_resume_phase2(adev);
2000 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2002 if (adev->is_atom_fw) {
2003 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2004 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2006 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2007 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2012 * amdgpu_device_init - initialize the driver
2014 * @adev: amdgpu_device pointer
2015 * @pdev: drm dev pointer
2016 * @pdev: pci dev pointer
2017 * @flags: driver flags
2019 * Initializes the driver info and hw (all asics).
2020 * Returns 0 for success or an error on failure.
2021 * Called at driver startup.
2023 int amdgpu_device_init(struct amdgpu_device *adev,
2024 struct drm_device *ddev,
2025 struct pci_dev *pdev,
2029 bool runtime = false;
2032 adev->shutdown = false;
2033 adev->dev = &pdev->dev;
2036 adev->flags = flags;
2037 adev->asic_type = flags & AMD_ASIC_MASK;
2038 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2039 adev->mc.gart_size = 512 * 1024 * 1024;
2040 adev->accel_working = false;
2041 adev->num_rings = 0;
2042 adev->mman.buffer_funcs = NULL;
2043 adev->mman.buffer_funcs_ring = NULL;
2044 adev->vm_manager.vm_pte_funcs = NULL;
2045 adev->vm_manager.vm_pte_num_rings = 0;
2046 adev->gart.gart_funcs = NULL;
2047 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2049 adev->smc_rreg = &amdgpu_invalid_rreg;
2050 adev->smc_wreg = &amdgpu_invalid_wreg;
2051 adev->pcie_rreg = &amdgpu_invalid_rreg;
2052 adev->pcie_wreg = &amdgpu_invalid_wreg;
2053 adev->pciep_rreg = &amdgpu_invalid_rreg;
2054 adev->pciep_wreg = &amdgpu_invalid_wreg;
2055 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2056 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2057 adev->didt_rreg = &amdgpu_invalid_rreg;
2058 adev->didt_wreg = &amdgpu_invalid_wreg;
2059 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2060 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
2061 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2062 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2065 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2066 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2067 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
2069 /* mutex initialization are all done here so we
2070 * can recall function without having locking issues */
2071 atomic_set(&adev->irq.ih.lock, 0);
2072 mutex_init(&adev->firmware.mutex);
2073 mutex_init(&adev->pm.mutex);
2074 mutex_init(&adev->gfx.gpu_clock_mutex);
2075 mutex_init(&adev->srbm_mutex);
2076 mutex_init(&adev->grbm_idx_mutex);
2077 mutex_init(&adev->mn_lock);
2078 hash_init(adev->mn_hash);
2080 amdgpu_check_arguments(adev);
2082 spin_lock_init(&adev->mmio_idx_lock);
2083 spin_lock_init(&adev->smc_idx_lock);
2084 spin_lock_init(&adev->pcie_idx_lock);
2085 spin_lock_init(&adev->uvd_ctx_idx_lock);
2086 spin_lock_init(&adev->didt_idx_lock);
2087 spin_lock_init(&adev->gc_cac_idx_lock);
2088 spin_lock_init(&adev->se_cac_idx_lock);
2089 spin_lock_init(&adev->audio_endpt_idx_lock);
2090 spin_lock_init(&adev->mm_stats.lock);
2092 INIT_LIST_HEAD(&adev->shadow_list);
2093 mutex_init(&adev->shadow_list_lock);
2095 INIT_LIST_HEAD(&adev->gtt_list);
2096 spin_lock_init(&adev->gtt_list_lock);
2098 INIT_LIST_HEAD(&adev->ring_lru_list);
2099 spin_lock_init(&adev->ring_lru_list_lock);
2101 INIT_DELAYED_WORK(&adev->late_init_work, amdgpu_late_init_func_handler);
2103 /* Registers mapping */
2104 /* TODO: block userspace mapping of io register */
2105 if (adev->asic_type >= CHIP_BONAIRE) {
2106 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2107 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2109 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2110 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2113 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2114 if (adev->rmmio == NULL) {
2117 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2118 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2120 if (adev->asic_type >= CHIP_BONAIRE)
2121 /* doorbell bar mapping */
2122 amdgpu_doorbell_init(adev);
2124 /* io port mapping */
2125 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2126 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2127 adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2128 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2132 if (adev->rio_mem == NULL)
2133 DRM_INFO("PCI I/O BAR is not found.\n");
2135 /* early init functions */
2136 r = amdgpu_early_init(adev);
2140 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2141 /* this will fail for cards that aren't VGA class devices, just
2143 vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
2145 if (amdgpu_runtime_pm == 1)
2147 if (amdgpu_device_is_px(ddev))
2149 if (!pci_is_thunderbolt_attached(adev->pdev))
2150 vga_switcheroo_register_client(adev->pdev,
2151 &amdgpu_switcheroo_ops, runtime);
2153 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2156 if (!amdgpu_get_bios(adev)) {
2161 r = amdgpu_atombios_init(adev);
2163 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2164 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2168 /* detect if we are with an SRIOV vbios */
2169 amdgpu_device_detect_sriov_bios(adev);
2171 /* Post card if necessary */
2172 if (amdgpu_vpost_needed(adev)) {
2174 dev_err(adev->dev, "no vBIOS found\n");
2175 amdgpu_vf_error_put(AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2179 DRM_INFO("GPU posting now...\n");
2180 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2182 dev_err(adev->dev, "gpu post error!\n");
2183 amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_POST_ERROR, 0, 0);
2187 DRM_INFO("GPU post is not needed\n");
2190 if (!adev->is_atom_fw) {
2191 /* Initialize clocks */
2192 r = amdgpu_atombios_get_clock_info(adev);
2194 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2195 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2198 /* init i2c buses */
2199 amdgpu_atombios_i2c_init(adev);
2203 r = amdgpu_fence_driver_init(adev);
2205 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2206 amdgpu_vf_error_put(AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2210 /* init the mode config */
2211 drm_mode_config_init(adev->ddev);
2213 r = amdgpu_init(adev);
2215 dev_err(adev->dev, "amdgpu_init failed\n");
2216 amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2221 adev->accel_working = true;
2223 amdgpu_vm_check_compute_bug(adev);
2225 /* Initialize the buffer migration limit. */
2226 if (amdgpu_moverate >= 0)
2227 max_MBps = amdgpu_moverate;
2229 max_MBps = 8; /* Allow 8 MB/s. */
2230 /* Get a log2 for easy divisions. */
2231 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2233 r = amdgpu_ib_pool_init(adev);
2235 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2236 amdgpu_vf_error_put(AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2240 r = amdgpu_ib_ring_tests(adev);
2242 DRM_ERROR("ib ring test failed (%d).\n", r);
2244 amdgpu_fbdev_init(adev);
2246 r = amdgpu_gem_debugfs_init(adev);
2248 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2250 r = amdgpu_debugfs_regs_init(adev);
2252 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2254 r = amdgpu_debugfs_test_ib_ring_init(adev);
2256 DRM_ERROR("registering register test ib ring debugfs failed (%d).\n", r);
2258 r = amdgpu_debugfs_firmware_init(adev);
2260 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2262 if ((amdgpu_testing & 1)) {
2263 if (adev->accel_working)
2264 amdgpu_test_moves(adev);
2266 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2268 if (amdgpu_benchmarking) {
2269 if (adev->accel_working)
2270 amdgpu_benchmark(adev, amdgpu_benchmarking);
2272 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2275 /* enable clockgating, etc. after ib tests, etc. since some blocks require
2276 * explicit gating rather than handling it automatically.
2278 r = amdgpu_late_init(adev);
2280 dev_err(adev->dev, "amdgpu_late_init failed\n");
2281 amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2288 amdgpu_vf_error_trans_all(adev);
2290 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2295 * amdgpu_device_fini - tear down the driver
2297 * @adev: amdgpu_device pointer
2299 * Tear down the driver info (all asics).
2300 * Called at driver shutdown.
2302 void amdgpu_device_fini(struct amdgpu_device *adev)
2306 DRM_INFO("amdgpu: finishing device.\n");
2307 adev->shutdown = true;
2308 if (adev->mode_info.mode_config_initialized)
2309 drm_crtc_force_disable_all(adev->ddev);
2310 /* evict vram memory */
2311 amdgpu_bo_evict_vram(adev);
2312 amdgpu_ib_pool_fini(adev);
2313 amdgpu_fence_driver_fini(adev);
2314 amdgpu_fbdev_fini(adev);
2315 r = amdgpu_fini(adev);
2316 if (adev->firmware.gpu_info_fw) {
2317 release_firmware(adev->firmware.gpu_info_fw);
2318 adev->firmware.gpu_info_fw = NULL;
2320 adev->accel_working = false;
2321 cancel_delayed_work_sync(&adev->late_init_work);
2322 /* free i2c buses */
2323 amdgpu_i2c_fini(adev);
2324 amdgpu_atombios_fini(adev);
2327 if (!pci_is_thunderbolt_attached(adev->pdev))
2328 vga_switcheroo_unregister_client(adev->pdev);
2329 if (adev->flags & AMD_IS_PX)
2330 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2331 vga_client_register(adev->pdev, NULL, NULL, NULL);
2333 pci_iounmap(adev->pdev, adev->rio_mem);
2334 adev->rio_mem = NULL;
2335 iounmap(adev->rmmio);
2337 if (adev->asic_type >= CHIP_BONAIRE)
2338 amdgpu_doorbell_fini(adev);
2339 amdgpu_debugfs_regs_cleanup(adev);
2347 * amdgpu_device_suspend - initiate device suspend
2349 * @pdev: drm dev pointer
2350 * @state: suspend state
2352 * Puts the hw in the suspend state (all asics).
2353 * Returns 0 for success or an error on failure.
2354 * Called at driver suspend.
2356 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2358 struct amdgpu_device *adev;
2359 struct drm_crtc *crtc;
2360 struct drm_connector *connector;
2363 if (dev == NULL || dev->dev_private == NULL) {
2367 adev = dev->dev_private;
2369 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2372 drm_kms_helper_poll_disable(dev);
2374 /* turn off display hw */
2375 drm_modeset_lock_all(dev);
2376 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2377 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2379 drm_modeset_unlock_all(dev);
2381 /* unpin the front buffers and cursors */
2382 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2383 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2384 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
2385 struct amdgpu_bo *robj;
2387 if (amdgpu_crtc->cursor_bo) {
2388 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2389 r = amdgpu_bo_reserve(aobj, true);
2391 amdgpu_bo_unpin(aobj);
2392 amdgpu_bo_unreserve(aobj);
2396 if (rfb == NULL || rfb->obj == NULL) {
2399 robj = gem_to_amdgpu_bo(rfb->obj);
2400 /* don't unpin kernel fb objects */
2401 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2402 r = amdgpu_bo_reserve(robj, true);
2404 amdgpu_bo_unpin(robj);
2405 amdgpu_bo_unreserve(robj);
2409 /* evict vram memory */
2410 amdgpu_bo_evict_vram(adev);
2412 amdgpu_fence_driver_suspend(adev);
2414 r = amdgpu_suspend(adev);
2416 /* evict remaining vram memory
2417 * This second call to evict vram is to evict the gart page table
2420 amdgpu_bo_evict_vram(adev);
2422 amdgpu_atombios_scratch_regs_save(adev);
2423 pci_save_state(dev->pdev);
2425 /* Shut down the device */
2426 pci_disable_device(dev->pdev);
2427 pci_set_power_state(dev->pdev, PCI_D3hot);
2429 r = amdgpu_asic_reset(adev);
2431 DRM_ERROR("amdgpu asic reset failed\n");
2436 amdgpu_fbdev_set_suspend(adev, 1);
2443 * amdgpu_device_resume - initiate device resume
2445 * @pdev: drm dev pointer
2447 * Bring the hw back to operating state (all asics).
2448 * Returns 0 for success or an error on failure.
2449 * Called at driver resume.
2451 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2453 struct drm_connector *connector;
2454 struct amdgpu_device *adev = dev->dev_private;
2455 struct drm_crtc *crtc;
2458 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2465 pci_set_power_state(dev->pdev, PCI_D0);
2466 pci_restore_state(dev->pdev);
2467 r = pci_enable_device(dev->pdev);
2471 amdgpu_atombios_scratch_regs_restore(adev);
2474 if (amdgpu_need_post(adev)) {
2475 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2477 DRM_ERROR("amdgpu asic init failed\n");
2480 r = amdgpu_resume(adev);
2482 DRM_ERROR("amdgpu_resume failed (%d).\n", r);
2485 amdgpu_fence_driver_resume(adev);
2488 r = amdgpu_ib_ring_tests(adev);
2490 DRM_ERROR("ib ring test failed (%d).\n", r);
2493 r = amdgpu_late_init(adev);
2498 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2499 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2501 if (amdgpu_crtc->cursor_bo) {
2502 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2503 r = amdgpu_bo_reserve(aobj, true);
2505 r = amdgpu_bo_pin(aobj,
2506 AMDGPU_GEM_DOMAIN_VRAM,
2507 &amdgpu_crtc->cursor_addr);
2509 DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2510 amdgpu_bo_unreserve(aobj);
2515 /* blat the mode back in */
2517 drm_helper_resume_force_mode(dev);
2518 /* turn on display hw */
2519 drm_modeset_lock_all(dev);
2520 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2521 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2523 drm_modeset_unlock_all(dev);
2526 drm_kms_helper_poll_enable(dev);
2529 * Most of the connector probing functions try to acquire runtime pm
2530 * refs to ensure that the GPU is powered on when connector polling is
2531 * performed. Since we're calling this from a runtime PM callback,
2532 * trying to acquire rpm refs will cause us to deadlock.
2534 * Since we're guaranteed to be holding the rpm lock, it's safe to
2535 * temporarily disable the rpm helpers so this doesn't deadlock us.
2538 dev->dev->power.disable_depth++;
2540 drm_helper_hpd_irq_event(dev);
2542 dev->dev->power.disable_depth--;
2546 amdgpu_fbdev_set_suspend(adev, 0);
2555 static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
2558 bool asic_hang = false;
2560 for (i = 0; i < adev->num_ip_blocks; i++) {
2561 if (!adev->ip_blocks[i].status.valid)
2563 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2564 adev->ip_blocks[i].status.hang =
2565 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2566 if (adev->ip_blocks[i].status.hang) {
2567 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2574 static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
2578 for (i = 0; i < adev->num_ip_blocks; i++) {
2579 if (!adev->ip_blocks[i].status.valid)
2581 if (adev->ip_blocks[i].status.hang &&
2582 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2583 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2592 static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
2596 for (i = 0; i < adev->num_ip_blocks; i++) {
2597 if (!adev->ip_blocks[i].status.valid)
2599 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2600 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2601 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2602 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)) {
2603 if (adev->ip_blocks[i].status.hang) {
2604 DRM_INFO("Some block need full reset!\n");
2612 static int amdgpu_soft_reset(struct amdgpu_device *adev)
2616 for (i = 0; i < adev->num_ip_blocks; i++) {
2617 if (!adev->ip_blocks[i].status.valid)
2619 if (adev->ip_blocks[i].status.hang &&
2620 adev->ip_blocks[i].version->funcs->soft_reset) {
2621 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2630 static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
2634 for (i = 0; i < adev->num_ip_blocks; i++) {
2635 if (!adev->ip_blocks[i].status.valid)
2637 if (adev->ip_blocks[i].status.hang &&
2638 adev->ip_blocks[i].version->funcs->post_soft_reset)
2639 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2647 bool amdgpu_need_backup(struct amdgpu_device *adev)
2649 if (adev->flags & AMD_IS_APU)
2652 return amdgpu_lockup_timeout > 0 ? true : false;
2655 static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
2656 struct amdgpu_ring *ring,
2657 struct amdgpu_bo *bo,
2658 struct dma_fence **fence)
2666 r = amdgpu_bo_reserve(bo, true);
2669 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2670 /* if bo has been evicted, then no need to recover */
2671 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2672 r = amdgpu_bo_validate(bo->shadow);
2674 DRM_ERROR("bo validate failed!\n");
2678 r = amdgpu_ttm_bind(&bo->shadow->tbo, &bo->shadow->tbo.mem);
2680 DRM_ERROR("%p bind failed\n", bo->shadow);
2684 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2687 DRM_ERROR("recover page table failed!\n");
2692 amdgpu_bo_unreserve(bo);
2697 * amdgpu_sriov_gpu_reset - reset the asic
2699 * @adev: amdgpu device pointer
2700 * @job: which job trigger hang
2702 * Attempt the reset the GPU if it has hung (all asics).
2704 * Returns 0 for success or an error on failure.
2706 int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
2710 struct amdgpu_bo *bo, *tmp;
2711 struct amdgpu_ring *ring;
2712 struct dma_fence *fence = NULL, *next = NULL;
2714 mutex_lock(&adev->virt.lock_reset);
2715 atomic_inc(&adev->gpu_reset_counter);
2716 adev->gfx.in_reset = true;
2719 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2721 /* we start from the ring trigger GPU hang */
2722 j = job ? job->ring->idx : 0;
2724 /* block scheduler */
2725 for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
2726 ring = adev->rings[i % AMDGPU_MAX_RINGS];
2727 if (!ring || !ring->sched.thread)
2730 kthread_park(ring->sched.thread);
2735 /* here give the last chance to check if job removed from mirror-list
2736 * since we already pay some time on kthread_park */
2737 if (job && list_empty(&job->base.node)) {
2738 kthread_unpark(ring->sched.thread);
2742 if (amd_sched_invalidate_job(&job->base, amdgpu_job_hang_limit))
2743 amd_sched_job_kickout(&job->base);
2745 /* only do job_reset on the hang ring if @job not NULL */
2746 amd_sched_hw_job_reset(&ring->sched);
2748 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2749 amdgpu_fence_driver_force_completion_ring(ring);
2752 /* request to take full control of GPU before re-initialization */
2754 amdgpu_virt_reset_gpu(adev);
2756 amdgpu_virt_request_full_gpu(adev, true);
2759 /* Resume IP prior to SMC */
2760 amdgpu_sriov_reinit_early(adev);
2762 /* we need recover gart prior to run SMC/CP/SDMA resume */
2763 amdgpu_ttm_recover_gart(adev);
2765 /* now we are okay to resume SMC/CP/SDMA */
2766 amdgpu_sriov_reinit_late(adev);
2768 amdgpu_irq_gpu_reset_resume_helper(adev);
2770 if (amdgpu_ib_ring_tests(adev))
2771 dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
2773 /* release full control of GPU after ib test */
2774 amdgpu_virt_release_full_gpu(adev, true);
2776 DRM_INFO("recover vram bo from shadow\n");
2778 ring = adev->mman.buffer_funcs_ring;
2779 mutex_lock(&adev->shadow_list_lock);
2780 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2782 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2784 r = dma_fence_wait(fence, false);
2786 WARN(r, "recovery from shadow isn't completed\n");
2791 dma_fence_put(fence);
2794 mutex_unlock(&adev->shadow_list_lock);
2797 r = dma_fence_wait(fence, false);
2799 WARN(r, "recovery from shadow isn't completed\n");
2801 dma_fence_put(fence);
2803 for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
2804 ring = adev->rings[i % AMDGPU_MAX_RINGS];
2805 if (!ring || !ring->sched.thread)
2808 if (job && j != i) {
2809 kthread_unpark(ring->sched.thread);
2813 amd_sched_job_recovery(&ring->sched);
2814 kthread_unpark(ring->sched.thread);
2817 drm_helper_resume_force_mode(adev->ddev);
2819 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2821 /* bad news, how to tell it to userspace ? */
2822 dev_info(adev->dev, "GPU reset failed\n");
2824 dev_info(adev->dev, "GPU reset successed!\n");
2827 adev->gfx.in_reset = false;
2828 mutex_unlock(&adev->virt.lock_reset);
2833 * amdgpu_gpu_reset - reset the asic
2835 * @adev: amdgpu device pointer
2837 * Attempt the reset the GPU if it has hung (all asics).
2838 * Returns 0 for success or an error on failure.
2840 int amdgpu_gpu_reset(struct amdgpu_device *adev)
2844 bool need_full_reset, vram_lost = false;
2846 if (!amdgpu_check_soft_reset(adev)) {
2847 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
2851 atomic_inc(&adev->gpu_reset_counter);
2854 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2856 /* block scheduler */
2857 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2858 struct amdgpu_ring *ring = adev->rings[i];
2860 if (!ring || !ring->sched.thread)
2862 kthread_park(ring->sched.thread);
2863 amd_sched_hw_job_reset(&ring->sched);
2865 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2866 amdgpu_fence_driver_force_completion(adev);
2868 need_full_reset = amdgpu_need_full_reset(adev);
2870 if (!need_full_reset) {
2871 amdgpu_pre_soft_reset(adev);
2872 r = amdgpu_soft_reset(adev);
2873 amdgpu_post_soft_reset(adev);
2874 if (r || amdgpu_check_soft_reset(adev)) {
2875 DRM_INFO("soft reset failed, will fallback to full reset!\n");
2876 need_full_reset = true;
2880 if (need_full_reset) {
2881 r = amdgpu_suspend(adev);
2884 amdgpu_atombios_scratch_regs_save(adev);
2885 r = amdgpu_asic_reset(adev);
2886 amdgpu_atombios_scratch_regs_restore(adev);
2888 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2891 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
2892 r = amdgpu_resume_phase1(adev);
2895 vram_lost = amdgpu_check_vram_lost(adev);
2897 DRM_ERROR("VRAM is lost!\n");
2898 atomic_inc(&adev->vram_lost_counter);
2900 r = amdgpu_ttm_recover_gart(adev);
2903 r = amdgpu_resume_phase2(adev);
2907 amdgpu_fill_reset_magic(adev);
2912 amdgpu_irq_gpu_reset_resume_helper(adev);
2913 r = amdgpu_ib_ring_tests(adev);
2915 dev_err(adev->dev, "ib ring test failed (%d).\n", r);
2916 r = amdgpu_suspend(adev);
2917 need_full_reset = true;
2921 * recovery vm page tables, since we cannot depend on VRAM is
2922 * consistent after gpu full reset.
2924 if (need_full_reset && amdgpu_need_backup(adev)) {
2925 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2926 struct amdgpu_bo *bo, *tmp;
2927 struct dma_fence *fence = NULL, *next = NULL;
2929 DRM_INFO("recover vram bo from shadow\n");
2930 mutex_lock(&adev->shadow_list_lock);
2931 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2933 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2935 r = dma_fence_wait(fence, false);
2937 WARN(r, "recovery from shadow isn't completed\n");
2942 dma_fence_put(fence);
2945 mutex_unlock(&adev->shadow_list_lock);
2947 r = dma_fence_wait(fence, false);
2949 WARN(r, "recovery from shadow isn't completed\n");
2951 dma_fence_put(fence);
2953 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2954 struct amdgpu_ring *ring = adev->rings[i];
2956 if (!ring || !ring->sched.thread)
2959 amd_sched_job_recovery(&ring->sched);
2960 kthread_unpark(ring->sched.thread);
2963 dev_err(adev->dev, "asic resume failed (%d).\n", r);
2964 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ASIC_RESUME_FAIL, 0, r);
2965 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2966 if (adev->rings[i] && adev->rings[i]->sched.thread) {
2967 kthread_unpark(adev->rings[i]->sched.thread);
2972 drm_helper_resume_force_mode(adev->ddev);
2974 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2976 /* bad news, how to tell it to userspace ? */
2977 dev_info(adev->dev, "GPU reset failed\n");
2978 amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
2981 dev_info(adev->dev, "GPU reset successed!\n");
2984 amdgpu_vf_error_trans_all(adev);
2988 void amdgpu_get_pcie_info(struct amdgpu_device *adev)
2993 if (amdgpu_pcie_gen_cap)
2994 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
2996 if (amdgpu_pcie_lane_cap)
2997 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
2999 /* covers APUs as well */
3000 if (pci_is_root_bus(adev->pdev->bus)) {
3001 if (adev->pm.pcie_gen_mask == 0)
3002 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3003 if (adev->pm.pcie_mlw_mask == 0)
3004 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3008 if (adev->pm.pcie_gen_mask == 0) {
3009 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
3011 adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3012 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
3013 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
3015 if (mask & DRM_PCIE_SPEED_25)
3016 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
3017 if (mask & DRM_PCIE_SPEED_50)
3018 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
3019 if (mask & DRM_PCIE_SPEED_80)
3020 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
3022 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3025 if (adev->pm.pcie_mlw_mask == 0) {
3026 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
3030 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
3031 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3032 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3033 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3034 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3035 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3036 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3039 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3040 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3041 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3042 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3043 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3044 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3047 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3048 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3049 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3050 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3051 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3054 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3055 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3056 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3057 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3060 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3061 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3062 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3065 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3066 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3069 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
3075 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3083 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
3084 const struct drm_info_list *files,
3089 for (i = 0; i < adev->debugfs_count; i++) {
3090 if (adev->debugfs[i].files == files) {
3091 /* Already registered */
3096 i = adev->debugfs_count + 1;
3097 if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
3098 DRM_ERROR("Reached maximum number of debugfs components.\n");
3099 DRM_ERROR("Report so we increase "
3100 "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
3103 adev->debugfs[adev->debugfs_count].files = files;
3104 adev->debugfs[adev->debugfs_count].num_files = nfiles;
3105 adev->debugfs_count = i;
3106 #if defined(CONFIG_DEBUG_FS)
3107 drm_debugfs_create_files(files, nfiles,
3108 adev->ddev->primary->debugfs_root,
3109 adev->ddev->primary);
3114 #if defined(CONFIG_DEBUG_FS)
3116 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
3117 size_t size, loff_t *pos)
3119 struct amdgpu_device *adev = file_inode(f)->i_private;
3122 bool pm_pg_lock, use_bank;
3123 unsigned instance_bank, sh_bank, se_bank;
3125 if (size & 0x3 || *pos & 0x3)
3128 /* are we reading registers for which a PG lock is necessary? */
3129 pm_pg_lock = (*pos >> 23) & 1;
3131 if (*pos & (1ULL << 62)) {
3132 se_bank = (*pos >> 24) & 0x3FF;
3133 sh_bank = (*pos >> 34) & 0x3FF;
3134 instance_bank = (*pos >> 44) & 0x3FF;
3136 if (se_bank == 0x3FF)
3137 se_bank = 0xFFFFFFFF;
3138 if (sh_bank == 0x3FF)
3139 sh_bank = 0xFFFFFFFF;
3140 if (instance_bank == 0x3FF)
3141 instance_bank = 0xFFFFFFFF;
3147 *pos &= (1UL << 22) - 1;
3150 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
3151 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
3153 mutex_lock(&adev->grbm_idx_mutex);
3154 amdgpu_gfx_select_se_sh(adev, se_bank,
3155 sh_bank, instance_bank);
3159 mutex_lock(&adev->pm.mutex);
3164 if (*pos > adev->rmmio_size)
3167 value = RREG32(*pos >> 2);
3168 r = put_user(value, (uint32_t *)buf);
3182 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3183 mutex_unlock(&adev->grbm_idx_mutex);
3187 mutex_unlock(&adev->pm.mutex);
3192 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
3193 size_t size, loff_t *pos)
3195 struct amdgpu_device *adev = file_inode(f)->i_private;
3198 bool pm_pg_lock, use_bank;
3199 unsigned instance_bank, sh_bank, se_bank;
3201 if (size & 0x3 || *pos & 0x3)
3204 /* are we reading registers for which a PG lock is necessary? */
3205 pm_pg_lock = (*pos >> 23) & 1;
3207 if (*pos & (1ULL << 62)) {
3208 se_bank = (*pos >> 24) & 0x3FF;
3209 sh_bank = (*pos >> 34) & 0x3FF;
3210 instance_bank = (*pos >> 44) & 0x3FF;
3212 if (se_bank == 0x3FF)
3213 se_bank = 0xFFFFFFFF;
3214 if (sh_bank == 0x3FF)
3215 sh_bank = 0xFFFFFFFF;
3216 if (instance_bank == 0x3FF)
3217 instance_bank = 0xFFFFFFFF;
3223 *pos &= (1UL << 22) - 1;
3226 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
3227 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
3229 mutex_lock(&adev->grbm_idx_mutex);
3230 amdgpu_gfx_select_se_sh(adev, se_bank,
3231 sh_bank, instance_bank);
3235 mutex_lock(&adev->pm.mutex);
3240 if (*pos > adev->rmmio_size)
3243 r = get_user(value, (uint32_t *)buf);
3247 WREG32(*pos >> 2, value);
3256 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3257 mutex_unlock(&adev->grbm_idx_mutex);
3261 mutex_unlock(&adev->pm.mutex);
3266 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
3267 size_t size, loff_t *pos)
3269 struct amdgpu_device *adev = file_inode(f)->i_private;
3273 if (size & 0x3 || *pos & 0x3)
3279 value = RREG32_PCIE(*pos >> 2);
3280 r = put_user(value, (uint32_t *)buf);
3293 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
3294 size_t size, loff_t *pos)
3296 struct amdgpu_device *adev = file_inode(f)->i_private;
3300 if (size & 0x3 || *pos & 0x3)
3306 r = get_user(value, (uint32_t *)buf);
3310 WREG32_PCIE(*pos >> 2, value);
3321 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
3322 size_t size, loff_t *pos)
3324 struct amdgpu_device *adev = file_inode(f)->i_private;
3328 if (size & 0x3 || *pos & 0x3)
3334 value = RREG32_DIDT(*pos >> 2);
3335 r = put_user(value, (uint32_t *)buf);
3348 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
3349 size_t size, loff_t *pos)
3351 struct amdgpu_device *adev = file_inode(f)->i_private;
3355 if (size & 0x3 || *pos & 0x3)
3361 r = get_user(value, (uint32_t *)buf);
3365 WREG32_DIDT(*pos >> 2, value);
3376 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
3377 size_t size, loff_t *pos)
3379 struct amdgpu_device *adev = file_inode(f)->i_private;
3383 if (size & 0x3 || *pos & 0x3)
3389 value = RREG32_SMC(*pos);
3390 r = put_user(value, (uint32_t *)buf);
3403 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
3404 size_t size, loff_t *pos)
3406 struct amdgpu_device *adev = file_inode(f)->i_private;
3410 if (size & 0x3 || *pos & 0x3)
3416 r = get_user(value, (uint32_t *)buf);
3420 WREG32_SMC(*pos, value);
3431 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
3432 size_t size, loff_t *pos)
3434 struct amdgpu_device *adev = file_inode(f)->i_private;
3437 uint32_t *config, no_regs = 0;
3439 if (size & 0x3 || *pos & 0x3)
3442 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
3446 /* version, increment each time something is added */
3447 config[no_regs++] = 3;
3448 config[no_regs++] = adev->gfx.config.max_shader_engines;
3449 config[no_regs++] = adev->gfx.config.max_tile_pipes;
3450 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
3451 config[no_regs++] = adev->gfx.config.max_sh_per_se;
3452 config[no_regs++] = adev->gfx.config.max_backends_per_se;
3453 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
3454 config[no_regs++] = adev->gfx.config.max_gprs;
3455 config[no_regs++] = adev->gfx.config.max_gs_threads;
3456 config[no_regs++] = adev->gfx.config.max_hw_contexts;
3457 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
3458 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
3459 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
3460 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
3461 config[no_regs++] = adev->gfx.config.num_tile_pipes;
3462 config[no_regs++] = adev->gfx.config.backend_enable_mask;
3463 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
3464 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
3465 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
3466 config[no_regs++] = adev->gfx.config.num_gpus;
3467 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
3468 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
3469 config[no_regs++] = adev->gfx.config.gb_addr_config;
3470 config[no_regs++] = adev->gfx.config.num_rbs;
3473 config[no_regs++] = adev->rev_id;
3474 config[no_regs++] = adev->pg_flags;
3475 config[no_regs++] = adev->cg_flags;
3478 config[no_regs++] = adev->family;
3479 config[no_regs++] = adev->external_rev_id;
3482 config[no_regs++] = adev->pdev->device;
3483 config[no_regs++] = adev->pdev->revision;
3484 config[no_regs++] = adev->pdev->subsystem_device;
3485 config[no_regs++] = adev->pdev->subsystem_vendor;
3487 while (size && (*pos < no_regs * 4)) {
3490 value = config[*pos >> 2];
3491 r = put_user(value, (uint32_t *)buf);
3507 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
3508 size_t size, loff_t *pos)
3510 struct amdgpu_device *adev = file_inode(f)->i_private;
3511 int idx, x, outsize, r, valuesize;
3512 uint32_t values[16];
3514 if (size & 3 || *pos & 0x3)
3517 if (amdgpu_dpm == 0)
3520 /* convert offset to sensor number */
3523 valuesize = sizeof(values);
3524 if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
3525 r = adev->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, idx, &values[0], &valuesize);
3526 else if (adev->pm.funcs && adev->pm.funcs->read_sensor)
3527 r = adev->pm.funcs->read_sensor(adev, idx, &values[0],
3532 if (size > valuesize)
3539 r = put_user(values[x++], (int32_t *)buf);
3546 return !r ? outsize : r;
3549 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
3550 size_t size, loff_t *pos)
3552 struct amdgpu_device *adev = f->f_inode->i_private;
3555 uint32_t offset, se, sh, cu, wave, simd, data[32];
3557 if (size & 3 || *pos & 3)
3561 offset = (*pos & 0x7F);
3562 se = ((*pos >> 7) & 0xFF);
3563 sh = ((*pos >> 15) & 0xFF);
3564 cu = ((*pos >> 23) & 0xFF);
3565 wave = ((*pos >> 31) & 0xFF);
3566 simd = ((*pos >> 37) & 0xFF);
3568 /* switch to the specific se/sh/cu */
3569 mutex_lock(&adev->grbm_idx_mutex);
3570 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3573 if (adev->gfx.funcs->read_wave_data)
3574 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
3576 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3577 mutex_unlock(&adev->grbm_idx_mutex);
3582 while (size && (offset < x * 4)) {
3585 value = data[offset >> 2];
3586 r = put_user(value, (uint32_t *)buf);
3599 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
3600 size_t size, loff_t *pos)
3602 struct amdgpu_device *adev = f->f_inode->i_private;
3605 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
3607 if (size & 3 || *pos & 3)
3611 offset = (*pos & 0xFFF); /* in dwords */
3612 se = ((*pos >> 12) & 0xFF);
3613 sh = ((*pos >> 20) & 0xFF);
3614 cu = ((*pos >> 28) & 0xFF);
3615 wave = ((*pos >> 36) & 0xFF);
3616 simd = ((*pos >> 44) & 0xFF);
3617 thread = ((*pos >> 52) & 0xFF);
3618 bank = ((*pos >> 60) & 1);
3620 data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
3624 /* switch to the specific se/sh/cu */
3625 mutex_lock(&adev->grbm_idx_mutex);
3626 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3629 if (adev->gfx.funcs->read_wave_vgprs)
3630 adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
3632 if (adev->gfx.funcs->read_wave_sgprs)
3633 adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
3636 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3637 mutex_unlock(&adev->grbm_idx_mutex);
3642 value = data[offset++];
3643 r = put_user(value, (uint32_t *)buf);
3659 static const struct file_operations amdgpu_debugfs_regs_fops = {
3660 .owner = THIS_MODULE,
3661 .read = amdgpu_debugfs_regs_read,
3662 .write = amdgpu_debugfs_regs_write,
3663 .llseek = default_llseek
3665 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
3666 .owner = THIS_MODULE,
3667 .read = amdgpu_debugfs_regs_didt_read,
3668 .write = amdgpu_debugfs_regs_didt_write,
3669 .llseek = default_llseek
3671 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
3672 .owner = THIS_MODULE,
3673 .read = amdgpu_debugfs_regs_pcie_read,
3674 .write = amdgpu_debugfs_regs_pcie_write,
3675 .llseek = default_llseek
3677 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
3678 .owner = THIS_MODULE,
3679 .read = amdgpu_debugfs_regs_smc_read,
3680 .write = amdgpu_debugfs_regs_smc_write,
3681 .llseek = default_llseek
3684 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
3685 .owner = THIS_MODULE,
3686 .read = amdgpu_debugfs_gca_config_read,
3687 .llseek = default_llseek
3690 static const struct file_operations amdgpu_debugfs_sensors_fops = {
3691 .owner = THIS_MODULE,
3692 .read = amdgpu_debugfs_sensor_read,
3693 .llseek = default_llseek
3696 static const struct file_operations amdgpu_debugfs_wave_fops = {
3697 .owner = THIS_MODULE,
3698 .read = amdgpu_debugfs_wave_read,
3699 .llseek = default_llseek
3701 static const struct file_operations amdgpu_debugfs_gpr_fops = {
3702 .owner = THIS_MODULE,
3703 .read = amdgpu_debugfs_gpr_read,
3704 .llseek = default_llseek
3707 static const struct file_operations *debugfs_regs[] = {
3708 &amdgpu_debugfs_regs_fops,
3709 &amdgpu_debugfs_regs_didt_fops,
3710 &amdgpu_debugfs_regs_pcie_fops,
3711 &amdgpu_debugfs_regs_smc_fops,
3712 &amdgpu_debugfs_gca_config_fops,
3713 &amdgpu_debugfs_sensors_fops,
3714 &amdgpu_debugfs_wave_fops,
3715 &amdgpu_debugfs_gpr_fops,
3718 static const char *debugfs_regs_names[] = {
3723 "amdgpu_gca_config",
3729 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3731 struct drm_minor *minor = adev->ddev->primary;
3732 struct dentry *ent, *root = minor->debugfs_root;
3735 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3736 ent = debugfs_create_file(debugfs_regs_names[i],
3737 S_IFREG | S_IRUGO, root,
3738 adev, debugfs_regs[i]);
3740 for (j = 0; j < i; j++) {
3741 debugfs_remove(adev->debugfs_regs[i]);
3742 adev->debugfs_regs[i] = NULL;
3744 return PTR_ERR(ent);
3748 i_size_write(ent->d_inode, adev->rmmio_size);
3749 adev->debugfs_regs[i] = ent;
3755 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
3759 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3760 if (adev->debugfs_regs[i]) {
3761 debugfs_remove(adev->debugfs_regs[i]);
3762 adev->debugfs_regs[i] = NULL;
3767 static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
3769 struct drm_info_node *node = (struct drm_info_node *) m->private;
3770 struct drm_device *dev = node->minor->dev;
3771 struct amdgpu_device *adev = dev->dev_private;
3774 /* hold on the scheduler */
3775 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
3776 struct amdgpu_ring *ring = adev->rings[i];
3778 if (!ring || !ring->sched.thread)
3780 kthread_park(ring->sched.thread);
3783 seq_printf(m, "run ib test:\n");
3784 r = amdgpu_ib_ring_tests(adev);
3786 seq_printf(m, "ib ring tests failed (%d).\n", r);
3788 seq_printf(m, "ib ring tests passed.\n");
3790 /* go on the scheduler */
3791 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
3792 struct amdgpu_ring *ring = adev->rings[i];
3794 if (!ring || !ring->sched.thread)
3796 kthread_unpark(ring->sched.thread);
3802 static const struct drm_info_list amdgpu_debugfs_test_ib_ring_list[] = {
3803 {"amdgpu_test_ib", &amdgpu_debugfs_test_ib}
3806 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
3808 return amdgpu_debugfs_add_files(adev,
3809 amdgpu_debugfs_test_ib_ring_list, 1);
3812 int amdgpu_debugfs_init(struct drm_minor *minor)
3817 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
3821 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3825 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }