drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33 #include <linux/iommu.h>
  34 #include <linux/pci.h>
  35 #include <linux/devcoredump.h>
  36 #include <generated/utsrelease.h>
  37 #include <linux/pci-p2pdma.h>
  38
  39 #include <drm/drm_atomic_helper.h>
  40 #include <drm/drm_fb_helper.h>
  41 #include <drm/drm_probe_helper.h>
  42 #include <drm/amdgpu_drm.h>
  43 #include <linux/vgaarb.h>
  44 #include <linux/vga_switcheroo.h>
  45 #include <linux/efi.h>
  46 #include "amdgpu.h"
  47 #include "amdgpu_trace.h"
  48 #include "amdgpu_i2c.h"
  49 #include "atom.h"
  50 #include "amdgpu_atombios.h"
  51 #include "amdgpu_atomfirmware.h"
  52 #include "amd_pcie.h"
  53 #ifdef CONFIG_DRM_AMDGPU_SI
  54 #include "si.h"
  55 #endif
  56 #ifdef CONFIG_DRM_AMDGPU_CIK
  57 #include "cik.h"
  58 #endif
  59 #include "vi.h"
  60 #include "soc15.h"
  61 #include "nv.h"
  62 #include "bif/bif_4_1_d.h"
  63 #include <linux/firmware.h>
  64 #include "amdgpu_vf_error.h"
  65
  66 #include "amdgpu_amdkfd.h"
  67 #include "amdgpu_pm.h"
  68
  69 #include "amdgpu_xgmi.h"
  70 #include "amdgpu_ras.h"
  71 #include "amdgpu_pmu.h"
  72 #include "amdgpu_fru_eeprom.h"
  73 #include "amdgpu_reset.h"
  74
  75 #include <linux/suspend.h>
  76 #include <drm/task_barrier.h>
  77 #include <linux/pm_runtime.h>
  78
  79 #include <drm/drm_drv.h>
  80
  81 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  82 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  83 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  84 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  85 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  86 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  87 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  88
  89 #define AMDGPU_RESUME_MS                2000
  90 #define AMDGPU_MAX_RETRY_LIMIT          2
  91 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
  92
  93 const char *amdgpu_asic_name[] = {
  94         "TAHITI",
  95         "PITCAIRN",
  96         "VERDE",
  97         "OLAND",
  98         "HAINAN",
  99         "BONAIRE",
 100         "KAVERI",
 101         "KABINI",
 102         "HAWAII",
 103         "MULLINS",
 104         "TOPAZ",
 105         "TONGA",
 106         "FIJI",
 107         "CARRIZO",
 108         "STONEY",
 109         "POLARIS10",
 110         "POLARIS11",
 111         "POLARIS12",
 112         "VEGAM",
 113         "VEGA10",
 114         "VEGA12",
 115         "VEGA20",
 116         "RAVEN",
 117         "ARCTURUS",
 118         "RENOIR",
 119         "ALDEBARAN",
 120         "NAVI10",
 121         "CYAN_SKILLFISH",
 122         "NAVI14",
 123         "NAVI12",
 124         "SIENNA_CICHLID",
 125         "NAVY_FLOUNDER",
 126         "VANGOGH",
 127         "DIMGREY_CAVEFISH",
 128         "BEIGE_GOBY",
 129         "YELLOW_CARP",
 130         "IP DISCOVERY",
 131         "LAST",
 132 };
 133
 134 /**
 135  * DOC: pcie_replay_count
 136  *
 137  * The amdgpu driver provides a sysfs API for reporting the total number
 138  * of PCIe replays (NAKs)
 139  * The file pcie_replay_count is used for this and returns the total
 140  * number of replays as a sum of the NAKs generated and NAKs received
 141  */
 142
 143 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 144                 struct device_attribute *attr, char *buf)
 145 {
 146         struct drm_device *ddev = dev_get_drvdata(dev);
 147         struct amdgpu_device *adev = drm_to_adev(ddev);
 148         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 149
 150         return sysfs_emit(buf, "%llu\n", cnt);
 151 }
 152
 153 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
 154                 amdgpu_device_get_pcie_replay_count, NULL);
 155
 156 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 157
 158 /**
 159  * DOC: product_name
 160  *
 161  * The amdgpu driver provides a sysfs API for reporting the product name
 162  * for the device
 163  * The file serial_number is used for this and returns the product name
 164  * as returned from the FRU.
 165  * NOTE: This is only available for certain server cards
 166  */
 167
 168 static ssize_t amdgpu_device_get_product_name(struct device *dev,
 169                 struct device_attribute *attr, char *buf)
 170 {
 171         struct drm_device *ddev = dev_get_drvdata(dev);
 172         struct amdgpu_device *adev = drm_to_adev(ddev);
 173
 174         return sysfs_emit(buf, "%s\n", adev->product_name);
 175 }
 176
 177 static DEVICE_ATTR(product_name, S_IRUGO,
 178                 amdgpu_device_get_product_name, NULL);
 179
 180 /**
 181  * DOC: product_number
 182  *
 183  * The amdgpu driver provides a sysfs API for reporting the part number
 184  * for the device
 185  * The file serial_number is used for this and returns the part number
 186  * as returned from the FRU.
 187  * NOTE: This is only available for certain server cards
 188  */
 189
 190 static ssize_t amdgpu_device_get_product_number(struct device *dev,
 191                 struct device_attribute *attr, char *buf)
 192 {
 193         struct drm_device *ddev = dev_get_drvdata(dev);
 194         struct amdgpu_device *adev = drm_to_adev(ddev);
 195
 196         return sysfs_emit(buf, "%s\n", adev->product_number);
 197 }
 198
 199 static DEVICE_ATTR(product_number, S_IRUGO,
 200                 amdgpu_device_get_product_number, NULL);
 201
 202 /**
 203  * DOC: serial_number
 204  *
 205  * The amdgpu driver provides a sysfs API for reporting the serial number
 206  * for the device
 207  * The file serial_number is used for this and returns the serial number
 208  * as returned from the FRU.
 209  * NOTE: This is only available for certain server cards
 210  */
 211
 212 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
 213                 struct device_attribute *attr, char *buf)
 214 {
 215         struct drm_device *ddev = dev_get_drvdata(dev);
 216         struct amdgpu_device *adev = drm_to_adev(ddev);
 217
 218         return sysfs_emit(buf, "%s\n", adev->serial);
 219 }
 220
 221 static DEVICE_ATTR(serial_number, S_IRUGO,
 222                 amdgpu_device_get_serial_number, NULL);
 223
 224 /**
 225  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 226  *
 227  * @dev: drm_device pointer
 228  *
 229  * Returns true if the device is a dGPU with ATPX power control,
 230  * otherwise return false.
 231  */
 232 bool amdgpu_device_supports_px(struct drm_device *dev)
 233 {
 234         struct amdgpu_device *adev = drm_to_adev(dev);
 235
 236         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 237                 return true;
 238         return false;
 239 }
 240
 241 /**
 242  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 243  *
 244  * @dev: drm_device pointer
 245  *
 246  * Returns true if the device is a dGPU with ACPI power control,
 247  * otherwise return false.
 248  */
 249 bool amdgpu_device_supports_boco(struct drm_device *dev)
 250 {
 251         struct amdgpu_device *adev = drm_to_adev(dev);
 252
 253         if (adev->has_pr3 ||
 254             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 255                 return true;
 256         return false;
 257 }
 258
 259 /**
 260  * amdgpu_device_supports_baco - Does the device support BACO
 261  *
 262  * @dev: drm_device pointer
 263  *
 264  * Returns true if the device supporte BACO,
 265  * otherwise return false.
 266  */
 267 bool amdgpu_device_supports_baco(struct drm_device *dev)
 268 {
 269         struct amdgpu_device *adev = drm_to_adev(dev);
 270
 271         return amdgpu_asic_supports_baco(adev);
 272 }
 273
 274 /**
 275  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 276  * smart shift support
 277  *
 278  * @dev: drm_device pointer
 279  *
 280  * Returns true if the device is a dGPU with Smart Shift support,
 281  * otherwise returns false.
 282  */
 283 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 284 {
 285         return (amdgpu_device_supports_boco(dev) &&
 286                 amdgpu_acpi_is_power_shift_control_supported());
 287 }
 288
 289 /*
 290  * VRAM access helper functions
 291  */
 292
 293 /**
 294  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
 295  *
 296  * @adev: amdgpu_device pointer
 297  * @pos: offset of the buffer in vram
 298  * @buf: virtual address of the buffer in system memory
 299  * @size: read/write size, sizeof(@buf) must > @size
 300  * @write: true - write to vram, otherwise - read from vram
 301  */
 302 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 303                              void *buf, size_t size, bool write)
 304 {
 305         unsigned long flags;
 306         uint32_t hi = ~0, tmp = 0;
 307         uint32_t *data = buf;
 308         uint64_t last;
 309         int idx;
 310
 311         if (!drm_dev_enter(adev_to_drm(adev), &idx))
 312                 return;
 313
 314         BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
 315
 316         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 317         for (last = pos + size; pos < last; pos += 4) {
 318                 tmp = pos >> 31;
 319
 320                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 321                 if (tmp != hi) {
 322                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 323                         hi = tmp;
 324                 }
 325                 if (write)
 326                         WREG32_NO_KIQ(mmMM_DATA, *data++);
 327                 else
 328                         *data++ = RREG32_NO_KIQ(mmMM_DATA);
 329         }
 330
 331         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 332         drm_dev_exit(idx);
 333 }
 334
 335 /**
 336  * amdgpu_device_aper_access - access vram by vram aperature
 337  *
 338  * @adev: amdgpu_device pointer
 339  * @pos: offset of the buffer in vram
 340  * @buf: virtual address of the buffer in system memory
 341  * @size: read/write size, sizeof(@buf) must > @size
 342  * @write: true - write to vram, otherwise - read from vram
 343  *
 344  * The return value means how many bytes have been transferred.
 345  */
 346 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 347                                  void *buf, size_t size, bool write)
 348 {
 349 #ifdef CONFIG_64BIT
 350         void __iomem *addr;
 351         size_t count = 0;
 352         uint64_t last;
 353
 354         if (!adev->mman.aper_base_kaddr)
 355                 return 0;
 356
 357         last = min(pos + size, adev->gmc.visible_vram_size);
 358         if (last > pos) {
 359                 addr = adev->mman.aper_base_kaddr + pos;
 360                 count = last - pos;
 361
 362                 if (write) {
 363                         memcpy_toio(addr, buf, count);
 364                         mb();
 365                         amdgpu_device_flush_hdp(adev, NULL);
 366                 } else {
 367                         amdgpu_device_invalidate_hdp(adev, NULL);
 368                         mb();
 369                         memcpy_fromio(buf, addr, count);
 370                 }
 371
 372         }
 373
 374         return count;
 375 #else
 376         return 0;
 377 #endif
 378 }
 379
 380 /**
 381  * amdgpu_device_vram_access - read/write a buffer in vram
 382  *
 383  * @adev: amdgpu_device pointer
 384  * @pos: offset of the buffer in vram
 385  * @buf: virtual address of the buffer in system memory
 386  * @size: read/write size, sizeof(@buf) must > @size
 387  * @write: true - write to vram, otherwise - read from vram
 388  */
 389 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 390                                void *buf, size_t size, bool write)
 391 {
 392         size_t count;
 393
 394         /* try to using vram apreature to access vram first */
 395         count = amdgpu_device_aper_access(adev, pos, buf, size, write);
 396         size -= count;
 397         if (size) {
 398                 /* using MM to access rest vram */
 399                 pos += count;
 400                 buf += count;
 401                 amdgpu_device_mm_access(adev, pos, buf, size, write);
 402         }
 403 }
 404
 405 /*
 406  * register access helper functions.
 407  */
 408
 409 /* Check if hw access should be skipped because of hotplug or device error */
 410 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 411 {
 412         if (adev->no_hw_access)
 413                 return true;
 414
 415 #ifdef CONFIG_LOCKDEP
 416         /*
 417          * This is a bit complicated to understand, so worth a comment. What we assert
 418          * here is that the GPU reset is not running on another thread in parallel.
 419          *
 420          * For this we trylock the read side of the reset semaphore, if that succeeds
 421          * we know that the reset is not running in paralell.
 422          *
 423          * If the trylock fails we assert that we are either already holding the read
 424          * side of the lock or are the reset thread itself and hold the write side of
 425          * the lock.
 426          */
 427         if (in_task()) {
 428                 if (down_read_trylock(&adev->reset_domain->sem))
 429                         up_read(&adev->reset_domain->sem);
 430                 else
 431                         lockdep_assert_held(&adev->reset_domain->sem);
 432         }
 433 #endif
 434         return false;
 435 }
 436
 437 /**
 438  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 439  *
 440  * @adev: amdgpu_device pointer
 441  * @reg: dword aligned register offset
 442  * @acc_flags: access flags which require special behavior
 443  *
 444  * Returns the 32 bit value from the offset specified.
 445  */
 446 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 447                             uint32_t reg, uint32_t acc_flags)
 448 {
 449         uint32_t ret;
 450
 451         if (amdgpu_device_skip_hw_access(adev))
 452                 return 0;
 453
 454         if ((reg * 4) < adev->rmmio_size) {
 455                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 456                     amdgpu_sriov_runtime(adev) &&
 457                     down_read_trylock(&adev->reset_domain->sem)) {
 458                         ret = amdgpu_kiq_rreg(adev, reg);
 459                         up_read(&adev->reset_domain->sem);
 460                 } else {
 461                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 462                 }
 463         } else {
 464                 ret = adev->pcie_rreg(adev, reg * 4);
 465         }
 466
 467         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 468
 469         return ret;
 470 }
 471
 472 /*
 473  * MMIO register read with bytes helper functions
 474  * @offset:bytes offset from MMIO start
 475  *
 476 */
 477
 478 /**
 479  * amdgpu_mm_rreg8 - read a memory mapped IO register
 480  *
 481  * @adev: amdgpu_device pointer
 482  * @offset: byte aligned register offset
 483  *
 484  * Returns the 8 bit value from the offset specified.
 485  */
 486 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 487 {
 488         if (amdgpu_device_skip_hw_access(adev))
 489                 return 0;
 490
 491         if (offset < adev->rmmio_size)
 492                 return (readb(adev->rmmio + offset));
 493         BUG();
 494 }
 495
 496 /*
 497  * MMIO register write with bytes helper functions
 498  * @offset:bytes offset from MMIO start
 499  * @value: the value want to be written to the register
 500  *
 501 */
 502 /**
 503  * amdgpu_mm_wreg8 - read a memory mapped IO register
 504  *
 505  * @adev: amdgpu_device pointer
 506  * @offset: byte aligned register offset
 507  * @value: 8 bit value to write
 508  *
 509  * Writes the value specified to the offset specified.
 510  */
 511 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 512 {
 513         if (amdgpu_device_skip_hw_access(adev))
 514                 return;
 515
 516         if (offset < adev->rmmio_size)
 517                 writeb(value, adev->rmmio + offset);
 518         else
 519                 BUG();
 520 }
 521
 522 /**
 523  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 524  *
 525  * @adev: amdgpu_device pointer
 526  * @reg: dword aligned register offset
 527  * @v: 32 bit value to write to the register
 528  * @acc_flags: access flags which require special behavior
 529  *
 530  * Writes the value specified to the offset specified.
 531  */
 532 void amdgpu_device_wreg(struct amdgpu_device *adev,
 533                         uint32_t reg, uint32_t v,
 534                         uint32_t acc_flags)
 535 {
 536         if (amdgpu_device_skip_hw_access(adev))
 537                 return;
 538
 539         if ((reg * 4) < adev->rmmio_size) {
 540                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 541                     amdgpu_sriov_runtime(adev) &&
 542                     down_read_trylock(&adev->reset_domain->sem)) {
 543                         amdgpu_kiq_wreg(adev, reg, v);
 544                         up_read(&adev->reset_domain->sem);
 545                 } else {
 546                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 547                 }
 548         } else {
 549                 adev->pcie_wreg(adev, reg * 4, v);
 550         }
 551
 552         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 553 }
 554
 555 /**
 556  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
 557  *
 558  * @adev: amdgpu_device pointer
 559  * @reg: mmio/rlc register
 560  * @v: value to write
 561  *
 562  * this function is invoked only for the debugfs register access
 563  */
 564 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 565                              uint32_t reg, uint32_t v)
 566 {
 567         if (amdgpu_device_skip_hw_access(adev))
 568                 return;
 569
 570         if (amdgpu_sriov_fullaccess(adev) &&
 571             adev->gfx.rlc.funcs &&
 572             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 573                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 574                         return amdgpu_sriov_wreg(adev, reg, v, 0, 0);
 575         } else if ((reg * 4) >= adev->rmmio_size) {
 576                 adev->pcie_wreg(adev, reg * 4, v);
 577         } else {
 578                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 579         }
 580 }
 581
 582 /**
 583  * amdgpu_mm_rdoorbell - read a doorbell dword
 584  *
 585  * @adev: amdgpu_device pointer
 586  * @index: doorbell index
 587  *
 588  * Returns the value in the doorbell aperture at the
 589  * requested doorbell index (CIK).
 590  */
 591 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 592 {
 593         if (amdgpu_device_skip_hw_access(adev))
 594                 return 0;
 595
 596         if (index < adev->doorbell.num_doorbells) {
 597                 return readl(adev->doorbell.ptr + index);
 598         } else {
 599                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 600                 return 0;
 601         }
 602 }
 603
 604 /**
 605  * amdgpu_mm_wdoorbell - write a doorbell dword
 606  *
 607  * @adev: amdgpu_device pointer
 608  * @index: doorbell index
 609  * @v: value to write
 610  *
 611  * Writes @v to the doorbell aperture at the
 612  * requested doorbell index (CIK).
 613  */
 614 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 615 {
 616         if (amdgpu_device_skip_hw_access(adev))
 617                 return;
 618
 619         if (index < adev->doorbell.num_doorbells) {
 620                 writel(v, adev->doorbell.ptr + index);
 621         } else {
 622                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 623         }
 624 }
 625
 626 /**
 627  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 628  *
 629  * @adev: amdgpu_device pointer
 630  * @index: doorbell index
 631  *
 632  * Returns the value in the doorbell aperture at the
 633  * requested doorbell index (VEGA10+).
 634  */
 635 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 636 {
 637         if (amdgpu_device_skip_hw_access(adev))
 638                 return 0;
 639
 640         if (index < adev->doorbell.num_doorbells) {
 641                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 642         } else {
 643                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 644                 return 0;
 645         }
 646 }
 647
 648 /**
 649  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 650  *
 651  * @adev: amdgpu_device pointer
 652  * @index: doorbell index
 653  * @v: value to write
 654  *
 655  * Writes @v to the doorbell aperture at the
 656  * requested doorbell index (VEGA10+).
 657  */
 658 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 659 {
 660         if (amdgpu_device_skip_hw_access(adev))
 661                 return;
 662
 663         if (index < adev->doorbell.num_doorbells) {
 664                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 665         } else {
 666                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 667         }
 668 }
 669
 670 /**
 671  * amdgpu_device_indirect_rreg - read an indirect register
 672  *
 673  * @adev: amdgpu_device pointer
 674  * @pcie_index: mmio register offset
 675  * @pcie_data: mmio register offset
 676  * @reg_addr: indirect register address to read from
 677  *
 678  * Returns the value of indirect register @reg_addr
 679  */
 680 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 681                                 u32 pcie_index, u32 pcie_data,
 682                                 u32 reg_addr)
 683 {
 684         unsigned long flags;
 685         u32 r;
 686         void __iomem *pcie_index_offset;
 687         void __iomem *pcie_data_offset;
 688
 689         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 690         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 691         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 692
 693         writel(reg_addr, pcie_index_offset);
 694         readl(pcie_index_offset);
 695         r = readl(pcie_data_offset);
 696         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 697
 698         return r;
 699 }
 700
 701 /**
 702  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 703  *
 704  * @adev: amdgpu_device pointer
 705  * @pcie_index: mmio register offset
 706  * @pcie_data: mmio register offset
 707  * @reg_addr: indirect register address to read from
 708  *
 709  * Returns the value of indirect register @reg_addr
 710  */
 711 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 712                                   u32 pcie_index, u32 pcie_data,
 713                                   u32 reg_addr)
 714 {
 715         unsigned long flags;
 716         u64 r;
 717         void __iomem *pcie_index_offset;
 718         void __iomem *pcie_data_offset;
 719
 720         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 721         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 722         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 723
 724         /* read low 32 bits */
 725         writel(reg_addr, pcie_index_offset);
 726         readl(pcie_index_offset);
 727         r = readl(pcie_data_offset);
 728         /* read high 32 bits */
 729         writel(reg_addr + 4, pcie_index_offset);
 730         readl(pcie_index_offset);
 731         r |= ((u64)readl(pcie_data_offset) << 32);
 732         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 733
 734         return r;
 735 }
 736
 737 /**
 738  * amdgpu_device_indirect_wreg - write an indirect register address
 739  *
 740  * @adev: amdgpu_device pointer
 741  * @pcie_index: mmio register offset
 742  * @pcie_data: mmio register offset
 743  * @reg_addr: indirect register offset
 744  * @reg_data: indirect register data
 745  *
 746  */
 747 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 748                                  u32 pcie_index, u32 pcie_data,
 749                                  u32 reg_addr, u32 reg_data)
 750 {
 751         unsigned long flags;
 752         void __iomem *pcie_index_offset;
 753         void __iomem *pcie_data_offset;
 754
 755         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 756         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 757         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 758
 759         writel(reg_addr, pcie_index_offset);
 760         readl(pcie_index_offset);
 761         writel(reg_data, pcie_data_offset);
 762         readl(pcie_data_offset);
 763         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 764 }
 765
 766 /**
 767  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 768  *
 769  * @adev: amdgpu_device pointer
 770  * @pcie_index: mmio register offset
 771  * @pcie_data: mmio register offset
 772  * @reg_addr: indirect register offset
 773  * @reg_data: indirect register data
 774  *
 775  */
 776 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 777                                    u32 pcie_index, u32 pcie_data,
 778                                    u32 reg_addr, u64 reg_data)
 779 {
 780         unsigned long flags;
 781         void __iomem *pcie_index_offset;
 782         void __iomem *pcie_data_offset;
 783
 784         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 785         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 786         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 787
 788         /* write low 32 bits */
 789         writel(reg_addr, pcie_index_offset);
 790         readl(pcie_index_offset);
 791         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 792         readl(pcie_data_offset);
 793         /* write high 32 bits */
 794         writel(reg_addr + 4, pcie_index_offset);
 795         readl(pcie_index_offset);
 796         writel((u32)(reg_data >> 32), pcie_data_offset);
 797         readl(pcie_data_offset);
 798         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 799 }
 800
 801 /**
 802  * amdgpu_invalid_rreg - dummy reg read function
 803  *
 804  * @adev: amdgpu_device pointer
 805  * @reg: offset of register
 806  *
 807  * Dummy register read function.  Used for register blocks
 808  * that certain asics don't have (all asics).
 809  * Returns the value in the register.
 810  */
 811 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 812 {
 813         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 814         BUG();
 815         return 0;
 816 }
 817
 818 /**
 819  * amdgpu_invalid_wreg - dummy reg write function
 820  *
 821  * @adev: amdgpu_device pointer
 822  * @reg: offset of register
 823  * @v: value to write to the register
 824  *
 825  * Dummy register read function.  Used for register blocks
 826  * that certain asics don't have (all asics).
 827  */
 828 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 829 {
 830         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 831                   reg, v);
 832         BUG();
 833 }
 834
 835 /**
 836  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 837  *
 838  * @adev: amdgpu_device pointer
 839  * @reg: offset of register
 840  *
 841  * Dummy register read function.  Used for register blocks
 842  * that certain asics don't have (all asics).
 843  * Returns the value in the register.
 844  */
 845 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
 846 {
 847         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
 848         BUG();
 849         return 0;
 850 }
 851
 852 /**
 853  * amdgpu_invalid_wreg64 - dummy reg write function
 854  *
 855  * @adev: amdgpu_device pointer
 856  * @reg: offset of register
 857  * @v: value to write to the register
 858  *
 859  * Dummy register read function.  Used for register blocks
 860  * that certain asics don't have (all asics).
 861  */
 862 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
 863 {
 864         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
 865                   reg, v);
 866         BUG();
 867 }
 868
 869 /**
 870  * amdgpu_block_invalid_rreg - dummy reg read function
 871  *
 872  * @adev: amdgpu_device pointer
 873  * @block: offset of instance
 874  * @reg: offset of register
 875  *
 876  * Dummy register read function.  Used for register blocks
 877  * that certain asics don't have (all asics).
 878  * Returns the value in the register.
 879  */
 880 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 881                                           uint32_t block, uint32_t reg)
 882 {
 883         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 884                   reg, block);
 885         BUG();
 886         return 0;
 887 }
 888
 889 /**
 890  * amdgpu_block_invalid_wreg - dummy reg write function
 891  *
 892  * @adev: amdgpu_device pointer
 893  * @block: offset of instance
 894  * @reg: offset of register
 895  * @v: value to write to the register
 896  *
 897  * Dummy register read function.  Used for register blocks
 898  * that certain asics don't have (all asics).
 899  */
 900 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 901                                       uint32_t block,
 902                                       uint32_t reg, uint32_t v)
 903 {
 904         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 905                   reg, block, v);
 906         BUG();
 907 }
 908
 909 /**
 910  * amdgpu_device_asic_init - Wrapper for atom asic_init
 911  *
 912  * @adev: amdgpu_device pointer
 913  *
 914  * Does any asic specific work and then calls atom asic init.
 915  */
 916 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
 917 {
 918         amdgpu_asic_pre_asic_init(adev);
 919
 920         if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0))
 921                 return amdgpu_atomfirmware_asic_init(adev, true);
 922         else
 923                 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
 924 }
 925
 926 /**
 927  * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
 928  *
 929  * @adev: amdgpu_device pointer
 930  *
 931  * Allocates a scratch page of VRAM for use by various things in the
 932  * driver.
 933  */
 934 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
 935 {
 936         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 937                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 938                                        &adev->vram_scratch.robj,
 939                                        &adev->vram_scratch.gpu_addr,
 940                                        (void **)&adev->vram_scratch.ptr);
 941 }
 942
 943 /**
 944  * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
 945  *
 946  * @adev: amdgpu_device pointer
 947  *
 948  * Frees the VRAM scratch page.
 949  */
 950 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
 951 {
 952         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 953 }
 954
 955 /**
 956  * amdgpu_device_program_register_sequence - program an array of registers.
 957  *
 958  * @adev: amdgpu_device pointer
 959  * @registers: pointer to the register array
 960  * @array_size: size of the register array
 961  *
 962  * Programs an array or registers with and and or masks.
 963  * This is a helper for setting golden registers.
 964  */
 965 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 966                                              const u32 *registers,
 967                                              const u32 array_size)
 968 {
 969         u32 tmp, reg, and_mask, or_mask;
 970         int i;
 971
 972         if (array_size % 3)
 973                 return;
 974
 975         for (i = 0; i < array_size; i +=3) {
 976                 reg = registers[i + 0];
 977                 and_mask = registers[i + 1];
 978                 or_mask = registers[i + 2];
 979
 980                 if (and_mask == 0xffffffff) {
 981                         tmp = or_mask;
 982                 } else {
 983                         tmp = RREG32(reg);
 984                         tmp &= ~and_mask;
 985                         if (adev->family >= AMDGPU_FAMILY_AI)
 986                                 tmp |= (or_mask & and_mask);
 987                         else
 988                                 tmp |= or_mask;
 989                 }
 990                 WREG32(reg, tmp);
 991         }
 992 }
 993
 994 /**
 995  * amdgpu_device_pci_config_reset - reset the GPU
 996  *
 997  * @adev: amdgpu_device pointer
 998  *
 999  * Resets the GPU using the pci config reset sequence.
1000  * Only applicable to asics prior to vega10.
1001  */
1002 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1003 {
1004         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1005 }
1006
1007 /**
1008  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1009  *
1010  * @adev: amdgpu_device pointer
1011  *
1012  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1013  */
1014 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1015 {
1016         return pci_reset_function(adev->pdev);
1017 }
1018
1019 /*
1020  * GPU doorbell aperture helpers function.
1021  */
1022 /**
1023  * amdgpu_device_doorbell_init - Init doorbell driver information.
1024  *
1025  * @adev: amdgpu_device pointer
1026  *
1027  * Init doorbell driver information (CIK)
1028  * Returns 0 on success, error on failure.
1029  */
1030 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
1031 {
1032
1033         /* No doorbell on SI hardware generation */
1034         if (adev->asic_type < CHIP_BONAIRE) {
1035                 adev->doorbell.base = 0;
1036                 adev->doorbell.size = 0;
1037                 adev->doorbell.num_doorbells = 0;
1038                 adev->doorbell.ptr = NULL;
1039                 return 0;
1040         }
1041
1042         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
1043                 return -EINVAL;
1044
1045         amdgpu_asic_init_doorbell_index(adev);
1046
1047         /* doorbell bar mapping */
1048         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
1049         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
1050
1051         if (adev->enable_mes) {
1052                 adev->doorbell.num_doorbells =
1053                         adev->doorbell.size / sizeof(u32);
1054         } else {
1055                 adev->doorbell.num_doorbells =
1056                         min_t(u32, adev->doorbell.size / sizeof(u32),
1057                               adev->doorbell_index.max_assignment+1);
1058                 if (adev->doorbell.num_doorbells == 0)
1059                         return -EINVAL;
1060
1061                 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
1062                  * paging queue doorbell use the second page. The
1063                  * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1064                  * doorbells are in the first page. So with paging queue enabled,
1065                  * the max num_doorbells should + 1 page (0x400 in dword)
1066                  */
1067                 if (adev->asic_type >= CHIP_VEGA10)
1068                         adev->doorbell.num_doorbells += 0x400;
1069         }
1070
1071         adev->doorbell.ptr = ioremap(adev->doorbell.base,
1072                                      adev->doorbell.num_doorbells *
1073                                      sizeof(u32));
1074         if (adev->doorbell.ptr == NULL)
1075                 return -ENOMEM;
1076
1077         return 0;
1078 }
1079
1080 /**
1081  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1082  *
1083  * @adev: amdgpu_device pointer
1084  *
1085  * Tear down doorbell driver information (CIK)
1086  */
1087 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1088 {
1089         iounmap(adev->doorbell.ptr);
1090         adev->doorbell.ptr = NULL;
1091 }
1092
1093
1094
1095 /*
1096  * amdgpu_device_wb_*()
1097  * Writeback is the method by which the GPU updates special pages in memory
1098  * with the status of certain GPU events (fences, ring pointers,etc.).
1099  */
1100
1101 /**
1102  * amdgpu_device_wb_fini - Disable Writeback and free memory
1103  *
1104  * @adev: amdgpu_device pointer
1105  *
1106  * Disables Writeback and frees the Writeback memory (all asics).
1107  * Used at driver shutdown.
1108  */
1109 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1110 {
1111         if (adev->wb.wb_obj) {
1112                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1113                                       &adev->wb.gpu_addr,
1114                                       (void **)&adev->wb.wb);
1115                 adev->wb.wb_obj = NULL;
1116         }
1117 }
1118
1119 /**
1120  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1121  *
1122  * @adev: amdgpu_device pointer
1123  *
1124  * Initializes writeback and allocates writeback memory (all asics).
1125  * Used at driver startup.
1126  * Returns 0 on success or an -error on failure.
1127  */
1128 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1129 {
1130         int r;
1131
1132         if (adev->wb.wb_obj == NULL) {
1133                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1134                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1135                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1136                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1137                                             (void **)&adev->wb.wb);
1138                 if (r) {
1139                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1140                         return r;
1141                 }
1142
1143                 adev->wb.num_wb = AMDGPU_MAX_WB;
1144                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1145
1146                 /* clear wb memory */
1147                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1148         }
1149
1150         return 0;
1151 }
1152
1153 /**
1154  * amdgpu_device_wb_get - Allocate a wb entry
1155  *
1156  * @adev: amdgpu_device pointer
1157  * @wb: wb index
1158  *
1159  * Allocate a wb slot for use by the driver (all asics).
1160  * Returns 0 on success or -EINVAL on failure.
1161  */
1162 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1163 {
1164         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1165
1166         if (offset < adev->wb.num_wb) {
1167                 __set_bit(offset, adev->wb.used);
1168                 *wb = offset << 3; /* convert to dw offset */
1169                 return 0;
1170         } else {
1171                 return -EINVAL;
1172         }
1173 }
1174
1175 /**
1176  * amdgpu_device_wb_free - Free a wb entry
1177  *
1178  * @adev: amdgpu_device pointer
1179  * @wb: wb index
1180  *
1181  * Free a wb slot allocated for use by the driver (all asics)
1182  */
1183 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1184 {
1185         wb >>= 3;
1186         if (wb < adev->wb.num_wb)
1187                 __clear_bit(wb, adev->wb.used);
1188 }
1189
1190 /**
1191  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1192  *
1193  * @adev: amdgpu_device pointer
1194  *
1195  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1196  * to fail, but if any of the BARs is not accessible after the size we abort
1197  * driver loading by returning -ENODEV.
1198  */
1199 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1200 {
1201         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1202         struct pci_bus *root;
1203         struct resource *res;
1204         unsigned i;
1205         u16 cmd;
1206         int r;
1207
1208         /* Bypass for VF */
1209         if (amdgpu_sriov_vf(adev))
1210                 return 0;
1211
1212         /* skip if the bios has already enabled large BAR */
1213         if (adev->gmc.real_vram_size &&
1214             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1215                 return 0;
1216
1217         /* Check if the root BUS has 64bit memory resources */
1218         root = adev->pdev->bus;
1219         while (root->parent)
1220                 root = root->parent;
1221
1222         pci_bus_for_each_resource(root, res, i) {
1223                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1224                     res->start > 0x100000000ull)
1225                         break;
1226         }
1227
1228         /* Trying to resize is pointless without a root hub window above 4GB */
1229         if (!res)
1230                 return 0;
1231
1232         /* Limit the BAR size to what is available */
1233         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1234                         rbar_size);
1235
1236         /* Disable memory decoding while we change the BAR addresses and size */
1237         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1238         pci_write_config_word(adev->pdev, PCI_COMMAND,
1239                               cmd & ~PCI_COMMAND_MEMORY);
1240
1241         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1242         amdgpu_device_doorbell_fini(adev);
1243         if (adev->asic_type >= CHIP_BONAIRE)
1244                 pci_release_resource(adev->pdev, 2);
1245
1246         pci_release_resource(adev->pdev, 0);
1247
1248         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1249         if (r == -ENOSPC)
1250                 DRM_INFO("Not enough PCI address space for a large BAR.");
1251         else if (r && r != -ENOTSUPP)
1252                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1253
1254         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1255
1256         /* When the doorbell or fb BAR isn't available we have no chance of
1257          * using the device.
1258          */
1259         r = amdgpu_device_doorbell_init(adev);
1260         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1261                 return -ENODEV;
1262
1263         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1264
1265         return 0;
1266 }
1267
1268 /*
1269  * GPU helpers function.
1270  */
1271 /**
1272  * amdgpu_device_need_post - check if the hw need post or not
1273  *
1274  * @adev: amdgpu_device pointer
1275  *
1276  * Check if the asic has been initialized (all asics) at driver startup
1277  * or post is needed if  hw reset is performed.
1278  * Returns true if need or false if not.
1279  */
1280 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1281 {
1282         uint32_t reg;
1283
1284         if (amdgpu_sriov_vf(adev))
1285                 return false;
1286
1287         if (amdgpu_passthrough(adev)) {
1288                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1289                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1290                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1291                  * vpost executed for smc version below 22.15
1292                  */
1293                 if (adev->asic_type == CHIP_FIJI) {
1294                         int err;
1295                         uint32_t fw_ver;
1296                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1297                         /* force vPost if error occured */
1298                         if (err)
1299                                 return true;
1300
1301                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1302                         if (fw_ver < 0x00160e00)
1303                                 return true;
1304                 }
1305         }
1306
1307         /* Don't post if we need to reset whole hive on init */
1308         if (adev->gmc.xgmi.pending_reset)
1309                 return false;
1310
1311         if (adev->has_hw_reset) {
1312                 adev->has_hw_reset = false;
1313                 return true;
1314         }
1315
1316         /* bios scratch used on CIK+ */
1317         if (adev->asic_type >= CHIP_BONAIRE)
1318                 return amdgpu_atombios_scratch_need_asic_init(adev);
1319
1320         /* check MEM_SIZE for older asics */
1321         reg = amdgpu_asic_get_config_memsize(adev);
1322
1323         if ((reg != 0) && (reg != 0xffffffff))
1324                 return false;
1325
1326         return true;
1327 }
1328
1329 /**
1330  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1331  *
1332  * @adev: amdgpu_device pointer
1333  *
1334  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1335  * be set for this device.
1336  *
1337  * Returns true if it should be used or false if not.
1338  */
1339 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1340 {
1341         switch (amdgpu_aspm) {
1342         case -1:
1343                 break;
1344         case 0:
1345                 return false;
1346         case 1:
1347                 return true;
1348         default:
1349                 return false;
1350         }
1351         return pcie_aspm_enabled(adev->pdev);
1352 }
1353
1354 /* if we get transitioned to only one device, take VGA back */
1355 /**
1356  * amdgpu_device_vga_set_decode - enable/disable vga decode
1357  *
1358  * @pdev: PCI device pointer
1359  * @state: enable/disable vga decode
1360  *
1361  * Enable/disable vga decode (all asics).
1362  * Returns VGA resource flags.
1363  */
1364 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1365                 bool state)
1366 {
1367         struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1368         amdgpu_asic_set_vga_state(adev, state);
1369         if (state)
1370                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1371                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1372         else
1373                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1374 }
1375
1376 /**
1377  * amdgpu_device_check_block_size - validate the vm block size
1378  *
1379  * @adev: amdgpu_device pointer
1380  *
1381  * Validates the vm block size specified via module parameter.
1382  * The vm block size defines number of bits in page table versus page directory,
1383  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1384  * page table and the remaining bits are in the page directory.
1385  */
1386 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1387 {
1388         /* defines number of bits in page table versus page directory,
1389          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1390          * page table and the remaining bits are in the page directory */
1391         if (amdgpu_vm_block_size == -1)
1392                 return;
1393
1394         if (amdgpu_vm_block_size < 9) {
1395                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1396                          amdgpu_vm_block_size);
1397                 amdgpu_vm_block_size = -1;
1398         }
1399 }
1400
1401 /**
1402  * amdgpu_device_check_vm_size - validate the vm size
1403  *
1404  * @adev: amdgpu_device pointer
1405  *
1406  * Validates the vm size in GB specified via module parameter.
1407  * The VM size is the size of the GPU virtual memory space in GB.
1408  */
1409 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1410 {
1411         /* no need to check the default value */
1412         if (amdgpu_vm_size == -1)
1413                 return;
1414
1415         if (amdgpu_vm_size < 1) {
1416                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1417                          amdgpu_vm_size);
1418                 amdgpu_vm_size = -1;
1419         }
1420 }
1421
1422 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1423 {
1424         struct sysinfo si;
1425         bool is_os_64 = (sizeof(void *) == 8);
1426         uint64_t total_memory;
1427         uint64_t dram_size_seven_GB = 0x1B8000000;
1428         uint64_t dram_size_three_GB = 0xB8000000;
1429
1430         if (amdgpu_smu_memory_pool_size == 0)
1431                 return;
1432
1433         if (!is_os_64) {
1434                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1435                 goto def_value;
1436         }
1437         si_meminfo(&si);
1438         total_memory = (uint64_t)si.totalram * si.mem_unit;
1439
1440         if ((amdgpu_smu_memory_pool_size == 1) ||
1441                 (amdgpu_smu_memory_pool_size == 2)) {
1442                 if (total_memory < dram_size_three_GB)
1443                         goto def_value1;
1444         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1445                 (amdgpu_smu_memory_pool_size == 8)) {
1446                 if (total_memory < dram_size_seven_GB)
1447                         goto def_value1;
1448         } else {
1449                 DRM_WARN("Smu memory pool size not supported\n");
1450                 goto def_value;
1451         }
1452         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1453
1454         return;
1455
1456 def_value1:
1457         DRM_WARN("No enough system memory\n");
1458 def_value:
1459         adev->pm.smu_prv_buffer_size = 0;
1460 }
1461
1462 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1463 {
1464         if (!(adev->flags & AMD_IS_APU) ||
1465             adev->asic_type < CHIP_RAVEN)
1466                 return 0;
1467
1468         switch (adev->asic_type) {
1469         case CHIP_RAVEN:
1470                 if (adev->pdev->device == 0x15dd)
1471                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1472                 if (adev->pdev->device == 0x15d8)
1473                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1474                 break;
1475         case CHIP_RENOIR:
1476                 if ((adev->pdev->device == 0x1636) ||
1477                     (adev->pdev->device == 0x164c))
1478                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1479                 else
1480                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1481                 break;
1482         case CHIP_VANGOGH:
1483                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1484                 break;
1485         case CHIP_YELLOW_CARP:
1486                 break;
1487         case CHIP_CYAN_SKILLFISH:
1488                 if ((adev->pdev->device == 0x13FE) ||
1489                     (adev->pdev->device == 0x143F))
1490                         adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1491                 break;
1492         default:
1493                 break;
1494         }
1495
1496         return 0;
1497 }
1498
1499 /**
1500  * amdgpu_device_check_arguments - validate module params
1501  *
1502  * @adev: amdgpu_device pointer
1503  *
1504  * Validates certain module parameters and updates
1505  * the associated values used by the driver (all asics).
1506  */
1507 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1508 {
1509         if (amdgpu_sched_jobs < 4) {
1510                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1511                          amdgpu_sched_jobs);
1512                 amdgpu_sched_jobs = 4;
1513         } else if (!is_power_of_2(amdgpu_sched_jobs)){
1514                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1515                          amdgpu_sched_jobs);
1516                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1517         }
1518
1519         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1520                 /* gart size must be greater or equal to 32M */
1521                 dev_warn(adev->dev, "gart size (%d) too small\n",
1522                          amdgpu_gart_size);
1523                 amdgpu_gart_size = -1;
1524         }
1525
1526         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1527                 /* gtt size must be greater or equal to 32M */
1528                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1529                                  amdgpu_gtt_size);
1530                 amdgpu_gtt_size = -1;
1531         }
1532
1533         /* valid range is between 4 and 9 inclusive */
1534         if (amdgpu_vm_fragment_size != -1 &&
1535             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1536                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1537                 amdgpu_vm_fragment_size = -1;
1538         }
1539
1540         if (amdgpu_sched_hw_submission < 2) {
1541                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1542                          amdgpu_sched_hw_submission);
1543                 amdgpu_sched_hw_submission = 2;
1544         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1545                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1546                          amdgpu_sched_hw_submission);
1547                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1548         }
1549
1550         if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1551                 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1552                 amdgpu_reset_method = -1;
1553         }
1554
1555         amdgpu_device_check_smu_prv_buffer_size(adev);
1556
1557         amdgpu_device_check_vm_size(adev);
1558
1559         amdgpu_device_check_block_size(adev);
1560
1561         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1562
1563         return 0;
1564 }
1565
1566 /**
1567  * amdgpu_switcheroo_set_state - set switcheroo state
1568  *
1569  * @pdev: pci dev pointer
1570  * @state: vga_switcheroo state
1571  *
1572  * Callback for the switcheroo driver.  Suspends or resumes
1573  * the asics before or after it is powered up using ACPI methods.
1574  */
1575 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1576                                         enum vga_switcheroo_state state)
1577 {
1578         struct drm_device *dev = pci_get_drvdata(pdev);
1579         int r;
1580
1581         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1582                 return;
1583
1584         if (state == VGA_SWITCHEROO_ON) {
1585                 pr_info("switched on\n");
1586                 /* don't suspend or resume card normally */
1587                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1588
1589                 pci_set_power_state(pdev, PCI_D0);
1590                 amdgpu_device_load_pci_state(pdev);
1591                 r = pci_enable_device(pdev);
1592                 if (r)
1593                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1594                 amdgpu_device_resume(dev, true);
1595
1596                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1597         } else {
1598                 pr_info("switched off\n");
1599                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1600                 amdgpu_device_suspend(dev, true);
1601                 amdgpu_device_cache_pci_state(pdev);
1602                 /* Shut down the device */
1603                 pci_disable_device(pdev);
1604                 pci_set_power_state(pdev, PCI_D3cold);
1605                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1606         }
1607 }
1608
1609 /**
1610  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1611  *
1612  * @pdev: pci dev pointer
1613  *
1614  * Callback for the switcheroo driver.  Check of the switcheroo
1615  * state can be changed.
1616  * Returns true if the state can be changed, false if not.
1617  */
1618 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1619 {
1620         struct drm_device *dev = pci_get_drvdata(pdev);
1621
1622         /*
1623         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1624         * locking inversion with the driver load path. And the access here is
1625         * completely racy anyway. So don't bother with locking for now.
1626         */
1627         return atomic_read(&dev->open_count) == 0;
1628 }
1629
1630 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1631         .set_gpu_state = amdgpu_switcheroo_set_state,
1632         .reprobe = NULL,
1633         .can_switch = amdgpu_switcheroo_can_switch,
1634 };
1635
1636 /**
1637  * amdgpu_device_ip_set_clockgating_state - set the CG state
1638  *
1639  * @dev: amdgpu_device pointer
1640  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1641  * @state: clockgating state (gate or ungate)
1642  *
1643  * Sets the requested clockgating state for all instances of
1644  * the hardware IP specified.
1645  * Returns the error code from the last instance.
1646  */
1647 int amdgpu_device_ip_set_clockgating_state(void *dev,
1648                                            enum amd_ip_block_type block_type,
1649                                            enum amd_clockgating_state state)
1650 {
1651         struct amdgpu_device *adev = dev;
1652         int i, r = 0;
1653
1654         for (i = 0; i < adev->num_ip_blocks; i++) {
1655                 if (!adev->ip_blocks[i].status.valid)
1656                         continue;
1657                 if (adev->ip_blocks[i].version->type != block_type)
1658                         continue;
1659                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1660                         continue;
1661                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1662                         (void *)adev, state);
1663                 if (r)
1664                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1665                                   adev->ip_blocks[i].version->funcs->name, r);
1666         }
1667         return r;
1668 }
1669
1670 /**
1671  * amdgpu_device_ip_set_powergating_state - set the PG state
1672  *
1673  * @dev: amdgpu_device pointer
1674  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1675  * @state: powergating state (gate or ungate)
1676  *
1677  * Sets the requested powergating state for all instances of
1678  * the hardware IP specified.
1679  * Returns the error code from the last instance.
1680  */
1681 int amdgpu_device_ip_set_powergating_state(void *dev,
1682                                            enum amd_ip_block_type block_type,
1683                                            enum amd_powergating_state state)
1684 {
1685         struct amdgpu_device *adev = dev;
1686         int i, r = 0;
1687
1688         for (i = 0; i < adev->num_ip_blocks; i++) {
1689                 if (!adev->ip_blocks[i].status.valid)
1690                         continue;
1691                 if (adev->ip_blocks[i].version->type != block_type)
1692                         continue;
1693                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1694                         continue;
1695                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1696                         (void *)adev, state);
1697                 if (r)
1698                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1699                                   adev->ip_blocks[i].version->funcs->name, r);
1700         }
1701         return r;
1702 }
1703
1704 /**
1705  * amdgpu_device_ip_get_clockgating_state - get the CG state
1706  *
1707  * @adev: amdgpu_device pointer
1708  * @flags: clockgating feature flags
1709  *
1710  * Walks the list of IPs on the device and updates the clockgating
1711  * flags for each IP.
1712  * Updates @flags with the feature flags for each hardware IP where
1713  * clockgating is enabled.
1714  */
1715 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1716                                             u64 *flags)
1717 {
1718         int i;
1719
1720         for (i = 0; i < adev->num_ip_blocks; i++) {
1721                 if (!adev->ip_blocks[i].status.valid)
1722                         continue;
1723                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1724                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1725         }
1726 }
1727
1728 /**
1729  * amdgpu_device_ip_wait_for_idle - wait for idle
1730  *
1731  * @adev: amdgpu_device pointer
1732  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1733  *
1734  * Waits for the request hardware IP to be idle.
1735  * Returns 0 for success or a negative error code on failure.
1736  */
1737 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1738                                    enum amd_ip_block_type block_type)
1739 {
1740         int i, r;
1741
1742         for (i = 0; i < adev->num_ip_blocks; i++) {
1743                 if (!adev->ip_blocks[i].status.valid)
1744                         continue;
1745                 if (adev->ip_blocks[i].version->type == block_type) {
1746                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1747                         if (r)
1748                                 return r;
1749                         break;
1750                 }
1751         }
1752         return 0;
1753
1754 }
1755
1756 /**
1757  * amdgpu_device_ip_is_idle - is the hardware IP idle
1758  *
1759  * @adev: amdgpu_device pointer
1760  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1761  *
1762  * Check if the hardware IP is idle or not.
1763  * Returns true if it the IP is idle, false if not.
1764  */
1765 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1766                               enum amd_ip_block_type block_type)
1767 {
1768         int i;
1769
1770         for (i = 0; i < adev->num_ip_blocks; i++) {
1771                 if (!adev->ip_blocks[i].status.valid)
1772                         continue;
1773                 if (adev->ip_blocks[i].version->type == block_type)
1774                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1775         }
1776         return true;
1777
1778 }
1779
1780 /**
1781  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1782  *
1783  * @adev: amdgpu_device pointer
1784  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1785  *
1786  * Returns a pointer to the hardware IP block structure
1787  * if it exists for the asic, otherwise NULL.
1788  */
1789 struct amdgpu_ip_block *
1790 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1791                               enum amd_ip_block_type type)
1792 {
1793         int i;
1794
1795         for (i = 0; i < adev->num_ip_blocks; i++)
1796                 if (adev->ip_blocks[i].version->type == type)
1797                         return &adev->ip_blocks[i];
1798
1799         return NULL;
1800 }
1801
1802 /**
1803  * amdgpu_device_ip_block_version_cmp
1804  *
1805  * @adev: amdgpu_device pointer
1806  * @type: enum amd_ip_block_type
1807  * @major: major version
1808  * @minor: minor version
1809  *
1810  * return 0 if equal or greater
1811  * return 1 if smaller or the ip_block doesn't exist
1812  */
1813 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1814                                        enum amd_ip_block_type type,
1815                                        u32 major, u32 minor)
1816 {
1817         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1818
1819         if (ip_block && ((ip_block->version->major > major) ||
1820                         ((ip_block->version->major == major) &&
1821                         (ip_block->version->minor >= minor))))
1822                 return 0;
1823
1824         return 1;
1825 }
1826
1827 /**
1828  * amdgpu_device_ip_block_add
1829  *
1830  * @adev: amdgpu_device pointer
1831  * @ip_block_version: pointer to the IP to add
1832  *
1833  * Adds the IP block driver information to the collection of IPs
1834  * on the asic.
1835  */
1836 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1837                                const struct amdgpu_ip_block_version *ip_block_version)
1838 {
1839         if (!ip_block_version)
1840                 return -EINVAL;
1841
1842         switch (ip_block_version->type) {
1843         case AMD_IP_BLOCK_TYPE_VCN:
1844                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
1845                         return 0;
1846                 break;
1847         case AMD_IP_BLOCK_TYPE_JPEG:
1848                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
1849                         return 0;
1850                 break;
1851         default:
1852                 break;
1853         }
1854
1855         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1856                   ip_block_version->funcs->name);
1857
1858         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1859
1860         return 0;
1861 }
1862
1863 /**
1864  * amdgpu_device_enable_virtual_display - enable virtual display feature
1865  *
1866  * @adev: amdgpu_device pointer
1867  *
1868  * Enabled the virtual display feature if the user has enabled it via
1869  * the module parameter virtual_display.  This feature provides a virtual
1870  * display hardware on headless boards or in virtualized environments.
1871  * This function parses and validates the configuration string specified by
1872  * the user and configues the virtual display configuration (number of
1873  * virtual connectors, crtcs, etc.) specified.
1874  */
1875 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1876 {
1877         adev->enable_virtual_display = false;
1878
1879         if (amdgpu_virtual_display) {
1880                 const char *pci_address_name = pci_name(adev->pdev);
1881                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1882
1883                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1884                 pciaddstr_tmp = pciaddstr;
1885                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1886                         pciaddname = strsep(&pciaddname_tmp, ",");
1887                         if (!strcmp("all", pciaddname)
1888                             || !strcmp(pci_address_name, pciaddname)) {
1889                                 long num_crtc;
1890                                 int res = -1;
1891
1892                                 adev->enable_virtual_display = true;
1893
1894                                 if (pciaddname_tmp)
1895                                         res = kstrtol(pciaddname_tmp, 10,
1896                                                       &num_crtc);
1897
1898                                 if (!res) {
1899                                         if (num_crtc < 1)
1900                                                 num_crtc = 1;
1901                                         if (num_crtc > 6)
1902                                                 num_crtc = 6;
1903                                         adev->mode_info.num_crtc = num_crtc;
1904                                 } else {
1905                                         adev->mode_info.num_crtc = 1;
1906                                 }
1907                                 break;
1908                         }
1909                 }
1910
1911                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1912                          amdgpu_virtual_display, pci_address_name,
1913                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1914
1915                 kfree(pciaddstr);
1916         }
1917 }
1918
1919 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
1920 {
1921         if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
1922                 adev->mode_info.num_crtc = 1;
1923                 adev->enable_virtual_display = true;
1924                 DRM_INFO("virtual_display:%d, num_crtc:%d\n",
1925                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1926         }
1927 }
1928
1929 /**
1930  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1931  *
1932  * @adev: amdgpu_device pointer
1933  *
1934  * Parses the asic configuration parameters specified in the gpu info
1935  * firmware and makes them availale to the driver for use in configuring
1936  * the asic.
1937  * Returns 0 on success, -EINVAL on failure.
1938  */
1939 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1940 {
1941         const char *chip_name;
1942         char fw_name[40];
1943         int err;
1944         const struct gpu_info_firmware_header_v1_0 *hdr;
1945
1946         adev->firmware.gpu_info_fw = NULL;
1947
1948         if (adev->mman.discovery_bin) {
1949                 /*
1950                  * FIXME: The bounding box is still needed by Navi12, so
1951                  * temporarily read it from gpu_info firmware. Should be dropped
1952                  * when DAL no longer needs it.
1953                  */
1954                 if (adev->asic_type != CHIP_NAVI12)
1955                         return 0;
1956         }
1957
1958         switch (adev->asic_type) {
1959         default:
1960                 return 0;
1961         case CHIP_VEGA10:
1962                 chip_name = "vega10";
1963                 break;
1964         case CHIP_VEGA12:
1965                 chip_name = "vega12";
1966                 break;
1967         case CHIP_RAVEN:
1968                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1969                         chip_name = "raven2";
1970                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1971                         chip_name = "picasso";
1972                 else
1973                         chip_name = "raven";
1974                 break;
1975         case CHIP_ARCTURUS:
1976                 chip_name = "arcturus";
1977                 break;
1978         case CHIP_NAVI12:
1979                 chip_name = "navi12";
1980                 break;
1981         }
1982
1983         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1984         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1985         if (err) {
1986                 dev_err(adev->dev,
1987                         "Failed to load gpu_info firmware \"%s\"\n",
1988                         fw_name);
1989                 goto out;
1990         }
1991         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1992         if (err) {
1993                 dev_err(adev->dev,
1994                         "Failed to validate gpu_info firmware \"%s\"\n",
1995                         fw_name);
1996                 goto out;
1997         }
1998
1999         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2000         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2001
2002         switch (hdr->version_major) {
2003         case 1:
2004         {
2005                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2006                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2007                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2008
2009                 /*
2010                  * Should be droped when DAL no longer needs it.
2011                  */
2012                 if (adev->asic_type == CHIP_NAVI12)
2013                         goto parse_soc_bounding_box;
2014
2015                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2016                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2017                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2018                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2019                 adev->gfx.config.max_texture_channel_caches =
2020                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
2021                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2022                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2023                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2024                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2025                 adev->gfx.config.double_offchip_lds_buf =
2026                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2027                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2028                 adev->gfx.cu_info.max_waves_per_simd =
2029                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2030                 adev->gfx.cu_info.max_scratch_slots_per_cu =
2031                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2032                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2033                 if (hdr->version_minor >= 1) {
2034                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2035                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2036                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2037                         adev->gfx.config.num_sc_per_sh =
2038                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2039                         adev->gfx.config.num_packer_per_sc =
2040                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2041                 }
2042
2043 parse_soc_bounding_box:
2044                 /*
2045                  * soc bounding box info is not integrated in disocovery table,
2046                  * we always need to parse it from gpu info firmware if needed.
2047                  */
2048                 if (hdr->version_minor == 2) {
2049                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2050                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2051                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2052                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2053                 }
2054                 break;
2055         }
2056         default:
2057                 dev_err(adev->dev,
2058                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2059                 err = -EINVAL;
2060                 goto out;
2061         }
2062 out:
2063         return err;
2064 }
2065
2066 /**
2067  * amdgpu_device_ip_early_init - run early init for hardware IPs
2068  *
2069  * @adev: amdgpu_device pointer
2070  *
2071  * Early initialization pass for hardware IPs.  The hardware IPs that make
2072  * up each asic are discovered each IP's early_init callback is run.  This
2073  * is the first stage in initializing the asic.
2074  * Returns 0 on success, negative error code on failure.
2075  */
2076 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2077 {
2078         struct drm_device *dev = adev_to_drm(adev);
2079         struct pci_dev *parent;
2080         int i, r;
2081
2082         amdgpu_device_enable_virtual_display(adev);
2083
2084         if (amdgpu_sriov_vf(adev)) {
2085                 r = amdgpu_virt_request_full_gpu(adev, true);
2086                 if (r)
2087                         return r;
2088         }
2089
2090         switch (adev->asic_type) {
2091 #ifdef CONFIG_DRM_AMDGPU_SI
2092         case CHIP_VERDE:
2093         case CHIP_TAHITI:
2094         case CHIP_PITCAIRN:
2095         case CHIP_OLAND:
2096         case CHIP_HAINAN:
2097                 adev->family = AMDGPU_FAMILY_SI;
2098                 r = si_set_ip_blocks(adev);
2099                 if (r)
2100                         return r;
2101                 break;
2102 #endif
2103 #ifdef CONFIG_DRM_AMDGPU_CIK
2104         case CHIP_BONAIRE:
2105         case CHIP_HAWAII:
2106         case CHIP_KAVERI:
2107         case CHIP_KABINI:
2108         case CHIP_MULLINS:
2109                 if (adev->flags & AMD_IS_APU)
2110                         adev->family = AMDGPU_FAMILY_KV;
2111                 else
2112                         adev->family = AMDGPU_FAMILY_CI;
2113
2114                 r = cik_set_ip_blocks(adev);
2115                 if (r)
2116                         return r;
2117                 break;
2118 #endif
2119         case CHIP_TOPAZ:
2120         case CHIP_TONGA:
2121         case CHIP_FIJI:
2122         case CHIP_POLARIS10:
2123         case CHIP_POLARIS11:
2124         case CHIP_POLARIS12:
2125         case CHIP_VEGAM:
2126         case CHIP_CARRIZO:
2127         case CHIP_STONEY:
2128                 if (adev->flags & AMD_IS_APU)
2129                         adev->family = AMDGPU_FAMILY_CZ;
2130                 else
2131                         adev->family = AMDGPU_FAMILY_VI;
2132
2133                 r = vi_set_ip_blocks(adev);
2134                 if (r)
2135                         return r;
2136                 break;
2137         default:
2138                 r = amdgpu_discovery_set_ip_blocks(adev);
2139                 if (r)
2140                         return r;
2141                 break;
2142         }
2143
2144         if (amdgpu_has_atpx() &&
2145             (amdgpu_is_atpx_hybrid() ||
2146              amdgpu_has_atpx_dgpu_power_cntl()) &&
2147             ((adev->flags & AMD_IS_APU) == 0) &&
2148             !pci_is_thunderbolt_attached(to_pci_dev(dev->dev)))
2149                 adev->flags |= AMD_IS_PX;
2150
2151         if (!(adev->flags & AMD_IS_APU)) {
2152                 parent = pci_upstream_bridge(adev->pdev);
2153                 adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2154         }
2155
2156         amdgpu_amdkfd_device_probe(adev);
2157
2158         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2159         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2160                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2161         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2162                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2163
2164         for (i = 0; i < adev->num_ip_blocks; i++) {
2165                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2166                         DRM_ERROR("disabled ip block: %d <%s>\n",
2167                                   i, adev->ip_blocks[i].version->funcs->name);
2168                         adev->ip_blocks[i].status.valid = false;
2169                 } else {
2170                         if (adev->ip_blocks[i].version->funcs->early_init) {
2171                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2172                                 if (r == -ENOENT) {
2173                                         adev->ip_blocks[i].status.valid = false;
2174                                 } else if (r) {
2175                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2176                                                   adev->ip_blocks[i].version->funcs->name, r);
2177                                         return r;
2178                                 } else {
2179                                         adev->ip_blocks[i].status.valid = true;
2180                                 }
2181                         } else {
2182                                 adev->ip_blocks[i].status.valid = true;
2183                         }
2184                 }
2185                 /* get the vbios after the asic_funcs are set up */
2186                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2187                         r = amdgpu_device_parse_gpu_info_fw(adev);
2188                         if (r)
2189                                 return r;
2190
2191                         /* Read BIOS */
2192                         if (!amdgpu_get_bios(adev))
2193                                 return -EINVAL;
2194
2195                         r = amdgpu_atombios_init(adev);
2196                         if (r) {
2197                                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2198                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2199                                 return r;
2200                         }
2201
2202                         /*get pf2vf msg info at it's earliest time*/
2203                         if (amdgpu_sriov_vf(adev))
2204                                 amdgpu_virt_init_data_exchange(adev);
2205
2206                 }
2207         }
2208
2209         adev->cg_flags &= amdgpu_cg_mask;
2210         adev->pg_flags &= amdgpu_pg_mask;
2211
2212         return 0;
2213 }
2214
2215 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2216 {
2217         int i, r;
2218
2219         for (i = 0; i < adev->num_ip_blocks; i++) {
2220                 if (!adev->ip_blocks[i].status.sw)
2221                         continue;
2222                 if (adev->ip_blocks[i].status.hw)
2223                         continue;
2224                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2225                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2226                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2227                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2228                         if (r) {
2229                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2230                                           adev->ip_blocks[i].version->funcs->name, r);
2231                                 return r;
2232                         }
2233                         adev->ip_blocks[i].status.hw = true;
2234                 }
2235         }
2236
2237         return 0;
2238 }
2239
2240 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2241 {
2242         int i, r;
2243
2244         for (i = 0; i < adev->num_ip_blocks; i++) {
2245                 if (!adev->ip_blocks[i].status.sw)
2246                         continue;
2247                 if (adev->ip_blocks[i].status.hw)
2248                         continue;
2249                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2250                 if (r) {
2251                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2252                                   adev->ip_blocks[i].version->funcs->name, r);
2253                         return r;
2254                 }
2255                 adev->ip_blocks[i].status.hw = true;
2256         }
2257
2258         return 0;
2259 }
2260
2261 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2262 {
2263         int r = 0;
2264         int i;
2265         uint32_t smu_version;
2266
2267         if (adev->asic_type >= CHIP_VEGA10) {
2268                 for (i = 0; i < adev->num_ip_blocks; i++) {
2269                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2270                                 continue;
2271
2272                         if (!adev->ip_blocks[i].status.sw)
2273                                 continue;
2274
2275                         /* no need to do the fw loading again if already done*/
2276                         if (adev->ip_blocks[i].status.hw == true)
2277                                 break;
2278
2279                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2280                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2281                                 if (r) {
2282                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2283                                                           adev->ip_blocks[i].version->funcs->name, r);
2284                                         return r;
2285                                 }
2286                         } else {
2287                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2288                                 if (r) {
2289                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2290                                                           adev->ip_blocks[i].version->funcs->name, r);
2291                                         return r;
2292                                 }
2293                         }
2294
2295                         adev->ip_blocks[i].status.hw = true;
2296                         break;
2297                 }
2298         }
2299
2300         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2301                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2302
2303         return r;
2304 }
2305
2306 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2307 {
2308         long timeout;
2309         int r, i;
2310
2311         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2312                 struct amdgpu_ring *ring = adev->rings[i];
2313
2314                 /* No need to setup the GPU scheduler for rings that don't need it */
2315                 if (!ring || ring->no_scheduler)
2316                         continue;
2317
2318                 switch (ring->funcs->type) {
2319                 case AMDGPU_RING_TYPE_GFX:
2320                         timeout = adev->gfx_timeout;
2321                         break;
2322                 case AMDGPU_RING_TYPE_COMPUTE:
2323                         timeout = adev->compute_timeout;
2324                         break;
2325                 case AMDGPU_RING_TYPE_SDMA:
2326                         timeout = adev->sdma_timeout;
2327                         break;
2328                 default:
2329                         timeout = adev->video_timeout;
2330                         break;
2331                 }
2332
2333                 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
2334                                    ring->num_hw_submission, amdgpu_job_hang_limit,
2335                                    timeout, adev->reset_domain->wq,
2336                                    ring->sched_score, ring->name,
2337                                    adev->dev);
2338                 if (r) {
2339                         DRM_ERROR("Failed to create scheduler on ring %s.\n",
2340                                   ring->name);
2341                         return r;
2342                 }
2343         }
2344
2345         return 0;
2346 }
2347
2348
2349 /**
2350  * amdgpu_device_ip_init - run init for hardware IPs
2351  *
2352  * @adev: amdgpu_device pointer
2353  *
2354  * Main initialization pass for hardware IPs.  The list of all the hardware
2355  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2356  * are run.  sw_init initializes the software state associated with each IP
2357  * and hw_init initializes the hardware associated with each IP.
2358  * Returns 0 on success, negative error code on failure.
2359  */
2360 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2361 {
2362         int i, r;
2363
2364         r = amdgpu_ras_init(adev);
2365         if (r)
2366                 return r;
2367
2368         for (i = 0; i < adev->num_ip_blocks; i++) {
2369                 if (!adev->ip_blocks[i].status.valid)
2370                         continue;
2371                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2372                 if (r) {
2373                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2374                                   adev->ip_blocks[i].version->funcs->name, r);
2375                         goto init_failed;
2376                 }
2377                 adev->ip_blocks[i].status.sw = true;
2378
2379                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2380                         /* need to do common hw init early so everything is set up for gmc */
2381                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2382                         if (r) {
2383                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2384                                 goto init_failed;
2385                         }
2386                         adev->ip_blocks[i].status.hw = true;
2387                 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2388                         /* need to do gmc hw init early so we can allocate gpu mem */
2389                         /* Try to reserve bad pages early */
2390                         if (amdgpu_sriov_vf(adev))
2391                                 amdgpu_virt_exchange_data(adev);
2392
2393                         r = amdgpu_device_vram_scratch_init(adev);
2394                         if (r) {
2395                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
2396                                 goto init_failed;
2397                         }
2398                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2399                         if (r) {
2400                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2401                                 goto init_failed;
2402                         }
2403                         r = amdgpu_device_wb_init(adev);
2404                         if (r) {
2405                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2406                                 goto init_failed;
2407                         }
2408                         adev->ip_blocks[i].status.hw = true;
2409
2410                         /* right after GMC hw init, we create CSA */
2411                         if (amdgpu_mcbp) {
2412                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2413                                                                 AMDGPU_GEM_DOMAIN_VRAM,
2414                                                                 AMDGPU_CSA_SIZE);
2415                                 if (r) {
2416                                         DRM_ERROR("allocate CSA failed %d\n", r);
2417                                         goto init_failed;
2418                                 }
2419                         }
2420                 }
2421         }
2422
2423         if (amdgpu_sriov_vf(adev))
2424                 amdgpu_virt_init_data_exchange(adev);
2425
2426         r = amdgpu_ib_pool_init(adev);
2427         if (r) {
2428                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2429                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2430                 goto init_failed;
2431         }
2432
2433         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2434         if (r)
2435                 goto init_failed;
2436
2437         r = amdgpu_device_ip_hw_init_phase1(adev);
2438         if (r)
2439                 goto init_failed;
2440
2441         r = amdgpu_device_fw_loading(adev);
2442         if (r)
2443                 goto init_failed;
2444
2445         r = amdgpu_device_ip_hw_init_phase2(adev);
2446         if (r)
2447                 goto init_failed;
2448
2449         /*
2450          * retired pages will be loaded from eeprom and reserved here,
2451          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2452          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2453          * for I2C communication which only true at this point.
2454          *
2455          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2456          * failure from bad gpu situation and stop amdgpu init process
2457          * accordingly. For other failed cases, it will still release all
2458          * the resource and print error message, rather than returning one
2459          * negative value to upper level.
2460          *
2461          * Note: theoretically, this should be called before all vram allocations
2462          * to protect retired page from abusing
2463          */
2464         r = amdgpu_ras_recovery_init(adev);
2465         if (r)
2466                 goto init_failed;
2467
2468         /**
2469          * In case of XGMI grab extra reference for reset domain for this device
2470          */
2471         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2472                 if (amdgpu_xgmi_add_device(adev) == 0) {
2473                         if (!amdgpu_sriov_vf(adev)) {
2474                                 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2475
2476                                 if (WARN_ON(!hive)) {
2477                                         r = -ENOENT;
2478                                         goto init_failed;
2479                                 }
2480
2481                                 if (!hive->reset_domain ||
2482                                     !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2483                                         r = -ENOENT;
2484                                         amdgpu_put_xgmi_hive(hive);
2485                                         goto init_failed;
2486                                 }
2487
2488                                 /* Drop the early temporary reset domain we created for device */
2489                                 amdgpu_reset_put_reset_domain(adev->reset_domain);
2490                                 adev->reset_domain = hive->reset_domain;
2491                                 amdgpu_put_xgmi_hive(hive);
2492                         }
2493                 }
2494         }
2495
2496         r = amdgpu_device_init_schedulers(adev);
2497         if (r)
2498                 goto init_failed;
2499
2500         /* Don't init kfd if whole hive need to be reset during init */
2501         if (!adev->gmc.xgmi.pending_reset)
2502                 amdgpu_amdkfd_device_init(adev);
2503
2504         amdgpu_fru_get_product_info(adev);
2505
2506 init_failed:
2507         if (amdgpu_sriov_vf(adev))
2508                 amdgpu_virt_release_full_gpu(adev, true);
2509
2510         return r;
2511 }
2512
2513 /**
2514  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2515  *
2516  * @adev: amdgpu_device pointer
2517  *
2518  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2519  * this function before a GPU reset.  If the value is retained after a
2520  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2521  */
2522 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2523 {
2524         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2525 }
2526
2527 /**
2528  * amdgpu_device_check_vram_lost - check if vram is valid
2529  *
2530  * @adev: amdgpu_device pointer
2531  *
2532  * Checks the reset magic value written to the gart pointer in VRAM.
2533  * The driver calls this after a GPU reset to see if the contents of
2534  * VRAM is lost or now.
2535  * returns true if vram is lost, false if not.
2536  */
2537 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2538 {
2539         if (memcmp(adev->gart.ptr, adev->reset_magic,
2540                         AMDGPU_RESET_MAGIC_NUM))
2541                 return true;
2542
2543         if (!amdgpu_in_reset(adev))
2544                 return false;
2545
2546         /*
2547          * For all ASICs with baco/mode1 reset, the VRAM is
2548          * always assumed to be lost.
2549          */
2550         switch (amdgpu_asic_reset_method(adev)) {
2551         case AMD_RESET_METHOD_BACO:
2552         case AMD_RESET_METHOD_MODE1:
2553                 return true;
2554         default:
2555                 return false;
2556         }
2557 }
2558
2559 /**
2560  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2561  *
2562  * @adev: amdgpu_device pointer
2563  * @state: clockgating state (gate or ungate)
2564  *
2565  * The list of all the hardware IPs that make up the asic is walked and the
2566  * set_clockgating_state callbacks are run.
2567  * Late initialization pass enabling clockgating for hardware IPs.
2568  * Fini or suspend, pass disabling clockgating for hardware IPs.
2569  * Returns 0 on success, negative error code on failure.
2570  */
2571
2572 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2573                                enum amd_clockgating_state state)
2574 {
2575         int i, j, r;
2576
2577         if (amdgpu_emu_mode == 1)
2578                 return 0;
2579
2580         for (j = 0; j < adev->num_ip_blocks; j++) {
2581                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2582                 if (!adev->ip_blocks[i].status.late_initialized)
2583                         continue;
2584                 /* skip CG for GFX on S0ix */
2585                 if (adev->in_s0ix &&
2586                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2587                         continue;
2588                 /* skip CG for VCE/UVD, it's handled specially */
2589                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2590                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2591                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2592                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2593                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2594                         /* enable clockgating to save power */
2595                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2596                                                                                      state);
2597                         if (r) {
2598                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2599                                           adev->ip_blocks[i].version->funcs->name, r);
2600                                 return r;
2601                         }
2602                 }
2603         }
2604
2605         return 0;
2606 }
2607
2608 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2609                                enum amd_powergating_state state)
2610 {
2611         int i, j, r;
2612
2613         if (amdgpu_emu_mode == 1)
2614                 return 0;
2615
2616         for (j = 0; j < adev->num_ip_blocks; j++) {
2617                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2618                 if (!adev->ip_blocks[i].status.late_initialized)
2619                         continue;
2620                 /* skip PG for GFX on S0ix */
2621                 if (adev->in_s0ix &&
2622                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2623                         continue;
2624                 /* skip CG for VCE/UVD, it's handled specially */
2625                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2626                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2627                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2628                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2629                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2630                         /* enable powergating to save power */
2631                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2632                                                                                         state);
2633                         if (r) {
2634                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2635                                           adev->ip_blocks[i].version->funcs->name, r);
2636                                 return r;
2637                         }
2638                 }
2639         }
2640         return 0;
2641 }
2642
2643 static int amdgpu_device_enable_mgpu_fan_boost(void)
2644 {
2645         struct amdgpu_gpu_instance *gpu_ins;
2646         struct amdgpu_device *adev;
2647         int i, ret = 0;
2648
2649         mutex_lock(&mgpu_info.mutex);
2650
2651         /*
2652          * MGPU fan boost feature should be enabled
2653          * only when there are two or more dGPUs in
2654          * the system
2655          */
2656         if (mgpu_info.num_dgpu < 2)
2657                 goto out;
2658
2659         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2660                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2661                 adev = gpu_ins->adev;
2662                 if (!(adev->flags & AMD_IS_APU) &&
2663                     !gpu_ins->mgpu_fan_enabled) {
2664                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2665                         if (ret)
2666                                 break;
2667
2668                         gpu_ins->mgpu_fan_enabled = 1;
2669                 }
2670         }
2671
2672 out:
2673         mutex_unlock(&mgpu_info.mutex);
2674
2675         return ret;
2676 }
2677
2678 /**
2679  * amdgpu_device_ip_late_init - run late init for hardware IPs
2680  *
2681  * @adev: amdgpu_device pointer
2682  *
2683  * Late initialization pass for hardware IPs.  The list of all the hardware
2684  * IPs that make up the asic is walked and the late_init callbacks are run.
2685  * late_init covers any special initialization that an IP requires
2686  * after all of the have been initialized or something that needs to happen
2687  * late in the init process.
2688  * Returns 0 on success, negative error code on failure.
2689  */
2690 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2691 {
2692         struct amdgpu_gpu_instance *gpu_instance;
2693         int i = 0, r;
2694
2695         for (i = 0; i < adev->num_ip_blocks; i++) {
2696                 if (!adev->ip_blocks[i].status.hw)
2697                         continue;
2698                 if (adev->ip_blocks[i].version->funcs->late_init) {
2699                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2700                         if (r) {
2701                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2702                                           adev->ip_blocks[i].version->funcs->name, r);
2703                                 return r;
2704                         }
2705                 }
2706                 adev->ip_blocks[i].status.late_initialized = true;
2707         }
2708
2709         r = amdgpu_ras_late_init(adev);
2710         if (r) {
2711                 DRM_ERROR("amdgpu_ras_late_init failed %d", r);
2712                 return r;
2713         }
2714
2715         amdgpu_ras_set_error_query_ready(adev, true);
2716
2717         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2718         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2719
2720         amdgpu_device_fill_reset_magic(adev);
2721
2722         r = amdgpu_device_enable_mgpu_fan_boost();
2723         if (r)
2724                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2725
2726         /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2727         if (amdgpu_passthrough(adev) && ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1)||
2728                                adev->asic_type == CHIP_ALDEBARAN ))
2729                 amdgpu_dpm_handle_passthrough_sbr(adev, true);
2730
2731         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2732                 mutex_lock(&mgpu_info.mutex);
2733
2734                 /*
2735                  * Reset device p-state to low as this was booted with high.
2736                  *
2737                  * This should be performed only after all devices from the same
2738                  * hive get initialized.
2739                  *
2740                  * However, it's unknown how many device in the hive in advance.
2741                  * As this is counted one by one during devices initializations.
2742                  *
2743                  * So, we wait for all XGMI interlinked devices initialized.
2744                  * This may bring some delays as those devices may come from
2745                  * different hives. But that should be OK.
2746                  */
2747                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2748                         for (i = 0; i < mgpu_info.num_gpu; i++) {
2749                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
2750                                 if (gpu_instance->adev->flags & AMD_IS_APU)
2751                                         continue;
2752
2753                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2754                                                 AMDGPU_XGMI_PSTATE_MIN);
2755                                 if (r) {
2756                                         DRM_ERROR("pstate setting failed (%d).\n", r);
2757                                         break;
2758                                 }
2759                         }
2760                 }
2761
2762                 mutex_unlock(&mgpu_info.mutex);
2763         }
2764
2765         return 0;
2766 }
2767
2768 /**
2769  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2770  *
2771  * @adev: amdgpu_device pointer
2772  *
2773  * For ASICs need to disable SMC first
2774  */
2775 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2776 {
2777         int i, r;
2778
2779         if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0))
2780                 return;
2781
2782         for (i = 0; i < adev->num_ip_blocks; i++) {
2783                 if (!adev->ip_blocks[i].status.hw)
2784                         continue;
2785                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2786                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2787                         /* XXX handle errors */
2788                         if (r) {
2789                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2790                                           adev->ip_blocks[i].version->funcs->name, r);
2791                         }
2792                         adev->ip_blocks[i].status.hw = false;
2793                         break;
2794                 }
2795         }
2796 }
2797
2798 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2799 {
2800         int i, r;
2801
2802         for (i = 0; i < adev->num_ip_blocks; i++) {
2803                 if (!adev->ip_blocks[i].version->funcs->early_fini)
2804                         continue;
2805
2806                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2807                 if (r) {
2808                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2809                                   adev->ip_blocks[i].version->funcs->name, r);
2810                 }
2811         }
2812
2813         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2814         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2815
2816         amdgpu_amdkfd_suspend(adev, false);
2817
2818         /* Workaroud for ASICs need to disable SMC first */
2819         amdgpu_device_smu_fini_early(adev);
2820
2821         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2822                 if (!adev->ip_blocks[i].status.hw)
2823                         continue;
2824
2825                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2826                 /* XXX handle errors */
2827                 if (r) {
2828                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2829                                   adev->ip_blocks[i].version->funcs->name, r);
2830                 }
2831
2832                 adev->ip_blocks[i].status.hw = false;
2833         }
2834
2835         if (amdgpu_sriov_vf(adev)) {
2836                 if (amdgpu_virt_release_full_gpu(adev, false))
2837                         DRM_ERROR("failed to release exclusive mode on fini\n");
2838         }
2839
2840         return 0;
2841 }
2842
2843 /**
2844  * amdgpu_device_ip_fini - run fini for hardware IPs
2845  *
2846  * @adev: amdgpu_device pointer
2847  *
2848  * Main teardown pass for hardware IPs.  The list of all the hardware
2849  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2850  * are run.  hw_fini tears down the hardware associated with each IP
2851  * and sw_fini tears down any software state associated with each IP.
2852  * Returns 0 on success, negative error code on failure.
2853  */
2854 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2855 {
2856         int i, r;
2857
2858         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2859                 amdgpu_virt_release_ras_err_handler_data(adev);
2860
2861         if (adev->gmc.xgmi.num_physical_nodes > 1)
2862                 amdgpu_xgmi_remove_device(adev);
2863
2864         amdgpu_amdkfd_device_fini_sw(adev);
2865
2866         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2867                 if (!adev->ip_blocks[i].status.sw)
2868                         continue;
2869
2870                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2871                         amdgpu_ucode_free_bo(adev);
2872                         amdgpu_free_static_csa(&adev->virt.csa_obj);
2873                         amdgpu_device_wb_fini(adev);
2874                         amdgpu_device_vram_scratch_fini(adev);
2875                         amdgpu_ib_pool_fini(adev);
2876                 }
2877
2878                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2879                 /* XXX handle errors */
2880                 if (r) {
2881                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2882                                   adev->ip_blocks[i].version->funcs->name, r);
2883                 }
2884                 adev->ip_blocks[i].status.sw = false;
2885                 adev->ip_blocks[i].status.valid = false;
2886         }
2887
2888         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2889                 if (!adev->ip_blocks[i].status.late_initialized)
2890                         continue;
2891                 if (adev->ip_blocks[i].version->funcs->late_fini)
2892                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2893                 adev->ip_blocks[i].status.late_initialized = false;
2894         }
2895
2896         amdgpu_ras_fini(adev);
2897
2898         return 0;
2899 }
2900
2901 /**
2902  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2903  *
2904  * @work: work_struct.
2905  */
2906 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2907 {
2908         struct amdgpu_device *adev =
2909                 container_of(work, struct amdgpu_device, delayed_init_work.work);
2910         int r;
2911
2912         r = amdgpu_ib_ring_tests(adev);
2913         if (r)
2914                 DRM_ERROR("ib ring test failed (%d).\n", r);
2915 }
2916
2917 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2918 {
2919         struct amdgpu_device *adev =
2920                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2921
2922         WARN_ON_ONCE(adev->gfx.gfx_off_state);
2923         WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
2924
2925         if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2926                 adev->gfx.gfx_off_state = true;
2927 }
2928
2929 /**
2930  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2931  *
2932  * @adev: amdgpu_device pointer
2933  *
2934  * Main suspend function for hardware IPs.  The list of all the hardware
2935  * IPs that make up the asic is walked, clockgating is disabled and the
2936  * suspend callbacks are run.  suspend puts the hardware and software state
2937  * in each IP into a state suitable for suspend.
2938  * Returns 0 on success, negative error code on failure.
2939  */
2940 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2941 {
2942         int i, r;
2943
2944         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2945         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2946
2947         /*
2948          * Per PMFW team's suggestion, driver needs to handle gfxoff
2949          * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
2950          * scenario. Add the missing df cstate disablement here.
2951          */
2952         if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
2953                 dev_warn(adev->dev, "Failed to disallow df cstate");
2954
2955         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2956                 if (!adev->ip_blocks[i].status.valid)
2957                         continue;
2958
2959                 /* displays are handled separately */
2960                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2961                         continue;
2962
2963                 /* XXX handle errors */
2964                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2965                 /* XXX handle errors */
2966                 if (r) {
2967                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2968                                   adev->ip_blocks[i].version->funcs->name, r);
2969                         return r;
2970                 }
2971
2972                 adev->ip_blocks[i].status.hw = false;
2973         }
2974
2975         return 0;
2976 }
2977
2978 /**
2979  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2980  *
2981  * @adev: amdgpu_device pointer
2982  *
2983  * Main suspend function for hardware IPs.  The list of all the hardware
2984  * IPs that make up the asic is walked, clockgating is disabled and the
2985  * suspend callbacks are run.  suspend puts the hardware and software state
2986  * in each IP into a state suitable for suspend.
2987  * Returns 0 on success, negative error code on failure.
2988  */
2989 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2990 {
2991         int i, r;
2992
2993         if (adev->in_s0ix)
2994                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
2995
2996         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2997                 if (!adev->ip_blocks[i].status.valid)
2998                         continue;
2999                 /* displays are handled in phase1 */
3000                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3001                         continue;
3002                 /* PSP lost connection when err_event_athub occurs */
3003                 if (amdgpu_ras_intr_triggered() &&
3004                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3005                         adev->ip_blocks[i].status.hw = false;
3006                         continue;
3007                 }
3008
3009                 /* skip unnecessary suspend if we do not initialize them yet */
3010                 if (adev->gmc.xgmi.pending_reset &&
3011                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3012                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3013                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3014                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3015                         adev->ip_blocks[i].status.hw = false;
3016                         continue;
3017                 }
3018
3019                 /* skip suspend of gfx/mes and psp for S0ix
3020                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
3021                  * like at runtime. PSP is also part of the always on hardware
3022                  * so no need to suspend it.
3023                  */
3024                 if (adev->in_s0ix &&
3025                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3026                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3027                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3028                         continue;
3029
3030                 /* XXX handle errors */
3031                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3032                 /* XXX handle errors */
3033                 if (r) {
3034                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3035                                   adev->ip_blocks[i].version->funcs->name, r);
3036                 }
3037                 adev->ip_blocks[i].status.hw = false;
3038                 /* handle putting the SMC in the appropriate state */
3039                 if(!amdgpu_sriov_vf(adev)){
3040                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3041                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3042                                 if (r) {
3043                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3044                                                         adev->mp1_state, r);
3045                                         return r;
3046                                 }
3047                         }
3048                 }
3049         }
3050
3051         return 0;
3052 }
3053
3054 /**
3055  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3056  *
3057  * @adev: amdgpu_device pointer
3058  *
3059  * Main suspend function for hardware IPs.  The list of all the hardware
3060  * IPs that make up the asic is walked, clockgating is disabled and the
3061  * suspend callbacks are run.  suspend puts the hardware and software state
3062  * in each IP into a state suitable for suspend.
3063  * Returns 0 on success, negative error code on failure.
3064  */
3065 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3066 {
3067         int r;
3068
3069         if (amdgpu_sriov_vf(adev)) {
3070                 amdgpu_virt_fini_data_exchange(adev);
3071                 amdgpu_virt_request_full_gpu(adev, false);
3072         }
3073
3074         r = amdgpu_device_ip_suspend_phase1(adev);
3075         if (r)
3076                 return r;
3077         r = amdgpu_device_ip_suspend_phase2(adev);
3078
3079         if (amdgpu_sriov_vf(adev))
3080                 amdgpu_virt_release_full_gpu(adev, false);
3081
3082         return r;
3083 }
3084
3085 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3086 {
3087         int i, r;
3088
3089         static enum amd_ip_block_type ip_order[] = {
3090                 AMD_IP_BLOCK_TYPE_COMMON,
3091                 AMD_IP_BLOCK_TYPE_GMC,
3092                 AMD_IP_BLOCK_TYPE_PSP,
3093                 AMD_IP_BLOCK_TYPE_IH,
3094         };
3095
3096         for (i = 0; i < adev->num_ip_blocks; i++) {
3097                 int j;
3098                 struct amdgpu_ip_block *block;
3099
3100                 block = &adev->ip_blocks[i];
3101                 block->status.hw = false;
3102
3103                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3104
3105                         if (block->version->type != ip_order[j] ||
3106                                 !block->status.valid)
3107                                 continue;
3108
3109                         r = block->version->funcs->hw_init(adev);
3110                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3111                         if (r)
3112                                 return r;
3113                         block->status.hw = true;
3114                 }
3115         }
3116
3117         return 0;
3118 }
3119
3120 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3121 {
3122         int i, r;
3123
3124         static enum amd_ip_block_type ip_order[] = {
3125                 AMD_IP_BLOCK_TYPE_SMC,
3126                 AMD_IP_BLOCK_TYPE_DCE,
3127                 AMD_IP_BLOCK_TYPE_GFX,
3128                 AMD_IP_BLOCK_TYPE_SDMA,
3129                 AMD_IP_BLOCK_TYPE_UVD,
3130                 AMD_IP_BLOCK_TYPE_VCE,
3131                 AMD_IP_BLOCK_TYPE_VCN
3132         };
3133
3134         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3135                 int j;
3136                 struct amdgpu_ip_block *block;
3137
3138                 for (j = 0; j < adev->num_ip_blocks; j++) {
3139                         block = &adev->ip_blocks[j];
3140
3141                         if (block->version->type != ip_order[i] ||
3142                                 !block->status.valid ||
3143                                 block->status.hw)
3144                                 continue;
3145
3146                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3147                                 r = block->version->funcs->resume(adev);
3148                         else
3149                                 r = block->version->funcs->hw_init(adev);
3150
3151                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3152                         if (r)
3153                                 return r;
3154                         block->status.hw = true;
3155                 }
3156         }
3157
3158         return 0;
3159 }
3160
3161 /**
3162  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3163  *
3164  * @adev: amdgpu_device pointer
3165  *
3166  * First resume function for hardware IPs.  The list of all the hardware
3167  * IPs that make up the asic is walked and the resume callbacks are run for
3168  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3169  * after a suspend and updates the software state as necessary.  This
3170  * function is also used for restoring the GPU after a GPU reset.
3171  * Returns 0 on success, negative error code on failure.
3172  */
3173 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3174 {
3175         int i, r;
3176
3177         for (i = 0; i < adev->num_ip_blocks; i++) {
3178                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3179                         continue;
3180                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3181                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3182                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3183                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3184
3185                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3186                         if (r) {
3187                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3188                                           adev->ip_blocks[i].version->funcs->name, r);
3189                                 return r;
3190                         }
3191                         adev->ip_blocks[i].status.hw = true;
3192                 }
3193         }
3194
3195         return 0;
3196 }
3197
3198 /**
3199  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3200  *
3201  * @adev: amdgpu_device pointer
3202  *
3203  * First resume function for hardware IPs.  The list of all the hardware
3204  * IPs that make up the asic is walked and the resume callbacks are run for
3205  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3206  * functional state after a suspend and updates the software state as
3207  * necessary.  This function is also used for restoring the GPU after a GPU
3208  * reset.
3209  * Returns 0 on success, negative error code on failure.
3210  */
3211 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3212 {
3213         int i, r;
3214
3215         for (i = 0; i < adev->num_ip_blocks; i++) {
3216                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3217                         continue;
3218                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3219                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3220                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3221                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3222                         continue;
3223                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3224                 if (r) {
3225                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3226                                   adev->ip_blocks[i].version->funcs->name, r);
3227                         return r;
3228                 }
3229                 adev->ip_blocks[i].status.hw = true;
3230
3231                 if (adev->in_s0ix && adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3232                         /* disable gfxoff for IP resume. The gfxoff will be re-enabled in
3233                          * amdgpu_device_resume() after IP resume.
3234                          */
3235                         amdgpu_gfx_off_ctrl(adev, false);
3236                         DRM_DEBUG("will disable gfxoff for re-initializing other blocks\n");
3237                 }
3238
3239         }
3240
3241         return 0;
3242 }
3243
3244 /**
3245  * amdgpu_device_ip_resume - run resume for hardware IPs
3246  *
3247  * @adev: amdgpu_device pointer
3248  *
3249  * Main resume function for hardware IPs.  The hardware IPs
3250  * are split into two resume functions because they are
3251  * are also used in in recovering from a GPU reset and some additional
3252  * steps need to be take between them.  In this case (S3/S4) they are
3253  * run sequentially.
3254  * Returns 0 on success, negative error code on failure.
3255  */
3256 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3257 {
3258         int r;
3259
3260         r = amdgpu_amdkfd_resume_iommu(adev);
3261         if (r)
3262                 return r;
3263
3264         r = amdgpu_device_ip_resume_phase1(adev);
3265         if (r)
3266                 return r;
3267
3268         r = amdgpu_device_fw_loading(adev);
3269         if (r)
3270                 return r;
3271
3272         r = amdgpu_device_ip_resume_phase2(adev);
3273
3274         return r;
3275 }
3276
3277 /**
3278  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3279  *
3280  * @adev: amdgpu_device pointer
3281  *
3282  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3283  */
3284 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3285 {
3286         if (amdgpu_sriov_vf(adev)) {
3287                 if (adev->is_atom_fw) {
3288                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3289                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3290                 } else {
3291                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3292                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3293                 }
3294
3295                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3296                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3297         }
3298 }
3299
3300 /**
3301  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3302  *
3303  * @asic_type: AMD asic type
3304  *
3305  * Check if there is DC (new modesetting infrastructre) support for an asic.
3306  * returns true if DC has support, false if not.
3307  */
3308 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3309 {
3310         switch (asic_type) {
3311 #ifdef CONFIG_DRM_AMDGPU_SI
3312         case CHIP_HAINAN:
3313 #endif
3314         case CHIP_TOPAZ:
3315                 /* chips with no display hardware */
3316                 return false;
3317 #if defined(CONFIG_DRM_AMD_DC)
3318         case CHIP_TAHITI:
3319         case CHIP_PITCAIRN:
3320         case CHIP_VERDE:
3321         case CHIP_OLAND:
3322                 /*
3323                  * We have systems in the wild with these ASICs that require
3324                  * LVDS and VGA support which is not supported with DC.
3325                  *
3326                  * Fallback to the non-DC driver here by default so as not to
3327                  * cause regressions.
3328                  */
3329 #if defined(CONFIG_DRM_AMD_DC_SI)
3330                 return amdgpu_dc > 0;
3331 #else
3332                 return false;
3333 #endif
3334         case CHIP_BONAIRE:
3335         case CHIP_KAVERI:
3336         case CHIP_KABINI:
3337         case CHIP_MULLINS:
3338                 /*
3339                  * We have systems in the wild with these ASICs that require
3340                  * VGA support which is not supported with DC.
3341                  *
3342                  * Fallback to the non-DC driver here by default so as not to
3343                  * cause regressions.
3344                  */
3345                 return amdgpu_dc > 0;
3346         default:
3347                 return amdgpu_dc != 0;
3348 #else
3349         default:
3350                 if (amdgpu_dc > 0)
3351                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3352                                          "but isn't supported by ASIC, ignoring\n");
3353                 return false;
3354 #endif
3355         }
3356 }
3357
3358 /**
3359  * amdgpu_device_has_dc_support - check if dc is supported
3360  *
3361  * @adev: amdgpu_device pointer
3362  *
3363  * Returns true for supported, false for not supported
3364  */
3365 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3366 {
3367         if (adev->enable_virtual_display ||
3368             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3369                 return false;
3370
3371         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3372 }
3373
3374 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3375 {
3376         struct amdgpu_device *adev =
3377                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3378         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3379
3380         /* It's a bug to not have a hive within this function */
3381         if (WARN_ON(!hive))
3382                 return;
3383
3384         /*
3385          * Use task barrier to synchronize all xgmi reset works across the
3386          * hive. task_barrier_enter and task_barrier_exit will block
3387          * until all the threads running the xgmi reset works reach
3388          * those points. task_barrier_full will do both blocks.
3389          */
3390         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3391
3392                 task_barrier_enter(&hive->tb);
3393                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3394
3395                 if (adev->asic_reset_res)
3396                         goto fail;
3397
3398                 task_barrier_exit(&hive->tb);
3399                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3400
3401                 if (adev->asic_reset_res)
3402                         goto fail;
3403
3404                 if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops &&
3405                     adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
3406                         adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev);
3407         } else {
3408
3409                 task_barrier_full(&hive->tb);
3410                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3411         }
3412
3413 fail:
3414         if (adev->asic_reset_res)
3415                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3416                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3417         amdgpu_put_xgmi_hive(hive);
3418 }
3419
3420 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3421 {
3422         char *input = amdgpu_lockup_timeout;
3423         char *timeout_setting = NULL;
3424         int index = 0;
3425         long timeout;
3426         int ret = 0;
3427
3428         /*
3429          * By default timeout for non compute jobs is 10000
3430          * and 60000 for compute jobs.
3431          * In SR-IOV or passthrough mode, timeout for compute
3432          * jobs are 60000 by default.
3433          */
3434         adev->gfx_timeout = msecs_to_jiffies(10000);
3435         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3436         if (amdgpu_sriov_vf(adev))
3437                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3438                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3439         else
3440                 adev->compute_timeout =  msecs_to_jiffies(60000);
3441
3442         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3443                 while ((timeout_setting = strsep(&input, ",")) &&
3444                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3445                         ret = kstrtol(timeout_setting, 0, &timeout);
3446                         if (ret)
3447                                 return ret;
3448
3449                         if (timeout == 0) {
3450                                 index++;
3451                                 continue;
3452                         } else if (timeout < 0) {
3453                                 timeout = MAX_SCHEDULE_TIMEOUT;
3454                                 dev_warn(adev->dev, "lockup timeout disabled");
3455                                 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3456                         } else {
3457                                 timeout = msecs_to_jiffies(timeout);
3458                         }
3459
3460                         switch (index++) {
3461                         case 0:
3462                                 adev->gfx_timeout = timeout;
3463                                 break;
3464                         case 1:
3465                                 adev->compute_timeout = timeout;
3466                                 break;
3467                         case 2:
3468                                 adev->sdma_timeout = timeout;
3469                                 break;
3470                         case 3:
3471                                 adev->video_timeout = timeout;
3472                                 break;
3473                         default:
3474                                 break;
3475                         }
3476                 }
3477                 /*
3478                  * There is only one value specified and
3479                  * it should apply to all non-compute jobs.
3480                  */
3481                 if (index == 1) {
3482                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3483                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3484                                 adev->compute_timeout = adev->gfx_timeout;
3485                 }
3486         }
3487
3488         return ret;
3489 }
3490
3491 /**
3492  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3493  *
3494  * @adev: amdgpu_device pointer
3495  *
3496  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3497  */
3498 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3499 {
3500         struct iommu_domain *domain;
3501
3502         domain = iommu_get_domain_for_dev(adev->dev);
3503         if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3504                 adev->ram_is_direct_mapped = true;
3505 }
3506
3507 static const struct attribute *amdgpu_dev_attributes[] = {
3508         &dev_attr_product_name.attr,
3509         &dev_attr_product_number.attr,
3510         &dev_attr_serial_number.attr,
3511         &dev_attr_pcie_replay_count.attr,
3512         NULL
3513 };
3514
3515 /**
3516  * amdgpu_device_init - initialize the driver
3517  *
3518  * @adev: amdgpu_device pointer
3519  * @flags: driver flags
3520  *
3521  * Initializes the driver info and hw (all asics).
3522  * Returns 0 for success or an error on failure.
3523  * Called at driver startup.
3524  */
3525 int amdgpu_device_init(struct amdgpu_device *adev,
3526                        uint32_t flags)
3527 {
3528         struct drm_device *ddev = adev_to_drm(adev);
3529         struct pci_dev *pdev = adev->pdev;
3530         int r, i;
3531         bool px = false;
3532         u32 max_MBps;
3533
3534         adev->shutdown = false;
3535         adev->flags = flags;
3536
3537         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3538                 adev->asic_type = amdgpu_force_asic_type;
3539         else
3540                 adev->asic_type = flags & AMD_ASIC_MASK;
3541
3542         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3543         if (amdgpu_emu_mode == 1)
3544                 adev->usec_timeout *= 10;
3545         adev->gmc.gart_size = 512 * 1024 * 1024;
3546         adev->accel_working = false;
3547         adev->num_rings = 0;
3548         RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3549         adev->mman.buffer_funcs = NULL;
3550         adev->mman.buffer_funcs_ring = NULL;
3551         adev->vm_manager.vm_pte_funcs = NULL;
3552         adev->vm_manager.vm_pte_num_scheds = 0;
3553         adev->gmc.gmc_funcs = NULL;
3554         adev->harvest_ip_mask = 0x0;
3555         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3556         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3557
3558         adev->smc_rreg = &amdgpu_invalid_rreg;
3559         adev->smc_wreg = &amdgpu_invalid_wreg;
3560         adev->pcie_rreg = &amdgpu_invalid_rreg;
3561         adev->pcie_wreg = &amdgpu_invalid_wreg;
3562         adev->pciep_rreg = &amdgpu_invalid_rreg;
3563         adev->pciep_wreg = &amdgpu_invalid_wreg;
3564         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3565         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3566         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3567         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3568         adev->didt_rreg = &amdgpu_invalid_rreg;
3569         adev->didt_wreg = &amdgpu_invalid_wreg;
3570         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3571         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3572         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3573         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3574
3575         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3576                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3577                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3578
3579         /* mutex initialization are all done here so we
3580          * can recall function without having locking issues */
3581         mutex_init(&adev->firmware.mutex);
3582         mutex_init(&adev->pm.mutex);
3583         mutex_init(&adev->gfx.gpu_clock_mutex);
3584         mutex_init(&adev->srbm_mutex);
3585         mutex_init(&adev->gfx.pipe_reserve_mutex);
3586         mutex_init(&adev->gfx.gfx_off_mutex);
3587         mutex_init(&adev->grbm_idx_mutex);
3588         mutex_init(&adev->mn_lock);
3589         mutex_init(&adev->virt.vf_errors.lock);
3590         hash_init(adev->mn_hash);
3591         mutex_init(&adev->psp.mutex);
3592         mutex_init(&adev->notifier_lock);
3593         mutex_init(&adev->pm.stable_pstate_ctx_lock);
3594         mutex_init(&adev->benchmark_mutex);
3595
3596         amdgpu_device_init_apu_flags(adev);
3597
3598         r = amdgpu_device_check_arguments(adev);
3599         if (r)
3600                 return r;
3601
3602         spin_lock_init(&adev->mmio_idx_lock);
3603         spin_lock_init(&adev->smc_idx_lock);
3604         spin_lock_init(&adev->pcie_idx_lock);
3605         spin_lock_init(&adev->uvd_ctx_idx_lock);
3606         spin_lock_init(&adev->didt_idx_lock);
3607         spin_lock_init(&adev->gc_cac_idx_lock);
3608         spin_lock_init(&adev->se_cac_idx_lock);
3609         spin_lock_init(&adev->audio_endpt_idx_lock);
3610         spin_lock_init(&adev->mm_stats.lock);
3611
3612         INIT_LIST_HEAD(&adev->shadow_list);
3613         mutex_init(&adev->shadow_list_lock);
3614
3615         INIT_LIST_HEAD(&adev->reset_list);
3616
3617         INIT_LIST_HEAD(&adev->ras_list);
3618
3619         INIT_DELAYED_WORK(&adev->delayed_init_work,
3620                           amdgpu_device_delayed_init_work_handler);
3621         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3622                           amdgpu_device_delay_enable_gfx_off);
3623
3624         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3625
3626         adev->gfx.gfx_off_req_count = 1;
3627         adev->gfx.gfx_off_residency = 0;
3628         adev->gfx.gfx_off_entrycount = 0;
3629         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3630
3631         atomic_set(&adev->throttling_logging_enabled, 1);
3632         /*
3633          * If throttling continues, logging will be performed every minute
3634          * to avoid log flooding. "-1" is subtracted since the thermal
3635          * throttling interrupt comes every second. Thus, the total logging
3636          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3637          * for throttling interrupt) = 60 seconds.
3638          */
3639         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3640         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3641
3642         /* Registers mapping */
3643         /* TODO: block userspace mapping of io register */
3644         if (adev->asic_type >= CHIP_BONAIRE) {
3645                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3646                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3647         } else {
3648                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3649                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3650         }
3651
3652         for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3653                 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3654
3655         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3656         if (adev->rmmio == NULL) {
3657                 return -ENOMEM;
3658         }
3659         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3660         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3661
3662         amdgpu_device_get_pcie_info(adev);
3663
3664         if (amdgpu_mcbp)
3665                 DRM_INFO("MCBP is enabled\n");
3666
3667         /*
3668          * Reset domain needs to be present early, before XGMI hive discovered
3669          * (if any) and intitialized to use reset sem and in_gpu reset flag
3670          * early on during init and before calling to RREG32.
3671          */
3672         adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3673         if (!adev->reset_domain)
3674                 return -ENOMEM;
3675
3676         /* detect hw virtualization here */
3677         amdgpu_detect_virtualization(adev);
3678
3679         r = amdgpu_device_get_job_timeout_settings(adev);
3680         if (r) {
3681                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3682                 return r;
3683         }
3684
3685         /* early init functions */
3686         r = amdgpu_device_ip_early_init(adev);
3687         if (r)
3688                 return r;
3689
3690         /* Enable TMZ based on IP_VERSION */
3691         amdgpu_gmc_tmz_set(adev);
3692
3693         amdgpu_gmc_noretry_set(adev);
3694         /* Need to get xgmi info early to decide the reset behavior*/
3695         if (adev->gmc.xgmi.supported) {
3696                 r = adev->gfxhub.funcs->get_xgmi_info(adev);
3697                 if (r)
3698                         return r;
3699         }
3700
3701         /* enable PCIE atomic ops */
3702         if (amdgpu_sriov_vf(adev))
3703                 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3704                         adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
3705                         (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3706         else
3707                 adev->have_atomics_support =
3708                         !pci_enable_atomic_ops_to_root(adev->pdev,
3709                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3710                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3711         if (!adev->have_atomics_support)
3712                 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3713
3714         /* doorbell bar mapping and doorbell index init*/
3715         amdgpu_device_doorbell_init(adev);
3716
3717         if (amdgpu_emu_mode == 1) {
3718                 /* post the asic on emulation mode */
3719                 emu_soc_asic_init(adev);
3720                 goto fence_driver_init;
3721         }
3722
3723         amdgpu_reset_init(adev);
3724
3725         /* detect if we are with an SRIOV vbios */
3726         amdgpu_device_detect_sriov_bios(adev);
3727
3728         /* check if we need to reset the asic
3729          *  E.g., driver was not cleanly unloaded previously, etc.
3730          */
3731         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3732                 if (adev->gmc.xgmi.num_physical_nodes) {
3733                         dev_info(adev->dev, "Pending hive reset.\n");
3734                         adev->gmc.xgmi.pending_reset = true;
3735                         /* Only need to init necessary block for SMU to handle the reset */
3736                         for (i = 0; i < adev->num_ip_blocks; i++) {
3737                                 if (!adev->ip_blocks[i].status.valid)
3738                                         continue;
3739                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3740                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3741                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3742                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3743                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
3744                                                 adev->ip_blocks[i].version->funcs->name);
3745                                         adev->ip_blocks[i].status.hw = true;
3746                                 }
3747                         }
3748                 } else {
3749                         r = amdgpu_asic_reset(adev);
3750                         if (r) {
3751                                 dev_err(adev->dev, "asic reset on init failed\n");
3752                                 goto failed;
3753                         }
3754                 }
3755         }
3756
3757         pci_enable_pcie_error_reporting(adev->pdev);
3758
3759         /* Post card if necessary */
3760         if (amdgpu_device_need_post(adev)) {
3761                 if (!adev->bios) {
3762                         dev_err(adev->dev, "no vBIOS found\n");
3763                         r = -EINVAL;
3764                         goto failed;
3765                 }
3766                 DRM_INFO("GPU posting now...\n");
3767                 r = amdgpu_device_asic_init(adev);
3768                 if (r) {
3769                         dev_err(adev->dev, "gpu post error!\n");
3770                         goto failed;
3771                 }
3772         }
3773
3774         if (adev->is_atom_fw) {
3775                 /* Initialize clocks */
3776                 r = amdgpu_atomfirmware_get_clock_info(adev);
3777                 if (r) {
3778                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3779                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3780                         goto failed;
3781                 }
3782         } else {
3783                 /* Initialize clocks */
3784                 r = amdgpu_atombios_get_clock_info(adev);
3785                 if (r) {
3786                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3787                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3788                         goto failed;
3789                 }
3790                 /* init i2c buses */
3791                 if (!amdgpu_device_has_dc_support(adev))
3792                         amdgpu_atombios_i2c_init(adev);
3793         }
3794
3795 fence_driver_init:
3796         /* Fence driver */
3797         r = amdgpu_fence_driver_sw_init(adev);
3798         if (r) {
3799                 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
3800                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3801                 goto failed;
3802         }
3803
3804         /* init the mode config */
3805         drm_mode_config_init(adev_to_drm(adev));
3806
3807         r = amdgpu_device_ip_init(adev);
3808         if (r) {
3809                 /* failed in exclusive mode due to timeout */
3810                 if (amdgpu_sriov_vf(adev) &&
3811                     !amdgpu_sriov_runtime(adev) &&
3812                     amdgpu_virt_mmio_blocked(adev) &&
3813                     !amdgpu_virt_wait_reset(adev)) {
3814                         dev_err(adev->dev, "VF exclusive mode timeout\n");
3815                         /* Don't send request since VF is inactive. */
3816                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3817                         adev->virt.ops = NULL;
3818                         r = -EAGAIN;
3819                         goto release_ras_con;
3820                 }
3821                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3822                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3823                 goto release_ras_con;
3824         }
3825
3826         amdgpu_fence_driver_hw_init(adev);
3827
3828         dev_info(adev->dev,
3829                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3830                         adev->gfx.config.max_shader_engines,
3831                         adev->gfx.config.max_sh_per_se,
3832                         adev->gfx.config.max_cu_per_sh,
3833                         adev->gfx.cu_info.number);
3834
3835         adev->accel_working = true;
3836
3837         amdgpu_vm_check_compute_bug(adev);
3838
3839         /* Initialize the buffer migration limit. */
3840         if (amdgpu_moverate >= 0)
3841                 max_MBps = amdgpu_moverate;
3842         else
3843                 max_MBps = 8; /* Allow 8 MB/s. */
3844         /* Get a log2 for easy divisions. */
3845         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3846
3847         r = amdgpu_pm_sysfs_init(adev);
3848         if (r) {
3849                 adev->pm_sysfs_en = false;
3850                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3851         } else
3852                 adev->pm_sysfs_en = true;
3853
3854         r = amdgpu_ucode_sysfs_init(adev);
3855         if (r) {
3856                 adev->ucode_sysfs_en = false;
3857                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3858         } else
3859                 adev->ucode_sysfs_en = true;
3860
3861         r = amdgpu_psp_sysfs_init(adev);
3862         if (r) {
3863                 adev->psp_sysfs_en = false;
3864                 if (!amdgpu_sriov_vf(adev))
3865                         DRM_ERROR("Creating psp sysfs failed\n");
3866         } else
3867                 adev->psp_sysfs_en = true;
3868
3869         /*
3870          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3871          * Otherwise the mgpu fan boost feature will be skipped due to the
3872          * gpu instance is counted less.
3873          */
3874         amdgpu_register_gpu_instance(adev);
3875
3876         /* enable clockgating, etc. after ib tests, etc. since some blocks require
3877          * explicit gating rather than handling it automatically.
3878          */
3879         if (!adev->gmc.xgmi.pending_reset) {
3880                 r = amdgpu_device_ip_late_init(adev);
3881                 if (r) {
3882                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3883                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3884                         goto release_ras_con;
3885                 }
3886                 /* must succeed. */
3887                 amdgpu_ras_resume(adev);
3888                 queue_delayed_work(system_wq, &adev->delayed_init_work,
3889                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3890         }
3891
3892         if (amdgpu_sriov_vf(adev))
3893                 flush_delayed_work(&adev->delayed_init_work);
3894
3895         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3896         if (r)
3897                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3898
3899         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3900                 r = amdgpu_pmu_init(adev);
3901         if (r)
3902                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3903
3904         /* Have stored pci confspace at hand for restore in sudden PCI error */
3905         if (amdgpu_device_cache_pci_state(adev->pdev))
3906                 pci_restore_state(pdev);
3907
3908         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3909         /* this will fail for cards that aren't VGA class devices, just
3910          * ignore it */
3911         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3912                 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
3913
3914         if (amdgpu_device_supports_px(ddev)) {
3915                 px = true;
3916                 vga_switcheroo_register_client(adev->pdev,
3917                                                &amdgpu_switcheroo_ops, px);
3918                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3919         }
3920
3921         if (adev->gmc.xgmi.pending_reset)
3922                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3923                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3924
3925         amdgpu_device_check_iommu_direct_map(adev);
3926
3927         return 0;
3928
3929 release_ras_con:
3930         amdgpu_release_ras_context(adev);
3931
3932 failed:
3933         amdgpu_vf_error_trans_all(adev);
3934
3935         return r;
3936 }
3937
3938 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
3939 {
3940
3941         /* Clear all CPU mappings pointing to this device */
3942         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
3943
3944         /* Unmap all mapped bars - Doorbell, registers and VRAM */
3945         amdgpu_device_doorbell_fini(adev);
3946
3947         iounmap(adev->rmmio);
3948         adev->rmmio = NULL;
3949         if (adev->mman.aper_base_kaddr)
3950                 iounmap(adev->mman.aper_base_kaddr);
3951         adev->mman.aper_base_kaddr = NULL;
3952
3953         /* Memory manager related */
3954         if (!adev->gmc.xgmi.connected_to_cpu) {
3955                 arch_phys_wc_del(adev->gmc.vram_mtrr);
3956                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
3957         }
3958 }
3959
3960 /**
3961  * amdgpu_device_fini_hw - tear down the driver
3962  *
3963  * @adev: amdgpu_device pointer
3964  *
3965  * Tear down the driver info (all asics).
3966  * Called at driver shutdown.
3967  */
3968 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
3969 {
3970         dev_info(adev->dev, "amdgpu: finishing device.\n");
3971         flush_delayed_work(&adev->delayed_init_work);
3972         adev->shutdown = true;
3973
3974         /* make sure IB test finished before entering exclusive mode
3975          * to avoid preemption on IB test
3976          * */
3977         if (amdgpu_sriov_vf(adev)) {
3978                 amdgpu_virt_request_full_gpu(adev, false);
3979                 amdgpu_virt_fini_data_exchange(adev);
3980         }
3981
3982         /* disable all interrupts */
3983         amdgpu_irq_disable_all(adev);
3984         if (adev->mode_info.mode_config_initialized){
3985                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
3986                         drm_helper_force_disable_all(adev_to_drm(adev));
3987                 else
3988                         drm_atomic_helper_shutdown(adev_to_drm(adev));
3989         }
3990         amdgpu_fence_driver_hw_fini(adev);
3991
3992         if (adev->mman.initialized) {
3993                 flush_delayed_work(&adev->mman.bdev.wq);
3994                 ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3995         }
3996
3997         if (adev->pm_sysfs_en)
3998                 amdgpu_pm_sysfs_fini(adev);
3999         if (adev->ucode_sysfs_en)
4000                 amdgpu_ucode_sysfs_fini(adev);
4001         if (adev->psp_sysfs_en)
4002                 amdgpu_psp_sysfs_fini(adev);
4003         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4004
4005         /* disable ras feature must before hw fini */
4006         amdgpu_ras_pre_fini(adev);
4007
4008         amdgpu_device_ip_fini_early(adev);
4009
4010         amdgpu_irq_fini_hw(adev);
4011
4012         if (adev->mman.initialized)
4013                 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4014
4015         amdgpu_gart_dummy_page_fini(adev);
4016
4017         amdgpu_device_unmap_mmio(adev);
4018
4019 }
4020
4021 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4022 {
4023         int idx;
4024
4025         amdgpu_fence_driver_sw_fini(adev);
4026         amdgpu_device_ip_fini(adev);
4027         release_firmware(adev->firmware.gpu_info_fw);
4028         adev->firmware.gpu_info_fw = NULL;
4029         adev->accel_working = false;
4030         dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4031
4032         amdgpu_reset_fini(adev);
4033
4034         /* free i2c buses */
4035         if (!amdgpu_device_has_dc_support(adev))
4036                 amdgpu_i2c_fini(adev);
4037
4038         if (amdgpu_emu_mode != 1)
4039                 amdgpu_atombios_fini(adev);
4040
4041         kfree(adev->bios);
4042         adev->bios = NULL;
4043         if (amdgpu_device_supports_px(adev_to_drm(adev))) {
4044                 vga_switcheroo_unregister_client(adev->pdev);
4045                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4046         }
4047         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4048                 vga_client_unregister(adev->pdev);
4049
4050         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4051
4052                 iounmap(adev->rmmio);
4053                 adev->rmmio = NULL;
4054                 amdgpu_device_doorbell_fini(adev);
4055                 drm_dev_exit(idx);
4056         }
4057
4058         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4059                 amdgpu_pmu_fini(adev);
4060         if (adev->mman.discovery_bin)
4061                 amdgpu_discovery_fini(adev);
4062
4063         amdgpu_reset_put_reset_domain(adev->reset_domain);
4064         adev->reset_domain = NULL;
4065
4066         kfree(adev->pci_state);
4067
4068 }
4069
4070 /**
4071  * amdgpu_device_evict_resources - evict device resources
4072  * @adev: amdgpu device object
4073  *
4074  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4075  * of the vram memory type. Mainly used for evicting device resources
4076  * at suspend time.
4077  *
4078  */
4079 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4080 {
4081         int ret;
4082
4083         /* No need to evict vram on APUs for suspend to ram or s2idle */
4084         if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4085                 return 0;
4086
4087         ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4088         if (ret)
4089                 DRM_WARN("evicting device resources failed\n");
4090         return ret;
4091 }
4092
4093 /*
4094  * Suspend & resume.
4095  */
4096 /**
4097  * amdgpu_device_suspend - initiate device suspend
4098  *
4099  * @dev: drm dev pointer
4100  * @fbcon : notify the fbdev of suspend
4101  *
4102  * Puts the hw in the suspend state (all asics).
4103  * Returns 0 for success or an error on failure.
4104  * Called at driver suspend.
4105  */
4106 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4107 {
4108         struct amdgpu_device *adev = drm_to_adev(dev);
4109         int r = 0;
4110
4111         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4112                 return 0;
4113
4114         adev->in_suspend = true;
4115
4116         /* Evict the majority of BOs before grabbing the full access */
4117         r = amdgpu_device_evict_resources(adev);
4118         if (r)
4119                 return r;
4120
4121         if (amdgpu_sriov_vf(adev)) {
4122                 amdgpu_virt_fini_data_exchange(adev);
4123                 r = amdgpu_virt_request_full_gpu(adev, false);
4124                 if (r)
4125                         return r;
4126         }
4127
4128         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4129                 DRM_WARN("smart shift update failed\n");
4130
4131         drm_kms_helper_poll_disable(dev);
4132
4133         if (fbcon)
4134                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4135
4136         cancel_delayed_work_sync(&adev->delayed_init_work);
4137
4138         amdgpu_ras_suspend(adev);
4139
4140         amdgpu_device_ip_suspend_phase1(adev);
4141
4142         if (!adev->in_s0ix)
4143                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4144
4145         r = amdgpu_device_evict_resources(adev);
4146         if (r)
4147                 return r;
4148
4149         amdgpu_fence_driver_hw_fini(adev);
4150
4151         amdgpu_device_ip_suspend_phase2(adev);
4152
4153         if (amdgpu_sriov_vf(adev))
4154                 amdgpu_virt_release_full_gpu(adev, false);
4155
4156         return 0;
4157 }
4158
4159 /**
4160  * amdgpu_device_resume - initiate device resume
4161  *
4162  * @dev: drm dev pointer
4163  * @fbcon : notify the fbdev of resume
4164  *
4165  * Bring the hw back to operating state (all asics).
4166  * Returns 0 for success or an error on failure.
4167  * Called at driver resume.
4168  */
4169 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4170 {
4171         struct amdgpu_device *adev = drm_to_adev(dev);
4172         int r = 0;
4173
4174         if (amdgpu_sriov_vf(adev)) {
4175                 r = amdgpu_virt_request_full_gpu(adev, true);
4176                 if (r)
4177                         return r;
4178         }
4179
4180         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4181                 return 0;
4182
4183         if (adev->in_s0ix)
4184                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4185
4186         /* post card */
4187         if (amdgpu_device_need_post(adev)) {
4188                 r = amdgpu_device_asic_init(adev);
4189                 if (r)
4190                         dev_err(adev->dev, "amdgpu asic init failed\n");
4191         }
4192
4193         r = amdgpu_device_ip_resume(adev);
4194
4195         if (r) {
4196                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4197                 goto exit;
4198         }
4199         amdgpu_fence_driver_hw_init(adev);
4200
4201         r = amdgpu_device_ip_late_init(adev);
4202         if (r)
4203                 goto exit;
4204
4205         queue_delayed_work(system_wq, &adev->delayed_init_work,
4206                            msecs_to_jiffies(AMDGPU_RESUME_MS));
4207
4208         if (!adev->in_s0ix) {
4209                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4210                 if (r)
4211                         goto exit;
4212         }
4213
4214 exit:
4215         if (amdgpu_sriov_vf(adev)) {
4216                 amdgpu_virt_init_data_exchange(adev);
4217                 amdgpu_virt_release_full_gpu(adev, true);
4218         }
4219
4220         if (r)
4221                 return r;
4222
4223         /* Make sure IB tests flushed */
4224         flush_delayed_work(&adev->delayed_init_work);
4225
4226         if (adev->in_s0ix) {
4227                 /* re-enable gfxoff after IP resume. This re-enables gfxoff after
4228                  * it was disabled for IP resume in amdgpu_device_ip_resume_phase2().
4229                  */
4230                 amdgpu_gfx_off_ctrl(adev, true);
4231                 DRM_DEBUG("will enable gfxoff for the mission mode\n");
4232         }
4233         if (fbcon)
4234                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4235
4236         drm_kms_helper_poll_enable(dev);
4237
4238         amdgpu_ras_resume(adev);
4239
4240         if (adev->mode_info.num_crtc) {
4241                 /*
4242                  * Most of the connector probing functions try to acquire runtime pm
4243                  * refs to ensure that the GPU is powered on when connector polling is
4244                  * performed. Since we're calling this from a runtime PM callback,
4245                  * trying to acquire rpm refs will cause us to deadlock.
4246                  *
4247                  * Since we're guaranteed to be holding the rpm lock, it's safe to
4248                  * temporarily disable the rpm helpers so this doesn't deadlock us.
4249                  */
4250 #ifdef CONFIG_PM
4251                 dev->dev->power.disable_depth++;
4252 #endif
4253                 if (!adev->dc_enabled)
4254                         drm_helper_hpd_irq_event(dev);
4255                 else
4256                         drm_kms_helper_hotplug_event(dev);
4257 #ifdef CONFIG_PM
4258                 dev->dev->power.disable_depth--;
4259 #endif
4260         }
4261         adev->in_suspend = false;
4262
4263         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4264                 DRM_WARN("smart shift update failed\n");
4265
4266         return 0;
4267 }
4268
4269 /**
4270  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4271  *
4272  * @adev: amdgpu_device pointer
4273  *
4274  * The list of all the hardware IPs that make up the asic is walked and
4275  * the check_soft_reset callbacks are run.  check_soft_reset determines
4276  * if the asic is still hung or not.
4277  * Returns true if any of the IPs are still in a hung state, false if not.
4278  */
4279 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4280 {
4281         int i;
4282         bool asic_hang = false;
4283
4284         if (amdgpu_sriov_vf(adev))
4285                 return true;
4286
4287         if (amdgpu_asic_need_full_reset(adev))
4288                 return true;
4289
4290         for (i = 0; i < adev->num_ip_blocks; i++) {
4291                 if (!adev->ip_blocks[i].status.valid)
4292                         continue;
4293                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4294                         adev->ip_blocks[i].status.hang =
4295                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4296                 if (adev->ip_blocks[i].status.hang) {
4297                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4298                         asic_hang = true;
4299                 }
4300         }
4301         return asic_hang;
4302 }
4303
4304 /**
4305  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4306  *
4307  * @adev: amdgpu_device pointer
4308  *
4309  * The list of all the hardware IPs that make up the asic is walked and the
4310  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4311  * handles any IP specific hardware or software state changes that are
4312  * necessary for a soft reset to succeed.
4313  * Returns 0 on success, negative error code on failure.
4314  */
4315 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4316 {
4317         int i, r = 0;
4318
4319         for (i = 0; i < adev->num_ip_blocks; i++) {
4320                 if (!adev->ip_blocks[i].status.valid)
4321                         continue;
4322                 if (adev->ip_blocks[i].status.hang &&
4323                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4324                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4325                         if (r)
4326                                 return r;
4327                 }
4328         }
4329
4330         return 0;
4331 }
4332
4333 /**
4334  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4335  *
4336  * @adev: amdgpu_device pointer
4337  *
4338  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4339  * reset is necessary to recover.
4340  * Returns true if a full asic reset is required, false if not.
4341  */
4342 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4343 {
4344         int i;
4345
4346         if (amdgpu_asic_need_full_reset(adev))
4347                 return true;
4348
4349         for (i = 0; i < adev->num_ip_blocks; i++) {
4350                 if (!adev->ip_blocks[i].status.valid)
4351                         continue;
4352                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4353                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4354                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4355                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4356                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4357                         if (adev->ip_blocks[i].status.hang) {
4358                                 dev_info(adev->dev, "Some block need full reset!\n");
4359                                 return true;
4360                         }
4361                 }
4362         }
4363         return false;
4364 }
4365
4366 /**
4367  * amdgpu_device_ip_soft_reset - do a soft reset
4368  *
4369  * @adev: amdgpu_device pointer
4370  *
4371  * The list of all the hardware IPs that make up the asic is walked and the
4372  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4373  * IP specific hardware or software state changes that are necessary to soft
4374  * reset the IP.
4375  * Returns 0 on success, negative error code on failure.
4376  */
4377 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4378 {
4379         int i, r = 0;
4380
4381         for (i = 0; i < adev->num_ip_blocks; i++) {
4382                 if (!adev->ip_blocks[i].status.valid)
4383                         continue;
4384                 if (adev->ip_blocks[i].status.hang &&
4385                     adev->ip_blocks[i].version->funcs->soft_reset) {
4386                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4387                         if (r)
4388                                 return r;
4389                 }
4390         }
4391
4392         return 0;
4393 }
4394
4395 /**
4396  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4397  *
4398  * @adev: amdgpu_device pointer
4399  *
4400  * The list of all the hardware IPs that make up the asic is walked and the
4401  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4402  * handles any IP specific hardware or software state changes that are
4403  * necessary after the IP has been soft reset.
4404  * Returns 0 on success, negative error code on failure.
4405  */
4406 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4407 {
4408         int i, r = 0;
4409
4410         for (i = 0; i < adev->num_ip_blocks; i++) {
4411                 if (!adev->ip_blocks[i].status.valid)
4412                         continue;
4413                 if (adev->ip_blocks[i].status.hang &&
4414                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4415                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4416                 if (r)
4417                         return r;
4418         }
4419
4420         return 0;
4421 }
4422
4423 /**
4424  * amdgpu_device_recover_vram - Recover some VRAM contents
4425  *
4426  * @adev: amdgpu_device pointer
4427  *
4428  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4429  * restore things like GPUVM page tables after a GPU reset where
4430  * the contents of VRAM might be lost.
4431  *
4432  * Returns:
4433  * 0 on success, negative error code on failure.
4434  */
4435 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4436 {
4437         struct dma_fence *fence = NULL, *next = NULL;
4438         struct amdgpu_bo *shadow;
4439         struct amdgpu_bo_vm *vmbo;
4440         long r = 1, tmo;
4441
4442         if (amdgpu_sriov_runtime(adev))
4443                 tmo = msecs_to_jiffies(8000);
4444         else
4445                 tmo = msecs_to_jiffies(100);
4446
4447         dev_info(adev->dev, "recover vram bo from shadow start\n");
4448         mutex_lock(&adev->shadow_list_lock);
4449         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4450                 shadow = &vmbo->bo;
4451                 /* No need to recover an evicted BO */
4452                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4453                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4454                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4455                         continue;
4456
4457                 r = amdgpu_bo_restore_shadow(shadow, &next);
4458                 if (r)
4459                         break;
4460
4461                 if (fence) {
4462                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4463                         dma_fence_put(fence);
4464                         fence = next;
4465                         if (tmo == 0) {
4466                                 r = -ETIMEDOUT;
4467                                 break;
4468                         } else if (tmo < 0) {
4469                                 r = tmo;
4470                                 break;
4471                         }
4472                 } else {
4473                         fence = next;
4474                 }
4475         }
4476         mutex_unlock(&adev->shadow_list_lock);
4477
4478         if (fence)
4479                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4480         dma_fence_put(fence);
4481
4482         if (r < 0 || tmo <= 0) {
4483                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4484                 return -EIO;
4485         }
4486
4487         dev_info(adev->dev, "recover vram bo from shadow done\n");
4488         return 0;
4489 }
4490
4491
4492 /**
4493  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4494  *
4495  * @adev: amdgpu_device pointer
4496  * @from_hypervisor: request from hypervisor
4497  *
4498  * do VF FLR and reinitialize Asic
4499  * return 0 means succeeded otherwise failed
4500  */
4501 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4502                                      bool from_hypervisor)
4503 {
4504         int r;
4505         struct amdgpu_hive_info *hive = NULL;
4506         int retry_limit = 0;
4507
4508 retry:
4509         amdgpu_amdkfd_pre_reset(adev);
4510
4511         if (from_hypervisor)
4512                 r = amdgpu_virt_request_full_gpu(adev, true);
4513         else
4514                 r = amdgpu_virt_reset_gpu(adev);
4515         if (r)
4516                 return r;
4517
4518         /* Resume IP prior to SMC */
4519         r = amdgpu_device_ip_reinit_early_sriov(adev);
4520         if (r)
4521                 goto error;
4522
4523         amdgpu_virt_init_data_exchange(adev);
4524
4525         r = amdgpu_device_fw_loading(adev);
4526         if (r)
4527                 return r;
4528
4529         /* now we are okay to resume SMC/CP/SDMA */
4530         r = amdgpu_device_ip_reinit_late_sriov(adev);
4531         if (r)
4532                 goto error;
4533
4534         hive = amdgpu_get_xgmi_hive(adev);
4535         /* Update PSP FW topology after reset */
4536         if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4537                 r = amdgpu_xgmi_update_topology(hive, adev);
4538
4539         if (hive)
4540                 amdgpu_put_xgmi_hive(hive);
4541
4542         if (!r) {
4543                 amdgpu_irq_gpu_reset_resume_helper(adev);
4544                 r = amdgpu_ib_ring_tests(adev);
4545
4546                 amdgpu_amdkfd_post_reset(adev);
4547         }
4548
4549 error:
4550         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4551                 amdgpu_inc_vram_lost(adev);
4552                 r = amdgpu_device_recover_vram(adev);
4553         }
4554         amdgpu_virt_release_full_gpu(adev, true);
4555
4556         if (AMDGPU_RETRY_SRIOV_RESET(r)) {
4557                 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
4558                         retry_limit++;
4559                         goto retry;
4560                 } else
4561                         DRM_ERROR("GPU reset retry is beyond the retry limit\n");
4562         }
4563
4564         return r;
4565 }
4566
4567 /**
4568  * amdgpu_device_has_job_running - check if there is any job in mirror list
4569  *
4570  * @adev: amdgpu_device pointer
4571  *
4572  * check if there is any job in mirror list
4573  */
4574 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4575 {
4576         int i;
4577         struct drm_sched_job *job;
4578
4579         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4580                 struct amdgpu_ring *ring = adev->rings[i];
4581
4582                 if (!ring || !ring->sched.thread)
4583                         continue;
4584
4585                 spin_lock(&ring->sched.job_list_lock);
4586                 job = list_first_entry_or_null(&ring->sched.pending_list,
4587                                                struct drm_sched_job, list);
4588                 spin_unlock(&ring->sched.job_list_lock);
4589                 if (job)
4590                         return true;
4591         }
4592         return false;
4593 }
4594
4595 /**
4596  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4597  *
4598  * @adev: amdgpu_device pointer
4599  *
4600  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4601  * a hung GPU.
4602  */
4603 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4604 {
4605
4606         if (amdgpu_gpu_recovery == 0)
4607                 goto disabled;
4608
4609         /* Skip soft reset check in fatal error mode */
4610         if (!amdgpu_ras_is_poison_mode_supported(adev))
4611                 return true;
4612
4613         if (!amdgpu_device_ip_check_soft_reset(adev)) {
4614                 dev_info(adev->dev,"Timeout, but no hardware hang detected.\n");
4615                 return false;
4616         }
4617
4618         if (amdgpu_sriov_vf(adev))
4619                 return true;
4620
4621         if (amdgpu_gpu_recovery == -1) {
4622                 switch (adev->asic_type) {
4623 #ifdef CONFIG_DRM_AMDGPU_SI
4624                 case CHIP_VERDE:
4625                 case CHIP_TAHITI:
4626                 case CHIP_PITCAIRN:
4627                 case CHIP_OLAND:
4628                 case CHIP_HAINAN:
4629 #endif
4630 #ifdef CONFIG_DRM_AMDGPU_CIK
4631                 case CHIP_KAVERI:
4632                 case CHIP_KABINI:
4633                 case CHIP_MULLINS:
4634 #endif
4635                 case CHIP_CARRIZO:
4636                 case CHIP_STONEY:
4637                 case CHIP_CYAN_SKILLFISH:
4638                         goto disabled;
4639                 default:
4640                         break;
4641                 }
4642         }
4643
4644         return true;
4645
4646 disabled:
4647                 dev_info(adev->dev, "GPU recovery disabled.\n");
4648                 return false;
4649 }
4650
4651 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4652 {
4653         u32 i;
4654         int ret = 0;
4655
4656         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4657
4658         dev_info(adev->dev, "GPU mode1 reset\n");
4659
4660         /* disable BM */
4661         pci_clear_master(adev->pdev);
4662
4663         amdgpu_device_cache_pci_state(adev->pdev);
4664
4665         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4666                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4667                 ret = amdgpu_dpm_mode1_reset(adev);
4668         } else {
4669                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4670                 ret = psp_gpu_reset(adev);
4671         }
4672
4673         if (ret)
4674                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4675
4676         amdgpu_device_load_pci_state(adev->pdev);
4677
4678         /* wait for asic to come out of reset */
4679         for (i = 0; i < adev->usec_timeout; i++) {
4680                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4681
4682                 if (memsize != 0xffffffff)
4683                         break;
4684                 udelay(1);
4685         }
4686
4687         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4688         return ret;
4689 }
4690
4691 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4692                                  struct amdgpu_reset_context *reset_context)
4693 {
4694         int i, r = 0;
4695         struct amdgpu_job *job = NULL;
4696         bool need_full_reset =
4697                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4698
4699         if (reset_context->reset_req_dev == adev)
4700                 job = reset_context->job;
4701
4702         if (amdgpu_sriov_vf(adev)) {
4703                 /* stop the data exchange thread */
4704                 amdgpu_virt_fini_data_exchange(adev);
4705         }
4706
4707         amdgpu_fence_driver_isr_toggle(adev, true);
4708
4709         /* block all schedulers and reset given job's ring */
4710         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4711                 struct amdgpu_ring *ring = adev->rings[i];
4712
4713                 if (!ring || !ring->sched.thread)
4714                         continue;
4715
4716                 /*clear job fence from fence drv to avoid force_completion
4717                  *leave NULL and vm flush fence in fence drv */
4718                 amdgpu_fence_driver_clear_job_fences(ring);
4719
4720                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4721                 amdgpu_fence_driver_force_completion(ring);
4722         }
4723
4724         amdgpu_fence_driver_isr_toggle(adev, false);
4725
4726         if (job && job->vm)
4727                 drm_sched_increase_karma(&job->base);
4728
4729         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4730         /* If reset handler not implemented, continue; otherwise return */
4731         if (r == -ENOSYS)
4732                 r = 0;
4733         else
4734                 return r;
4735
4736         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4737         if (!amdgpu_sriov_vf(adev)) {
4738
4739                 if (!need_full_reset)
4740                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4741
4742                 if (!need_full_reset && amdgpu_gpu_recovery) {
4743                         amdgpu_device_ip_pre_soft_reset(adev);
4744                         r = amdgpu_device_ip_soft_reset(adev);
4745                         amdgpu_device_ip_post_soft_reset(adev);
4746                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4747                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4748                                 need_full_reset = true;
4749                         }
4750                 }
4751
4752                 if (need_full_reset)
4753                         r = amdgpu_device_ip_suspend(adev);
4754                 if (need_full_reset)
4755                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4756                 else
4757                         clear_bit(AMDGPU_NEED_FULL_RESET,
4758                                   &reset_context->flags);
4759         }
4760
4761         return r;
4762 }
4763
4764 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
4765 {
4766         int i;
4767
4768         lockdep_assert_held(&adev->reset_domain->sem);
4769
4770         for (i = 0; i < adev->num_regs; i++) {
4771                 adev->reset_dump_reg_value[i] = RREG32(adev->reset_dump_reg_list[i]);
4772                 trace_amdgpu_reset_reg_dumps(adev->reset_dump_reg_list[i],
4773                                              adev->reset_dump_reg_value[i]);
4774         }
4775
4776         return 0;
4777 }
4778
4779 #ifdef CONFIG_DEV_COREDUMP
4780 static ssize_t amdgpu_devcoredump_read(char *buffer, loff_t offset,
4781                 size_t count, void *data, size_t datalen)
4782 {
4783         struct drm_printer p;
4784         struct amdgpu_device *adev = data;
4785         struct drm_print_iterator iter;
4786         int i;
4787
4788         iter.data = buffer;
4789         iter.offset = 0;
4790         iter.start = offset;
4791         iter.remain = count;
4792
4793         p = drm_coredump_printer(&iter);
4794
4795         drm_printf(&p, "**** AMDGPU Device Coredump ****\n");
4796         drm_printf(&p, "kernel: " UTS_RELEASE "\n");
4797         drm_printf(&p, "module: " KBUILD_MODNAME "\n");
4798         drm_printf(&p, "time: %lld.%09ld\n", adev->reset_time.tv_sec, adev->reset_time.tv_nsec);
4799         if (adev->reset_task_info.pid)
4800                 drm_printf(&p, "process_name: %s PID: %d\n",
4801                            adev->reset_task_info.process_name,
4802                            adev->reset_task_info.pid);
4803
4804         if (adev->reset_vram_lost)
4805                 drm_printf(&p, "VRAM is lost due to GPU reset!\n");
4806         if (adev->num_regs) {
4807                 drm_printf(&p, "AMDGPU register dumps:\nOffset:     Value:\n");
4808
4809                 for (i = 0; i < adev->num_regs; i++)
4810                         drm_printf(&p, "0x%08x: 0x%08x\n",
4811                                    adev->reset_dump_reg_list[i],
4812                                    adev->reset_dump_reg_value[i]);
4813         }
4814
4815         return count - iter.remain;
4816 }
4817
4818 static void amdgpu_devcoredump_free(void *data)
4819 {
4820 }
4821
4822 static void amdgpu_reset_capture_coredumpm(struct amdgpu_device *adev)
4823 {
4824         struct drm_device *dev = adev_to_drm(adev);
4825
4826         ktime_get_ts64(&adev->reset_time);
4827         dev_coredumpm(dev->dev, THIS_MODULE, adev, 0, GFP_KERNEL,
4828                       amdgpu_devcoredump_read, amdgpu_devcoredump_free);
4829 }
4830 #endif
4831
4832 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4833                          struct amdgpu_reset_context *reset_context)
4834 {
4835         struct amdgpu_device *tmp_adev = NULL;
4836         bool need_full_reset, skip_hw_reset, vram_lost = false;
4837         int r = 0;
4838         bool gpu_reset_for_dev_remove = 0;
4839
4840         /* Try reset handler method first */
4841         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4842                                     reset_list);
4843         amdgpu_reset_reg_dumps(tmp_adev);
4844
4845         reset_context->reset_device_list = device_list_handle;
4846         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4847         /* If reset handler not implemented, continue; otherwise return */
4848         if (r == -ENOSYS)
4849                 r = 0;
4850         else
4851                 return r;
4852
4853         /* Reset handler not implemented, use the default method */
4854         need_full_reset =
4855                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4856         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4857
4858         gpu_reset_for_dev_remove =
4859                 test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
4860                         test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4861
4862         /*
4863          * ASIC reset has to be done on all XGMI hive nodes ASAP
4864          * to allow proper links negotiation in FW (within 1 sec)
4865          */
4866         if (!skip_hw_reset && need_full_reset) {
4867                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4868                         /* For XGMI run all resets in parallel to speed up the process */
4869                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4870                                 tmp_adev->gmc.xgmi.pending_reset = false;
4871                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4872                                         r = -EALREADY;
4873                         } else
4874                                 r = amdgpu_asic_reset(tmp_adev);
4875
4876                         if (r) {
4877                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4878                                          r, adev_to_drm(tmp_adev)->unique);
4879                                 break;
4880                         }
4881                 }
4882
4883                 /* For XGMI wait for all resets to complete before proceed */
4884                 if (!r) {
4885                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4886                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4887                                         flush_work(&tmp_adev->xgmi_reset_work);
4888                                         r = tmp_adev->asic_reset_res;
4889                                         if (r)
4890                                                 break;
4891                                 }
4892                         }
4893                 }
4894         }
4895
4896         if (!r && amdgpu_ras_intr_triggered()) {
4897                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4898                         if (tmp_adev->mmhub.ras && tmp_adev->mmhub.ras->ras_block.hw_ops &&
4899                             tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
4900                                 tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(tmp_adev);
4901                 }
4902
4903                 amdgpu_ras_intr_cleared();
4904         }
4905
4906         /* Since the mode1 reset affects base ip blocks, the
4907          * phase1 ip blocks need to be resumed. Otherwise there
4908          * will be a BIOS signature error and the psp bootloader
4909          * can't load kdb on the next amdgpu install.
4910          */
4911         if (gpu_reset_for_dev_remove) {
4912                 list_for_each_entry(tmp_adev, device_list_handle, reset_list)
4913                         amdgpu_device_ip_resume_phase1(tmp_adev);
4914
4915                 goto end;
4916         }
4917
4918         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4919                 if (need_full_reset) {
4920                         /* post card */
4921                         r = amdgpu_device_asic_init(tmp_adev);
4922                         if (r) {
4923                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
4924                         } else {
4925                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4926                                 r = amdgpu_amdkfd_resume_iommu(tmp_adev);
4927                                 if (r)
4928                                         goto out;
4929
4930                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4931                                 if (r)
4932                                         goto out;
4933
4934                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4935 #ifdef CONFIG_DEV_COREDUMP
4936                                 tmp_adev->reset_vram_lost = vram_lost;
4937                                 memset(&tmp_adev->reset_task_info, 0,
4938                                                 sizeof(tmp_adev->reset_task_info));
4939                                 if (reset_context->job && reset_context->job->vm)
4940                                         tmp_adev->reset_task_info =
4941                                                 reset_context->job->vm->task_info;
4942                                 amdgpu_reset_capture_coredumpm(tmp_adev);
4943 #endif
4944                                 if (vram_lost) {
4945                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
4946                                         amdgpu_inc_vram_lost(tmp_adev);
4947                                 }
4948
4949                                 r = amdgpu_device_fw_loading(tmp_adev);
4950                                 if (r)
4951                                         return r;
4952
4953                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4954                                 if (r)
4955                                         goto out;
4956
4957                                 if (vram_lost)
4958                                         amdgpu_device_fill_reset_magic(tmp_adev);
4959
4960                                 /*
4961                                  * Add this ASIC as tracked as reset was already
4962                                  * complete successfully.
4963                                  */
4964                                 amdgpu_register_gpu_instance(tmp_adev);
4965
4966                                 if (!reset_context->hive &&
4967                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4968                                         amdgpu_xgmi_add_device(tmp_adev);
4969
4970                                 r = amdgpu_device_ip_late_init(tmp_adev);
4971                                 if (r)
4972                                         goto out;
4973
4974                                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
4975
4976                                 /*
4977                                  * The GPU enters bad state once faulty pages
4978                                  * by ECC has reached the threshold, and ras
4979                                  * recovery is scheduled next. So add one check
4980                                  * here to break recovery if it indeed exceeds
4981                                  * bad page threshold, and remind user to
4982                                  * retire this GPU or setting one bigger
4983                                  * bad_page_threshold value to fix this once
4984                                  * probing driver again.
4985                                  */
4986                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
4987                                         /* must succeed. */
4988                                         amdgpu_ras_resume(tmp_adev);
4989                                 } else {
4990                                         r = -EINVAL;
4991                                         goto out;
4992                                 }
4993
4994                                 /* Update PSP FW topology after reset */
4995                                 if (reset_context->hive &&
4996                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4997                                         r = amdgpu_xgmi_update_topology(
4998                                                 reset_context->hive, tmp_adev);
4999                         }
5000                 }
5001
5002 out:
5003                 if (!r) {
5004                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5005                         r = amdgpu_ib_ring_tests(tmp_adev);
5006                         if (r) {
5007                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5008                                 need_full_reset = true;
5009                                 r = -EAGAIN;
5010                                 goto end;
5011                         }
5012                 }
5013
5014                 if (!r)
5015                         r = amdgpu_device_recover_vram(tmp_adev);
5016                 else
5017                         tmp_adev->asic_reset_res = r;
5018         }
5019
5020 end:
5021         if (need_full_reset)
5022                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5023         else
5024                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5025         return r;
5026 }
5027
5028 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5029 {
5030
5031         switch (amdgpu_asic_reset_method(adev)) {
5032         case AMD_RESET_METHOD_MODE1:
5033                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5034                 break;
5035         case AMD_RESET_METHOD_MODE2:
5036                 adev->mp1_state = PP_MP1_STATE_RESET;
5037                 break;
5038         default:
5039                 adev->mp1_state = PP_MP1_STATE_NONE;
5040                 break;
5041         }
5042 }
5043
5044 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5045 {
5046         amdgpu_vf_error_trans_all(adev);
5047         adev->mp1_state = PP_MP1_STATE_NONE;
5048 }
5049
5050 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5051 {
5052         struct pci_dev *p = NULL;
5053
5054         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5055                         adev->pdev->bus->number, 1);
5056         if (p) {
5057                 pm_runtime_enable(&(p->dev));
5058                 pm_runtime_resume(&(p->dev));
5059         }
5060
5061         pci_dev_put(p);
5062 }
5063
5064 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5065 {
5066         enum amd_reset_method reset_method;
5067         struct pci_dev *p = NULL;
5068         u64 expires;
5069
5070         /*
5071          * For now, only BACO and mode1 reset are confirmed
5072          * to suffer the audio issue without proper suspended.
5073          */
5074         reset_method = amdgpu_asic_reset_method(adev);
5075         if ((reset_method != AMD_RESET_METHOD_BACO) &&
5076              (reset_method != AMD_RESET_METHOD_MODE1))
5077                 return -EINVAL;
5078
5079         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5080                         adev->pdev->bus->number, 1);
5081         if (!p)
5082                 return -ENODEV;
5083
5084         expires = pm_runtime_autosuspend_expiration(&(p->dev));
5085         if (!expires)
5086                 /*
5087                  * If we cannot get the audio device autosuspend delay,
5088                  * a fixed 4S interval will be used. Considering 3S is
5089                  * the audio controller default autosuspend delay setting.
5090                  * 4S used here is guaranteed to cover that.
5091                  */
5092                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5093
5094         while (!pm_runtime_status_suspended(&(p->dev))) {
5095                 if (!pm_runtime_suspend(&(p->dev)))
5096                         break;
5097
5098                 if (expires < ktime_get_mono_fast_ns()) {
5099                         dev_warn(adev->dev, "failed to suspend display audio\n");
5100                         pci_dev_put(p);
5101                         /* TODO: abort the succeeding gpu reset? */
5102                         return -ETIMEDOUT;
5103                 }
5104         }
5105
5106         pm_runtime_disable(&(p->dev));
5107
5108         pci_dev_put(p);
5109         return 0;
5110 }
5111
5112 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5113 {
5114         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5115
5116 #if defined(CONFIG_DEBUG_FS)
5117         if (!amdgpu_sriov_vf(adev))
5118                 cancel_work(&adev->reset_work);
5119 #endif
5120
5121         if (adev->kfd.dev)
5122                 cancel_work(&adev->kfd.reset_work);
5123
5124         if (amdgpu_sriov_vf(adev))
5125                 cancel_work(&adev->virt.flr_work);
5126
5127         if (con && adev->ras_enabled)
5128                 cancel_work(&con->recovery_work);
5129
5130 }
5131
5132 /**
5133  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5134  *
5135  * @adev: amdgpu_device pointer
5136  * @job: which job trigger hang
5137  *
5138  * Attempt to reset the GPU if it has hung (all asics).
5139  * Attempt to do soft-reset or full-reset and reinitialize Asic
5140  * Returns 0 for success or an error on failure.
5141  */
5142
5143 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5144                               struct amdgpu_job *job,
5145                               struct amdgpu_reset_context *reset_context)
5146 {
5147         struct list_head device_list, *device_list_handle =  NULL;
5148         bool job_signaled = false;
5149         struct amdgpu_hive_info *hive = NULL;
5150         struct amdgpu_device *tmp_adev = NULL;
5151         int i, r = 0;
5152         bool need_emergency_restart = false;
5153         bool audio_suspended = false;
5154         bool gpu_reset_for_dev_remove = false;
5155
5156         gpu_reset_for_dev_remove =
5157                         test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5158                                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5159
5160         /*
5161          * Special case: RAS triggered and full reset isn't supported
5162          */
5163         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5164
5165         /*
5166          * Flush RAM to disk so that after reboot
5167          * the user can read log and see why the system rebooted.
5168          */
5169         if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
5170                 DRM_WARN("Emergency reboot.");
5171
5172                 ksys_sync_helper();
5173                 emergency_restart();
5174         }
5175
5176         dev_info(adev->dev, "GPU %s begin!\n",
5177                 need_emergency_restart ? "jobs stop":"reset");
5178
5179         if (!amdgpu_sriov_vf(adev))
5180                 hive = amdgpu_get_xgmi_hive(adev);
5181         if (hive)
5182                 mutex_lock(&hive->hive_lock);
5183
5184         reset_context->job = job;
5185         reset_context->hive = hive;
5186         /*
5187          * Build list of devices to reset.
5188          * In case we are in XGMI hive mode, resort the device list
5189          * to put adev in the 1st position.
5190          */
5191         INIT_LIST_HEAD(&device_list);
5192         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5193                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5194                         list_add_tail(&tmp_adev->reset_list, &device_list);
5195                         if (gpu_reset_for_dev_remove && adev->shutdown)
5196                                 tmp_adev->shutdown = true;
5197                 }
5198                 if (!list_is_first(&adev->reset_list, &device_list))
5199                         list_rotate_to_front(&adev->reset_list, &device_list);
5200                 device_list_handle = &device_list;
5201         } else {
5202                 list_add_tail(&adev->reset_list, &device_list);
5203                 device_list_handle = &device_list;
5204         }
5205
5206         /* We need to lock reset domain only once both for XGMI and single device */
5207         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5208                                     reset_list);
5209         amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5210
5211         /* block all schedulers and reset given job's ring */
5212         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5213
5214                 amdgpu_device_set_mp1_state(tmp_adev);
5215
5216                 /*
5217                  * Try to put the audio codec into suspend state
5218                  * before gpu reset started.
5219                  *
5220                  * Due to the power domain of the graphics device
5221                  * is shared with AZ power domain. Without this,
5222                  * we may change the audio hardware from behind
5223                  * the audio driver's back. That will trigger
5224                  * some audio codec errors.
5225                  */
5226                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5227                         audio_suspended = true;
5228
5229                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5230
5231                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5232
5233                 if (!amdgpu_sriov_vf(tmp_adev))
5234                         amdgpu_amdkfd_pre_reset(tmp_adev);
5235
5236                 /*
5237                  * Mark these ASICs to be reseted as untracked first
5238                  * And add them back after reset completed
5239                  */
5240                 amdgpu_unregister_gpu_instance(tmp_adev);
5241
5242                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5243
5244                 /* disable ras on ALL IPs */
5245                 if (!need_emergency_restart &&
5246                       amdgpu_device_ip_need_full_reset(tmp_adev))
5247                         amdgpu_ras_suspend(tmp_adev);
5248
5249                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5250                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5251
5252                         if (!ring || !ring->sched.thread)
5253                                 continue;
5254
5255                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5256
5257                         if (need_emergency_restart)
5258                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5259                 }
5260                 atomic_inc(&tmp_adev->gpu_reset_counter);
5261         }
5262
5263         if (need_emergency_restart)
5264                 goto skip_sched_resume;
5265
5266         /*
5267          * Must check guilty signal here since after this point all old
5268          * HW fences are force signaled.
5269          *
5270          * job->base holds a reference to parent fence
5271          */
5272         if (job && dma_fence_is_signaled(&job->hw_fence)) {
5273                 job_signaled = true;
5274                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5275                 goto skip_hw_reset;
5276         }
5277
5278 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5279         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5280                 if (gpu_reset_for_dev_remove) {
5281                         /* Workaroud for ASICs need to disable SMC first */
5282                         amdgpu_device_smu_fini_early(tmp_adev);
5283                 }
5284                 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5285                 /*TODO Should we stop ?*/
5286                 if (r) {
5287                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5288                                   r, adev_to_drm(tmp_adev)->unique);
5289                         tmp_adev->asic_reset_res = r;
5290                 }
5291
5292                 /*
5293                  * Drop all pending non scheduler resets. Scheduler resets
5294                  * were already dropped during drm_sched_stop
5295                  */
5296                 amdgpu_device_stop_pending_resets(tmp_adev);
5297         }
5298
5299         /* Actual ASIC resets if needed.*/
5300         /* Host driver will handle XGMI hive reset for SRIOV */
5301         if (amdgpu_sriov_vf(adev)) {
5302                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5303                 if (r)
5304                         adev->asic_reset_res = r;
5305
5306                 /* Aldebaran supports ras in SRIOV, so need resume ras during reset */
5307                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
5308                         amdgpu_ras_resume(adev);
5309         } else {
5310                 r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5311                 if (r && r == -EAGAIN)
5312                         goto retry;
5313
5314                 if (!r && gpu_reset_for_dev_remove)
5315                         goto recover_end;
5316         }
5317
5318 skip_hw_reset:
5319
5320         /* Post ASIC reset for all devs .*/
5321         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5322
5323                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5324                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5325
5326                         if (!ring || !ring->sched.thread)
5327                                 continue;
5328
5329                         drm_sched_start(&ring->sched, true);
5330                 }
5331
5332                 if (adev->enable_mes && adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3))
5333                         amdgpu_mes_self_test(tmp_adev);
5334
5335                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) {
5336                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5337                 }
5338
5339                 if (tmp_adev->asic_reset_res)
5340                         r = tmp_adev->asic_reset_res;
5341
5342                 tmp_adev->asic_reset_res = 0;
5343
5344                 if (r) {
5345                         /* bad news, how to tell it to userspace ? */
5346                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5347                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5348                 } else {
5349                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5350                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5351                                 DRM_WARN("smart shift update failed\n");
5352                 }
5353         }
5354
5355 skip_sched_resume:
5356         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5357                 /* unlock kfd: SRIOV would do it separately */
5358                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5359                         amdgpu_amdkfd_post_reset(tmp_adev);
5360
5361                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5362                  * need to bring up kfd here if it's not be initialized before
5363                  */
5364                 if (!adev->kfd.init_complete)
5365                         amdgpu_amdkfd_device_init(adev);
5366
5367                 if (audio_suspended)
5368                         amdgpu_device_resume_display_audio(tmp_adev);
5369
5370                 amdgpu_device_unset_mp1_state(tmp_adev);
5371
5372                 amdgpu_ras_set_error_query_ready(tmp_adev, true);
5373         }
5374
5375 recover_end:
5376         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5377                                             reset_list);
5378         amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5379
5380         if (hive) {
5381                 mutex_unlock(&hive->hive_lock);
5382                 amdgpu_put_xgmi_hive(hive);
5383         }
5384
5385         if (r)
5386                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5387
5388         atomic_set(&adev->reset_domain->reset_res, r);
5389         return r;
5390 }
5391
5392 /**
5393  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5394  *
5395  * @adev: amdgpu_device pointer
5396  *
5397  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5398  * and lanes) of the slot the device is in. Handles APUs and
5399  * virtualized environments where PCIE config space may not be available.
5400  */
5401 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5402 {
5403         struct pci_dev *pdev;
5404         enum pci_bus_speed speed_cap, platform_speed_cap;
5405         enum pcie_link_width platform_link_width;
5406
5407         if (amdgpu_pcie_gen_cap)
5408                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5409
5410         if (amdgpu_pcie_lane_cap)
5411                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5412
5413         /* covers APUs as well */
5414         if (pci_is_root_bus(adev->pdev->bus)) {
5415                 if (adev->pm.pcie_gen_mask == 0)
5416                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5417                 if (adev->pm.pcie_mlw_mask == 0)
5418                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5419                 return;
5420         }
5421
5422         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5423                 return;
5424
5425         pcie_bandwidth_available(adev->pdev, NULL,
5426                                  &platform_speed_cap, &platform_link_width);
5427
5428         if (adev->pm.pcie_gen_mask == 0) {
5429                 /* asic caps */
5430                 pdev = adev->pdev;
5431                 speed_cap = pcie_get_speed_cap(pdev);
5432                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5433                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5434                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5435                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5436                 } else {
5437                         if (speed_cap == PCIE_SPEED_32_0GT)
5438                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5439                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5440                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5441                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5442                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5443                         else if (speed_cap == PCIE_SPEED_16_0GT)
5444                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5445                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5446                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5447                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5448                         else if (speed_cap == PCIE_SPEED_8_0GT)
5449                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5450                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5451                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5452                         else if (speed_cap == PCIE_SPEED_5_0GT)
5453                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5454                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5455                         else
5456                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5457                 }
5458                 /* platform caps */
5459                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5460                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5461                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5462                 } else {
5463                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5464                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5465                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5466                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5467                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5468                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5469                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5470                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5471                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5472                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5473                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5474                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5475                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5476                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5477                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5478                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5479                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5480                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5481                         else
5482                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5483
5484                 }
5485         }
5486         if (adev->pm.pcie_mlw_mask == 0) {
5487                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5488                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5489                 } else {
5490                         switch (platform_link_width) {
5491                         case PCIE_LNK_X32:
5492                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5493                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5494                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5495                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5496                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5497                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5498                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5499                                 break;
5500                         case PCIE_LNK_X16:
5501                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5502                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5503                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5504                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5505                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5506                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5507                                 break;
5508                         case PCIE_LNK_X12:
5509                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5510                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5511                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5512                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5513                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5514                                 break;
5515                         case PCIE_LNK_X8:
5516                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5517                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5518                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5519                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5520                                 break;
5521                         case PCIE_LNK_X4:
5522                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5523                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5524                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5525                                 break;
5526                         case PCIE_LNK_X2:
5527                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5528                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5529                                 break;
5530                         case PCIE_LNK_X1:
5531                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5532                                 break;
5533                         default:
5534                                 break;
5535                         }
5536                 }
5537         }
5538 }
5539
5540 /**
5541  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5542  *
5543  * @adev: amdgpu_device pointer
5544  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5545  *
5546  * Return true if @peer_adev can access (DMA) @adev through the PCIe
5547  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5548  * @peer_adev.
5549  */
5550 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5551                                       struct amdgpu_device *peer_adev)
5552 {
5553 #ifdef CONFIG_HSA_AMD_P2P
5554         uint64_t address_mask = peer_adev->dev->dma_mask ?
5555                 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5556         resource_size_t aper_limit =
5557                 adev->gmc.aper_base + adev->gmc.aper_size - 1;
5558         bool p2p_access =
5559                 !adev->gmc.xgmi.connected_to_cpu &&
5560                 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5561
5562         return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5563                 adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5564                 !(adev->gmc.aper_base & address_mask ||
5565                   aper_limit & address_mask));
5566 #else
5567         return false;
5568 #endif
5569 }
5570
5571 int amdgpu_device_baco_enter(struct drm_device *dev)
5572 {
5573         struct amdgpu_device *adev = drm_to_adev(dev);
5574         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5575
5576         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5577                 return -ENOTSUPP;
5578
5579         if (ras && adev->ras_enabled &&
5580             adev->nbio.funcs->enable_doorbell_interrupt)
5581                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5582
5583         return amdgpu_dpm_baco_enter(adev);
5584 }
5585
5586 int amdgpu_device_baco_exit(struct drm_device *dev)
5587 {
5588         struct amdgpu_device *adev = drm_to_adev(dev);
5589         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5590         int ret = 0;
5591
5592         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5593                 return -ENOTSUPP;
5594
5595         ret = amdgpu_dpm_baco_exit(adev);
5596         if (ret)
5597                 return ret;
5598
5599         if (ras && adev->ras_enabled &&
5600             adev->nbio.funcs->enable_doorbell_interrupt)
5601                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5602
5603         if (amdgpu_passthrough(adev) &&
5604             adev->nbio.funcs->clear_doorbell_interrupt)
5605                 adev->nbio.funcs->clear_doorbell_interrupt(adev);
5606
5607         return 0;
5608 }
5609
5610 /**
5611  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5612  * @pdev: PCI device struct
5613  * @state: PCI channel state
5614  *
5615  * Description: Called when a PCI error is detected.
5616  *
5617  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5618  */
5619 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5620 {
5621         struct drm_device *dev = pci_get_drvdata(pdev);
5622         struct amdgpu_device *adev = drm_to_adev(dev);
5623         int i;
5624
5625         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5626
5627         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5628                 DRM_WARN("No support for XGMI hive yet...");
5629                 return PCI_ERS_RESULT_DISCONNECT;
5630         }
5631
5632         adev->pci_channel_state = state;
5633
5634         switch (state) {
5635         case pci_channel_io_normal:
5636                 return PCI_ERS_RESULT_CAN_RECOVER;
5637         /* Fatal error, prepare for slot reset */
5638         case pci_channel_io_frozen:
5639                 /*
5640                  * Locking adev->reset_domain->sem will prevent any external access
5641                  * to GPU during PCI error recovery
5642                  */
5643                 amdgpu_device_lock_reset_domain(adev->reset_domain);
5644                 amdgpu_device_set_mp1_state(adev);
5645
5646                 /*
5647                  * Block any work scheduling as we do for regular GPU reset
5648                  * for the duration of the recovery
5649                  */
5650                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5651                         struct amdgpu_ring *ring = adev->rings[i];
5652
5653                         if (!ring || !ring->sched.thread)
5654                                 continue;
5655
5656                         drm_sched_stop(&ring->sched, NULL);
5657                 }
5658                 atomic_inc(&adev->gpu_reset_counter);
5659                 return PCI_ERS_RESULT_NEED_RESET;
5660         case pci_channel_io_perm_failure:
5661                 /* Permanent error, prepare for device removal */
5662                 return PCI_ERS_RESULT_DISCONNECT;
5663         }
5664
5665         return PCI_ERS_RESULT_NEED_RESET;
5666 }
5667
5668 /**
5669  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5670  * @pdev: pointer to PCI device
5671  */
5672 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5673 {
5674
5675         DRM_INFO("PCI error: mmio enabled callback!!\n");
5676
5677         /* TODO - dump whatever for debugging purposes */
5678
5679         /* This called only if amdgpu_pci_error_detected returns
5680          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5681          * works, no need to reset slot.
5682          */
5683
5684         return PCI_ERS_RESULT_RECOVERED;
5685 }
5686
5687 /**
5688  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5689  * @pdev: PCI device struct
5690  *
5691  * Description: This routine is called by the pci error recovery
5692  * code after the PCI slot has been reset, just before we
5693  * should resume normal operations.
5694  */
5695 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5696 {
5697         struct drm_device *dev = pci_get_drvdata(pdev);
5698         struct amdgpu_device *adev = drm_to_adev(dev);
5699         int r, i;
5700         struct amdgpu_reset_context reset_context;
5701         u32 memsize;
5702         struct list_head device_list;
5703
5704         DRM_INFO("PCI error: slot reset callback!!\n");
5705
5706         memset(&reset_context, 0, sizeof(reset_context));
5707
5708         INIT_LIST_HEAD(&device_list);
5709         list_add_tail(&adev->reset_list, &device_list);
5710
5711         /* wait for asic to come out of reset */
5712         msleep(500);
5713
5714         /* Restore PCI confspace */
5715         amdgpu_device_load_pci_state(pdev);
5716
5717         /* confirm  ASIC came out of reset */
5718         for (i = 0; i < adev->usec_timeout; i++) {
5719                 memsize = amdgpu_asic_get_config_memsize(adev);
5720
5721                 if (memsize != 0xffffffff)
5722                         break;
5723                 udelay(1);
5724         }
5725         if (memsize == 0xffffffff) {
5726                 r = -ETIME;
5727                 goto out;
5728         }
5729
5730         reset_context.method = AMD_RESET_METHOD_NONE;
5731         reset_context.reset_req_dev = adev;
5732         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5733         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5734
5735         adev->no_hw_access = true;
5736         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5737         adev->no_hw_access = false;
5738         if (r)
5739                 goto out;
5740
5741         r = amdgpu_do_asic_reset(&device_list, &reset_context);
5742
5743 out:
5744         if (!r) {
5745                 if (amdgpu_device_cache_pci_state(adev->pdev))
5746                         pci_restore_state(adev->pdev);
5747
5748                 DRM_INFO("PCIe error recovery succeeded\n");
5749         } else {
5750                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5751                 amdgpu_device_unset_mp1_state(adev);
5752                 amdgpu_device_unlock_reset_domain(adev->reset_domain);
5753         }
5754
5755         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5756 }
5757
5758 /**
5759  * amdgpu_pci_resume() - resume normal ops after PCI reset
5760  * @pdev: pointer to PCI device
5761  *
5762  * Called when the error recovery driver tells us that its
5763  * OK to resume normal operation.
5764  */
5765 void amdgpu_pci_resume(struct pci_dev *pdev)
5766 {
5767         struct drm_device *dev = pci_get_drvdata(pdev);
5768         struct amdgpu_device *adev = drm_to_adev(dev);
5769         int i;
5770
5771
5772         DRM_INFO("PCI error: resume callback!!\n");
5773
5774         /* Only continue execution for the case of pci_channel_io_frozen */
5775         if (adev->pci_channel_state != pci_channel_io_frozen)
5776                 return;
5777
5778         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5779                 struct amdgpu_ring *ring = adev->rings[i];
5780
5781                 if (!ring || !ring->sched.thread)
5782                         continue;
5783
5784                 drm_sched_start(&ring->sched, true);
5785         }
5786
5787         amdgpu_device_unset_mp1_state(adev);
5788         amdgpu_device_unlock_reset_domain(adev->reset_domain);
5789 }
5790
5791 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5792 {
5793         struct drm_device *dev = pci_get_drvdata(pdev);
5794         struct amdgpu_device *adev = drm_to_adev(dev);
5795         int r;
5796
5797         r = pci_save_state(pdev);
5798         if (!r) {
5799                 kfree(adev->pci_state);
5800
5801                 adev->pci_state = pci_store_saved_state(pdev);
5802
5803                 if (!adev->pci_state) {
5804                         DRM_ERROR("Failed to store PCI saved state");
5805                         return false;
5806                 }
5807         } else {
5808                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
5809                 return false;
5810         }
5811
5812         return true;
5813 }
5814
5815 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5816 {
5817         struct drm_device *dev = pci_get_drvdata(pdev);
5818         struct amdgpu_device *adev = drm_to_adev(dev);
5819         int r;
5820
5821         if (!adev->pci_state)
5822                 return false;
5823
5824         r = pci_load_saved_state(pdev, adev->pci_state);
5825
5826         if (!r) {
5827                 pci_restore_state(pdev);
5828         } else {
5829                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
5830                 return false;
5831         }
5832
5833         return true;
5834 }
5835
5836 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
5837                 struct amdgpu_ring *ring)
5838 {
5839 #ifdef CONFIG_X86_64
5840         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5841                 return;
5842 #endif
5843         if (adev->gmc.xgmi.connected_to_cpu)
5844                 return;
5845
5846         if (ring && ring->funcs->emit_hdp_flush)
5847                 amdgpu_ring_emit_hdp_flush(ring);
5848         else
5849                 amdgpu_asic_flush_hdp(adev, ring);
5850 }
5851
5852 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
5853                 struct amdgpu_ring *ring)
5854 {
5855 #ifdef CONFIG_X86_64
5856         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5857                 return;
5858 #endif
5859         if (adev->gmc.xgmi.connected_to_cpu)
5860                 return;
5861
5862         amdgpu_asic_invalidate_hdp(adev, ring);
5863 }
5864
5865 int amdgpu_in_reset(struct amdgpu_device *adev)
5866 {
5867         return atomic_read(&adev->reset_domain->in_gpu_reset);
5868         }
5869
5870 /**
5871  * amdgpu_device_halt() - bring hardware to some kind of halt state
5872  *
5873  * @adev: amdgpu_device pointer
5874  *
5875  * Bring hardware to some kind of halt state so that no one can touch it
5876  * any more. It will help to maintain error context when error occurred.
5877  * Compare to a simple hang, the system will keep stable at least for SSH
5878  * access. Then it should be trivial to inspect the hardware state and
5879  * see what's going on. Implemented as following:
5880  *
5881  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
5882  *    clears all CPU mappings to device, disallows remappings through page faults
5883  * 2. amdgpu_irq_disable_all() disables all interrupts
5884  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
5885  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
5886  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
5887  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
5888  *    flush any in flight DMA operations
5889  */
5890 void amdgpu_device_halt(struct amdgpu_device *adev)
5891 {
5892         struct pci_dev *pdev = adev->pdev;
5893         struct drm_device *ddev = adev_to_drm(adev);
5894
5895         drm_dev_unplug(ddev);
5896
5897         amdgpu_irq_disable_all(adev);
5898
5899         amdgpu_fence_driver_hw_fini(adev);
5900
5901         adev->no_hw_access = true;
5902
5903         amdgpu_device_unmap_mmio(adev);
5904
5905         pci_disable_device(pdev);
5906         pci_wait_for_pending_transaction(pdev);
5907 }
5908
5909 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
5910                                 u32 reg)
5911 {
5912         unsigned long flags, address, data;
5913         u32 r;
5914
5915         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5916         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5917
5918         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5919         WREG32(address, reg * 4);
5920         (void)RREG32(address);
5921         r = RREG32(data);
5922         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5923         return r;
5924 }
5925
5926 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
5927                                 u32 reg, u32 v)
5928 {
5929         unsigned long flags, address, data;
5930
5931         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5932         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5933
5934         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5935         WREG32(address, reg * 4);
5936         (void)RREG32(address);
5937         WREG32(data, v);
5938         (void)RREG32(data);
5939         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5940 }
5941
5942 /**
5943  * amdgpu_device_switch_gang - switch to a new gang
5944  * @adev: amdgpu_device pointer
5945  * @gang: the gang to switch to
5946  *
5947  * Try to switch to a new gang.
5948  * Returns: NULL if we switched to the new gang or a reference to the current
5949  * gang leader.
5950  */
5951 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
5952                                             struct dma_fence *gang)
5953 {
5954         struct dma_fence *old = NULL;
5955
5956         do {
5957                 dma_fence_put(old);
5958                 rcu_read_lock();
5959                 old = dma_fence_get_rcu_safe(&adev->gang_submit);
5960                 rcu_read_unlock();
5961
5962                 if (old == gang)
5963                         break;
5964
5965                 if (!dma_fence_is_signaled(old))
5966                         return old;
5967
5968         } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
5969                          old, gang) != old);
5970
5971         dma_fence_put(old);
5972         return NULL;
5973 }
5974
5975 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
5976 {
5977         switch (adev->asic_type) {
5978 #ifdef CONFIG_DRM_AMDGPU_SI
5979         case CHIP_HAINAN:
5980 #endif
5981         case CHIP_TOPAZ:
5982                 /* chips with no display hardware */
5983                 return false;
5984 #ifdef CONFIG_DRM_AMDGPU_SI
5985         case CHIP_TAHITI:
5986         case CHIP_PITCAIRN:
5987         case CHIP_VERDE:
5988         case CHIP_OLAND:
5989 #endif
5990 #ifdef CONFIG_DRM_AMDGPU_CIK
5991         case CHIP_BONAIRE:
5992         case CHIP_HAWAII:
5993         case CHIP_KAVERI:
5994         case CHIP_KABINI:
5995         case CHIP_MULLINS:
5996 #endif
5997         case CHIP_TONGA:
5998         case CHIP_FIJI:
5999         case CHIP_POLARIS10:
6000         case CHIP_POLARIS11:
6001         case CHIP_POLARIS12:
6002         case CHIP_VEGAM:
6003         case CHIP_CARRIZO:
6004         case CHIP_STONEY:
6005                 /* chips with display hardware */
6006                 return true;
6007         default:
6008                 /* IP discovery */
6009                 if (!adev->ip_versions[DCE_HWIP][0] ||
6010                     (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6011                         return false;
6012                 return true;
6013         }
6014 }