drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "amdgpu.h"
  24 #include "mmhub_v9_4.h"
  25
  26 #include "mmhub/mmhub_9_4_1_offset.h"
  27 #include "mmhub/mmhub_9_4_1_sh_mask.h"
  28 #include "mmhub/mmhub_9_4_1_default.h"
  29 #include "athub/athub_1_0_offset.h"
  30 #include "athub/athub_1_0_sh_mask.h"
  31 #include "vega10_enum.h"
  32
  33 #include "soc15_common.h"
  34
  35 #define MMHUB_NUM_INSTANCES                     2
  36 #define MMHUB_INSTANCE_REGISTER_OFFSET          0x3000
  37
  38 u64 mmhub_v9_4_get_fb_location(struct amdgpu_device *adev)
  39 {
  40         /* The base should be same b/t 2 mmhubs on Acrturus. Read one here. */
  41         u64 base = RREG32_SOC15(MMHUB, 0, mmVMSHAREDVC0_MC_VM_FB_LOCATION_BASE);
  42
  43         base &= VMSHAREDVC0_MC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
  44         base <<= 24;
  45
  46         return base;
  47 }
  48
  49 void mmhub_v9_4_setup_vm_pt_regs(struct amdgpu_device *adev, int hubid,
  50                                 uint32_t vmid, uint64_t value)
  51 {
  52         /* two registers distance between mmVML2VC0_VM_CONTEXT0_* to
  53          * mmVML2VC0_VM_CONTEXT1_*
  54          */
  55         int dist = mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
  56                         - mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
  57
  58         WREG32_SOC15_OFFSET(MMHUB, 0,
  59                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
  60                             dist * vmid + hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  61                             lower_32_bits(value));
  62
  63         WREG32_SOC15_OFFSET(MMHUB, 0,
  64                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
  65                             dist * vmid + hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  66                             upper_32_bits(value));
  67
  68 }
  69
  70 static void mmhub_v9_4_init_gart_aperture_regs(struct amdgpu_device *adev,
  71                                                int hubid)
  72 {
  73         uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
  74
  75         mmhub_v9_4_setup_vm_pt_regs(adev, hubid, 0, pt_base);
  76
  77         WREG32_SOC15_OFFSET(MMHUB, 0,
  78                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
  79                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  80                             (u32)(adev->gmc.gart_start >> 12));
  81         WREG32_SOC15_OFFSET(MMHUB, 0,
  82                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
  83                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  84                             (u32)(adev->gmc.gart_start >> 44));
  85
  86         WREG32_SOC15_OFFSET(MMHUB, 0,
  87                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
  88                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  89                             (u32)(adev->gmc.gart_end >> 12));
  90         WREG32_SOC15_OFFSET(MMHUB, 0,
  91                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
  92                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  93                             (u32)(adev->gmc.gart_end >> 44));
  94 }
  95
  96 static void mmhub_v9_4_init_system_aperture_regs(struct amdgpu_device *adev,
  97                                                  int hubid)
  98 {
  99         uint64_t value;
 100         uint32_t tmp;
 101
 102         /* Program the AGP BAR */
 103         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_AGP_BASE,
 104                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 105                             0);
 106         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_AGP_TOP,
 107                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 108                             adev->gmc.agp_end >> 24);
 109         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_AGP_BOT,
 110                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 111                             adev->gmc.agp_start >> 24);
 112
 113         /* Program the system aperture low logical page number. */
 114         WREG32_SOC15_OFFSET(MMHUB, 0,
 115                             mmVMSHAREDVC0_MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 116                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 117                             min(adev->gmc.vram_start, adev->gmc.agp_start)
 118                             >> 18);
 119         WREG32_SOC15_OFFSET(MMHUB, 0,
 120                             mmVMSHAREDVC0_MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 121                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 122                             max(adev->gmc.vram_end, adev->gmc.agp_end) >> 18);
 123
 124         /* Set default page address. */
 125         value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start +
 126                 adev->vm_manager.vram_base_offset;
 127         WREG32_SOC15_OFFSET(MMHUB, 0,
 128                         mmVMSHAREDPF0_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
 129                         hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 130                         (u32)(value >> 12));
 131         WREG32_SOC15_OFFSET(MMHUB, 0,
 132                         mmVMSHAREDPF0_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
 133                         hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 134                         (u32)(value >> 44));
 135
 136         /* Program "protection fault". */
 137         WREG32_SOC15_OFFSET(MMHUB, 0,
 138                             mmVML2PF0_VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
 139                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 140                             (u32)(adev->dummy_page_addr >> 12));
 141         WREG32_SOC15_OFFSET(MMHUB, 0,
 142                             mmVML2PF0_VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 143                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 144                             (u32)((u64)adev->dummy_page_addr >> 44));
 145
 146         tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 147                                   mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL2,
 148                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 149         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL2,
 150                             ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
 151         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL2,
 152                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 153 }
 154
 155 static void mmhub_v9_4_init_tlb_regs(struct amdgpu_device *adev, int hubid)
 156 {
 157         uint32_t tmp;
 158
 159         /* Setup TLB control */
 160         tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 161                            mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 162                            hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 163
 164         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 165                             ENABLE_L1_TLB, 1);
 166         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 167                             SYSTEM_ACCESS_MODE, 3);
 168         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 169                             ENABLE_ADVANCED_DRIVER_MODEL, 1);
 170         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 171                             SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
 172         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 173                             ECO_BITS, 0);
 174         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 175                             MTYPE, MTYPE_UC);/* XXX for emulation. */
 176         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 177                             ATC_EN, 1);
 178
 179         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 180                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 181 }
 182
 183 static void mmhub_v9_4_init_cache_regs(struct amdgpu_device *adev, int hubid)
 184 {
 185         uint32_t tmp;
 186
 187         /* Setup L2 cache */
 188         tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 189                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 190         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 191                             ENABLE_L2_CACHE, 1);
 192         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 193                             ENABLE_L2_FRAGMENT_PROCESSING, 1);
 194         /* XXX for emulation, Refer to closed source code.*/
 195         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 196                             L2_PDE0_CACHE_TAG_GENERATION_MODE, 0);
 197         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 198                             PDE_FAULT_CLASSIFICATION, 1);
 199         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 200                             CONTEXT1_IDENTITY_ACCESS_MODE, 1);
 201         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 202                             IDENTITY_MODE_FRAGMENT_SIZE, 0);
 203         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 204                      hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 205
 206         tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL2,
 207                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 208         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL2,
 209                             INVALIDATE_ALL_L1_TLBS, 1);
 210         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL2,
 211                             INVALIDATE_L2_CACHE, 1);
 212         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL2,
 213                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 214
 215         tmp = mmVML2PF0_VM_L2_CNTL3_DEFAULT;
 216         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL3,
 217                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 218
 219         tmp = mmVML2PF0_VM_L2_CNTL4_DEFAULT;
 220         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL4,
 221                             VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
 222         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL4,
 223                             VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
 224         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL4,
 225                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 226 }
 227
 228 static void mmhub_v9_4_enable_system_domain(struct amdgpu_device *adev,
 229                                             int hubid)
 230 {
 231         uint32_t tmp;
 232
 233         tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT0_CNTL,
 234                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 235         tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
 236         tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
 237         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT0_CNTL,
 238                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 239 }
 240
 241 static void mmhub_v9_4_disable_identity_aperture(struct amdgpu_device *adev,
 242                                                  int hubid)
 243 {
 244         WREG32_SOC15_OFFSET(MMHUB, 0,
 245                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
 246                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0XFFFFFFFF);
 247         WREG32_SOC15_OFFSET(MMHUB, 0,
 248                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
 249                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0x0000000F);
 250
 251         WREG32_SOC15_OFFSET(MMHUB, 0,
 252                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
 253                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 254         WREG32_SOC15_OFFSET(MMHUB, 0,
 255                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
 256                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 257
 258         WREG32_SOC15_OFFSET(MMHUB, 0,
 259                     mmVML2PF0_VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
 260                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 261         WREG32_SOC15_OFFSET(MMHUB, 0,
 262                     mmVML2PF0_VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
 263                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 264 }
 265
 266 static void mmhub_v9_4_setup_vmid_config(struct amdgpu_device *adev, int hubid)
 267 {
 268         uint32_t tmp;
 269         int i;
 270
 271         for (i = 0; i <= 14; i++) {
 272                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT1_CNTL,
 273                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i);
 274                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 275                                     ENABLE_CONTEXT, 1);
 276                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 277                                     PAGE_TABLE_DEPTH,
 278                                     adev->vm_manager.num_level);
 279                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 280                                     RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 281                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 282                                     DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
 283                                     1);
 284                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 285                                     PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 286                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 287                                     VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 288                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 289                                     READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 290                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 291                                     WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 292                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 293                                     EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 294                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 295                                     PAGE_TABLE_BLOCK_SIZE,
 296                                     adev->vm_manager.block_size - 9);
 297                 /* Send no-retry XNACK on fault to suppress VM fault storm. */
 298                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 299                                     RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
 300                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT1_CNTL,
 301                                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i,
 302                                     tmp);
 303                 WREG32_SOC15_OFFSET(MMHUB, 0,
 304                             mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
 305                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2, 0);
 306                 WREG32_SOC15_OFFSET(MMHUB, 0,
 307                             mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
 308                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2, 0);
 309                 WREG32_SOC15_OFFSET(MMHUB, 0,
 310                                 mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
 311                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2,
 312                                 lower_32_bits(adev->vm_manager.max_pfn - 1));
 313                 WREG32_SOC15_OFFSET(MMHUB, 0,
 314                                 mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
 315                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2,
 316                                 upper_32_bits(adev->vm_manager.max_pfn - 1));
 317         }
 318 }
 319
 320 static void mmhub_v9_4_program_invalidation(struct amdgpu_device *adev,
 321                                             int hubid)
 322 {
 323         unsigned i;
 324
 325         for (i = 0; i < 18; ++i) {
 326                 WREG32_SOC15_OFFSET(MMHUB, 0,
 327                                 mmVML2VC0_VM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
 328                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + 2 * i,
 329                                 0xffffffff);
 330                 WREG32_SOC15_OFFSET(MMHUB, 0,
 331                                 mmVML2VC0_VM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
 332                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + 2 * i,
 333                                 0x1f);
 334         }
 335 }
 336
 337 int mmhub_v9_4_gart_enable(struct amdgpu_device *adev)
 338 {
 339         int i;
 340
 341         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 342                 if (amdgpu_sriov_vf(adev)) {
 343                         /*
 344                          * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase
 345                          * they are VF copy registers so vbios post doesn't
 346                          * program them, for SRIOV driver need to program them
 347                          */
 348                         WREG32_SOC15_OFFSET(MMHUB, 0,
 349                                      mmVMSHAREDVC0_MC_VM_FB_LOCATION_BASE,
 350                                      i * MMHUB_INSTANCE_REGISTER_OFFSET,
 351                                      adev->gmc.vram_start >> 24);
 352                         WREG32_SOC15_OFFSET(MMHUB, 0,
 353                                      mmVMSHAREDVC0_MC_VM_FB_LOCATION_TOP,
 354                                      i * MMHUB_INSTANCE_REGISTER_OFFSET,
 355                                      adev->gmc.vram_end >> 24);
 356                 }
 357
 358                 /* GART Enable. */
 359                 mmhub_v9_4_init_gart_aperture_regs(adev, i);
 360                 mmhub_v9_4_init_system_aperture_regs(adev, i);
 361                 mmhub_v9_4_init_tlb_regs(adev, i);
 362                 mmhub_v9_4_init_cache_regs(adev, i);
 363
 364                 mmhub_v9_4_enable_system_domain(adev, i);
 365                 mmhub_v9_4_disable_identity_aperture(adev, i);
 366                 mmhub_v9_4_setup_vmid_config(adev, i);
 367                 mmhub_v9_4_program_invalidation(adev, i);
 368         }
 369
 370         return 0;
 371 }
 372
 373 void mmhub_v9_4_gart_disable(struct amdgpu_device *adev)
 374 {
 375         u32 tmp;
 376         u32 i, j;
 377
 378         for (j = 0; j < MMHUB_NUM_INSTANCES; j++) {
 379                 /* Disable all tables */
 380                 for (i = 0; i < 16; i++)
 381                         WREG32_SOC15_OFFSET(MMHUB, 0,
 382                                             mmVML2VC0_VM_CONTEXT0_CNTL,
 383                                             j * MMHUB_INSTANCE_REGISTER_OFFSET +
 384                                             i, 0);
 385
 386                 /* Setup TLB control */
 387                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 388                                    mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 389                                    j * MMHUB_INSTANCE_REGISTER_OFFSET);
 390                 tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 391                                     ENABLE_L1_TLB, 0);
 392                 tmp = REG_SET_FIELD(tmp,
 393                                     VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 394                                     ENABLE_ADVANCED_DRIVER_MODEL, 0);
 395                 WREG32_SOC15_OFFSET(MMHUB, 0,
 396                                     mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 397                                     j * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 398
 399                 /* Setup L2 cache */
 400                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 401                                           j * MMHUB_INSTANCE_REGISTER_OFFSET);
 402                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 403                                     ENABLE_L2_CACHE, 0);
 404                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 405                                     j * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 406                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL3,
 407                                     j * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 408         }
 409 }
 410
 411 /**
 412  * mmhub_v1_0_set_fault_enable_default - update GART/VM fault handling
 413  *
 414  * @adev: amdgpu_device pointer
 415  * @value: true redirects VM faults to the default page
 416  */
 417 void mmhub_v9_4_set_fault_enable_default(struct amdgpu_device *adev, bool value)
 418 {
 419         u32 tmp;
 420         int i;
 421
 422         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 423                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 424                                           mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 425                                           i * MMHUB_INSTANCE_REGISTER_OFFSET);
 426                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 427                                     RANGE_PROTECTION_FAULT_ENABLE_DEFAULT,
 428                                     value);
 429                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 430                                     PDE0_PROTECTION_FAULT_ENABLE_DEFAULT,
 431                                     value);
 432                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 433                                     PDE1_PROTECTION_FAULT_ENABLE_DEFAULT,
 434                                     value);
 435                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 436                                     PDE2_PROTECTION_FAULT_ENABLE_DEFAULT,
 437                                     value);
 438                 tmp = REG_SET_FIELD(tmp,
 439                             VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 440                             TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
 441                             value);
 442                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 443                                     NACK_PROTECTION_FAULT_ENABLE_DEFAULT,
 444                                     value);
 445                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 446                                     DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
 447                                     value);
 448                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 449                                     VALID_PROTECTION_FAULT_ENABLE_DEFAULT,
 450                                     value);
 451                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 452                                     READ_PROTECTION_FAULT_ENABLE_DEFAULT,
 453                                     value);
 454                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 455                                     WRITE_PROTECTION_FAULT_ENABLE_DEFAULT,
 456                                     value);
 457                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 458                                     EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT,
 459                                     value);
 460                 if (!value) {
 461                         tmp = REG_SET_FIELD(tmp,
 462                                             VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 463                                             CRASH_ON_NO_RETRY_FAULT, 1);
 464                         tmp = REG_SET_FIELD(tmp,
 465                                             VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 466                                             CRASH_ON_RETRY_FAULT, 1);
 467                 }
 468
 469                 WREG32_SOC15_OFFSET(MMHUB, 0,
 470                                     mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 471                                     i * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 472         }
 473 }
 474
 475 void mmhub_v9_4_init(struct amdgpu_device *adev)
 476 {
 477         struct amdgpu_vmhub *hub[MMHUB_NUM_INSTANCES] =
 478                 {&adev->vmhub[AMDGPU_MMHUB_0], &adev->vmhub[AMDGPU_MMHUB_1]};
 479         int i;
 480
 481         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 482                 hub[i]->ctx0_ptb_addr_lo32 =
 483                         SOC15_REG_OFFSET(MMHUB, 0,
 484                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) +
 485                             i * MMHUB_INSTANCE_REGISTER_OFFSET;
 486                 hub[i]->ctx0_ptb_addr_hi32 =
 487                         SOC15_REG_OFFSET(MMHUB, 0,
 488                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) +
 489                             i * MMHUB_INSTANCE_REGISTER_OFFSET;
 490                 hub[i]->vm_inv_eng0_req =
 491                         SOC15_REG_OFFSET(MMHUB, 0,
 492                                          mmVML2VC0_VM_INVALIDATE_ENG0_REQ) +
 493                                          i * MMHUB_INSTANCE_REGISTER_OFFSET;
 494                 hub[i]->vm_inv_eng0_ack =
 495                         SOC15_REG_OFFSET(MMHUB, 0,
 496                                          mmVML2VC0_VM_INVALIDATE_ENG0_ACK) +
 497                                          i * MMHUB_INSTANCE_REGISTER_OFFSET;
 498                 hub[i]->vm_context0_cntl =
 499                         SOC15_REG_OFFSET(MMHUB, 0,
 500                                          mmVML2VC0_VM_CONTEXT0_CNTL) +
 501                                          i * MMHUB_INSTANCE_REGISTER_OFFSET;
 502                 hub[i]->vm_l2_pro_fault_status =
 503                         SOC15_REG_OFFSET(MMHUB, 0,
 504                                     mmVML2PF0_VM_L2_PROTECTION_FAULT_STATUS) +
 505                                     i * MMHUB_INSTANCE_REGISTER_OFFSET;
 506                 hub[i]->vm_l2_pro_fault_cntl =
 507                         SOC15_REG_OFFSET(MMHUB, 0,
 508                                     mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL) +
 509                                     i * MMHUB_INSTANCE_REGISTER_OFFSET;
 510         }
 511 }