drm/amdgpu: Update invalid PTE flag setting

Update the invalid PTE flag setting with TF enabled.
This is to ensure, in addition to transitioning the
retry fault to a no-retry fault, it also causes the
wavefront to enter the trap handler. With the current
setting, the fault only transitions to a no-retry fault.
Additionally, have 2 sets of invalid PTE settings, one for
TF enabled, the other for TF disabled. The setting with
TF disabled, doesn't work with TF enabled.

Signed-off-by: Mukul Joshi <mukul.joshi@amd.com>
Acked-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 56d73fa..fdc25cd 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -331,6 +331,8 @@ struct amdgpu_gmc {
        u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32[16];
        u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32[16];
        u64 MC_VM_MX_L1_TLB_CNTL;
+
+       u64 noretry_flags;
 };
 
 #define amdgpu_gmc_flush_gpu_tlb(adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type)))

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 291977b..8eda8f7 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -2604,7 +2604,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
                /* Intentionally setting invalid PTE flag
                 * combination to force a no-retry-fault
                 */
-               flags = AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT;
+               flags = AMDGPU_VM_NORETRY_FLAGS;
                value = 0;
        } else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
                /* Redirect the access to the dummy page */

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 9c85d49..b81fcb9 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -84,7 +84,13 @@ struct amdgpu_mem_stats;
 /* PDE Block Fragment Size for VEGA10 */
 #define AMDGPU_PDE_BFS(a)      ((uint64_t)a << 59)
 
+/* Flag combination to set no-retry with TF disabled */
+#define AMDGPU_VM_NORETRY_FLAGS        (AMDGPU_PTE_EXECUTABLE | AMDGPU_PDE_PTE | \
+                               AMDGPU_PTE_TF)
 
+/* Flag combination to set no-retry with TF enabled */
+#define AMDGPU_VM_NORETRY_FLAGS_TF (AMDGPU_PTE_VALID | AMDGPU_PTE_SYSTEM | \
+                                  AMDGPU_PTE_PRT)
 /* For GFX9 */
 #define AMDGPU_PTE_MTYPE_VG10(a)       ((uint64_t)(a) << 57)
 #define AMDGPU_PTE_MTYPE_VG10_MASK     AMDGPU_PTE_MTYPE_VG10(3ULL)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index dea1a64..24ddf6a 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -778,6 +778,27 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
                                        1, 0, flags);
 }
 
+/**
+ * amdgpu_vm_pte_update_noretry_flags - Update PTE no-retry flags
+ *
+ * @adev - amdgpu_device pointer
+ * @flags: pointer to PTE flags
+ *
+ * Update PTE no-retry flags when TF is enabled.
+ */
+static void amdgpu_vm_pte_update_noretry_flags(struct amdgpu_device *adev,
+                                               uint64_t *flags)
+{
+       /*
+        * Update no-retry flags with the corresponding TF
+        * no-retry combination.
+        */
+       if ((*flags & AMDGPU_VM_NORETRY_FLAGS) == AMDGPU_VM_NORETRY_FLAGS) {
+               *flags &= ~AMDGPU_VM_NORETRY_FLAGS;
+               *flags |= adev->gmc.noretry_flags;
+       }
+}
+
 /*
  * amdgpu_vm_pte_update_flags - figure out flags for PTE updates
  *
@@ -804,6 +825,16 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
                flags |= AMDGPU_PTE_EXECUTABLE;
        }
 
+       /*
+        * Update no-retry flags to use the no-retry flag combination
+        * with TF enabled. The AMDGPU_VM_NORETRY_FLAGS flag combination
+        * does not work when TF is enabled. So, replace them with
+        * AMDGPU_VM_NORETRY_FLAGS_TF flag combination which works for
+        * all cases.
+        */
+       if (level == AMDGPU_VM_PTB)
+               amdgpu_vm_pte_update_noretry_flags(adev, &flags);
+
        /* APUs mapping system memory may need different MTYPEs on different
         * NUMA nodes. Only do this for contiguous ranges that can be assumed
         * to be on the same NUMA node.

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index 0c8a479..13b89f7 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -751,6 +751,7 @@ static int gmc_v10_0_early_init(void *handle)
        adev->gmc.private_aperture_start = 0x1000000000000000ULL;
        adev->gmc.private_aperture_end =
                adev->gmc.private_aperture_start + (4ULL << 30) - 1;
+       adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
 
        return 0;
 }

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
index c571f0d..c68ecb7 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
@@ -651,6 +651,7 @@ static int gmc_v11_0_early_init(void *handle)
        adev->gmc.private_aperture_start = 0x1000000000000000ULL;
        adev->gmc.private_aperture_end =
                adev->gmc.private_aperture_start + (4ULL << 30) - 1;
+       adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
 
        return 0;
 }

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index acd2b40..027b146 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -942,6 +942,7 @@ static int gmc_v7_0_early_init(void *handle)
                adev->gmc.shared_aperture_end + 1;
        adev->gmc.private_aperture_end =
                adev->gmc.private_aperture_start + (4ULL << 30) - 1;
+       adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
 
        return 0;
 }

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 85dead2..581ed92 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -1056,6 +1056,7 @@ static int gmc_v8_0_early_init(void *handle)
                adev->gmc.shared_aperture_end + 1;
        adev->gmc.private_aperture_end =
                adev->gmc.private_aperture_start + (4ULL << 30) - 1;
+       adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
 
        return 0;
 }

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 67e669e..ebdbc82 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -1622,6 +1622,7 @@ static int gmc_v9_0_early_init(void *handle)
        adev->gmc.private_aperture_start = 0x1000000000000000ULL;
        adev->gmc.private_aperture_end =
                adev->gmc.private_aperture_start + (4ULL << 30) - 1;
+       adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
 
        return 0;
 }