drm/amdgpu: Fix the race condition for draining retry fault

[ Upstream commit f844732e3ad9c4b78df7436232949b8d2096d1a6 ]

Issue:
In the scenario where svm_range_restore_pages is called, but
svm->checkpoint_ts has not been set and the retry fault has not been
drained, svm_range_unmap_from_cpu is triggered and calls svm_range_free.
Meanwhile, svm_range_restore_pages continues execution and reaches
svm_range_from_addr. This results in a "failed to find prange..." error,
 causing the page recovery to fail.

How to fix:
Move the timestamp check code under the protection of svm->lock.

v2:
Make sure all right locks are released before go out.

v3:
Directly goto out_unlock_svms, and return -EAGAIN.

v4:
Refine code.

Signed-off-by: Emily Deng <Emily.Deng@amd.com>
Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 8c61dee5ca0db10668b1d73ad1979dc66d62e5a3..b50283864dcd26d140009434d71c0aa3fec8b459 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -2992,19 +2992,6 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
                goto out;
        }
 
-       /* check if this page fault time stamp is before svms->checkpoint_ts */
-       if (svms->checkpoint_ts[gpuidx] != 0) {
-               if (amdgpu_ih_ts_after(ts,  svms->checkpoint_ts[gpuidx])) {
-                       pr_debug("draining retry fault, drop fault 0x%llx\n", addr);
-                       r = 0;
-                       goto out;
-               } else
-                       /* ts is after svms->checkpoint_ts now, reset svms->checkpoint_ts
-                        * to zero to avoid following ts wrap around give wrong comparing
-                        */
-                       svms->checkpoint_ts[gpuidx] = 0;
-       }
-
        if (!p->xnack_enabled) {
                pr_debug("XNACK not enabled for pasid 0x%x\n", pasid);
                r = -EFAULT;
@@ -3024,6 +3011,21 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
        mmap_read_lock(mm);
 retry_write_locked:
        mutex_lock(&svms->lock);
+
+       /* check if this page fault time stamp is before svms->checkpoint_ts */
+       if (svms->checkpoint_ts[gpuidx] != 0) {
+               if (amdgpu_ih_ts_after(ts,  svms->checkpoint_ts[gpuidx])) {
+                       pr_debug("draining retry fault, drop fault 0x%llx\n", addr);
+                       r = -EAGAIN;
+                       goto out_unlock_svms;
+               } else {
+                       /* ts is after svms->checkpoint_ts now, reset svms->checkpoint_ts
+                        * to zero to avoid following ts wrap around give wrong comparing
+                        */
+                       svms->checkpoint_ts[gpuidx] = 0;
+               }
+       }
+
        prange = svm_range_from_addr(svms, addr, NULL);
        if (!prange) {
                pr_debug("failed to find prange svms 0x%p address [0x%llx]\n",
@@ -3148,7 +3150,8 @@ out_unlock_svms:
        mutex_unlock(&svms->lock);
        mmap_read_unlock(mm);
 
-       svm_range_count_fault(node, p, gpuidx);
+       if (r != -EAGAIN)
+               svm_range_count_fault(node, p, gpuidx);
 
        mmput(mm);
 out: