drm/amdgpu: add ih waiter on process until checkpoint

Add IH function to allow caller to wait until ring entries are processed
until the checkpoint write pointer.

This will be primarily used by HMM to drain pending page fault interrupts
before memory unmap to prevent HMM from handling stale interrupts.

Suggested-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
index dc852af4f3b76f6a33bfa0784addd7f80f7c443d..1024065f1f036259f0e12acfaca715b932a0047f 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
@@ -99,6 +99,8 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih,
                ih->rptr_addr = adev->wb.gpu_addr + rptr_offs * 4;
                ih->rptr_cpu = &adev->wb.wb[rptr_offs];
        }
+
+       init_waitqueue_head(&ih->wait_process);
        return 0;
 }
 
@@ -160,6 +162,52 @@ void amdgpu_ih_ring_write(struct amdgpu_ih_ring *ih, const uint32_t *iv,
        }
 }
 
+/* Waiter helper that checks current rptr matches or passes checkpoint wptr */
+static bool amdgpu_ih_has_checkpoint_processed(struct amdgpu_device *adev,
+                                       struct amdgpu_ih_ring *ih,
+                                       uint32_t checkpoint_wptr,
+                                       uint32_t *prev_rptr)
+{
+       uint32_t cur_rptr = ih->rptr | (*prev_rptr & ~ih->ptr_mask);
+
+       /* rptr has wrapped. */
+       if (cur_rptr < *prev_rptr)
+               cur_rptr += ih->ptr_mask + 1;
+       *prev_rptr = cur_rptr;
+
+       return cur_rptr >= checkpoint_wptr;
+}
+
+/**
+ * amdgpu_ih_wait_on_checkpoint_process - wait to process IVs up to checkpoint
+ *
+ * @adev: amdgpu_device pointer
+ * @ih: ih ring to process
+ *
+ * Used to ensure ring has processed IVs up to the checkpoint write pointer.
+ */
+int amdgpu_ih_wait_on_checkpoint_process(struct amdgpu_device *adev,
+                                       struct amdgpu_ih_ring *ih)
+{
+       uint32_t checkpoint_wptr, rptr;
+
+       if (!ih->enabled || adev->shutdown)
+               return -ENODEV;
+
+       checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih);
+       /* Order wptr with rptr. */
+       rmb();
+       rptr = READ_ONCE(ih->rptr);
+
+       /* wptr has wrapped. */
+       if (rptr > checkpoint_wptr)
+               checkpoint_wptr += ih->ptr_mask + 1;
+
+       return wait_event_interruptible(ih->wait_process,
+                               amdgpu_ih_has_checkpoint_processed(adev, ih,
+                                               checkpoint_wptr, &rptr));
+}
+
 /**
  * amdgpu_ih_process - interrupt handler
  *
@@ -195,6 +243,7 @@ restart_ih:
        }
 
        amdgpu_ih_set_rptr(adev, ih);
+       wake_up_all(&ih->wait_process);
        atomic_set(&ih->lock, 0);
 
        /* make sure wptr hasn't changed while processing */

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
index 6ed4a85fc7c31e7916472257dc243a6d2c09afc3..87ec6d20dbe0dbcb713cfe0a3c0785e02c1fa1c5 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
@@ -66,6 +66,9 @@ struct amdgpu_ih_ring {
        unsigned                rptr;
        atomic_t                lock;
        struct amdgpu_ih_regs   ih_regs;
+
+       /* For waiting on IH processing at checkpoint. */
+       wait_queue_head_t wait_process;
 };
 
 /* provided by the ih block */
@@ -87,6 +90,8 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih,
 void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih);
 void amdgpu_ih_ring_write(struct amdgpu_ih_ring *ih, const uint32_t *iv,
                          unsigned int num_dw);
+int amdgpu_ih_wait_on_checkpoint_process(struct amdgpu_device *adev,
+                                       struct amdgpu_ih_ring *ih);
 int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih);
 void amdgpu_ih_decode_iv_helper(struct amdgpu_device *adev,
                                struct amdgpu_ih_ring *ih,