nouveau: eliminate busy waiting on fences

Signed-off-by: Karol Herbst <kherbst@redhat.com>
Reviewed-by: Adam Jackson <ajax@redhat.com>
Reviewed-by: Emma Anholt <emma@anholt.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19543>

diff --git a/src/gallium/drivers/nouveau/nouveau_fence.c b/src/gallium/drivers/nouveau/nouveau_fence.c
index 9fad60c..450174f 100644 (file)
--- a/src/gallium/drivers/nouveau/nouveau_fence.c
+++ b/src/gallium/drivers/nouveau/nouveau_fence.c
@@ -40,6 +40,12 @@ nouveau_fence_new(struct nouveau_context *nv, struct nouveau_fence **fence)
    if (!*fence)
       return false;
 
+   int ret = nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART, 0x1000, 0x1000, NULL, &(*fence)->bo);
+   if (ret) {
+      FREE(*fence);
+      return false;
+   }
+
    (*fence)->screen = nv->screen;
    (*fence)->context = nv;
    (*fence)->ref = 1;
@@ -86,7 +92,7 @@ _nouveau_fence_emit(struct nouveau_fence *fence)
 
    fence_list->tail = fence;
 
-   fence_list->emit(&fence->context->pipe, &fence->sequence);
+   fence_list->emit(&fence->context->pipe, &fence->sequence, fence->bo);
 
    assert(fence->state == NOUVEAU_FENCE_STATE_EMITTING);
    fence->state = NOUVEAU_FENCE_STATE_EMITTED;
@@ -119,6 +125,7 @@ nouveau_fence_del(struct nouveau_fence *fence)
       nouveau_fence_trigger_work(fence);
    }
 
+   nouveau_bo_ref(NULL, &fence->bo);
    FREE(fence);
 }
 
@@ -239,7 +246,6 @@ _nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *deb
 {
    struct nouveau_screen *screen = fence->screen;
    struct nouveau_fence_list *fence_list = &screen->fence;
-   uint32_t spins = 0;
    int64_t start = 0;
 
    simple_mtx_assert_locked(&fence_list->lock);
@@ -250,30 +256,27 @@ _nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *deb
    if (!nouveau_fence_kick(fence))
       return false;
 
-   do {
-      if (fence->state == NOUVEAU_FENCE_STATE_SIGNALLED) {
-         if (debug && debug->debug_message)
-            util_debug_message(debug, PERF_INFO,
-                               "stalled %.3f ms waiting for fence",
-                               (os_time_get_nano() - start) / 1000000.f);
-         return true;
+   if (fence->state < NOUVEAU_FENCE_STATE_SIGNALLED) {
+      NOUVEAU_DRV_STAT(screen, any_non_kernel_fence_sync_count, 1);
+      int ret = nouveau_bo_wait(fence->bo, NOUVEAU_BO_RDWR, screen->client);
+      if (ret) {
+         debug_printf("Wait on fence %u (ack = %u, next = %u) errored with %s !\n",
+                      fence->sequence,
+                      fence_list->sequence_ack, fence_list->sequence, strerror(ret));
+         return false;
       }
-      if (!spins)
-         NOUVEAU_DRV_STAT(screen, any_non_kernel_fence_sync_count, 1);
-      spins++;
-#if DETECT_OS_UNIX
-      if (!(spins % 8)) /* donate a few cycles */
-         sched_yield();
-#endif
 
       _nouveau_fence_update(screen, false);
-   } while (spins < NOUVEAU_FENCE_MAX_SPINS);
+      if (fence->state != NOUVEAU_FENCE_STATE_SIGNALLED)
+         return false;
 
-   debug_printf("Wait on fence %u (ack = %u, next = %u) timed out !\n",
-                fence->sequence,
-                fence_list->sequence_ack, fence_list->sequence);
+      if (debug && debug->debug_message)
+         util_debug_message(debug, PERF_INFO,
+                            "stalled %.3f ms waiting for fence",
+                            (os_time_get_nano() - start) / 1000000.f);
+   }
 
-   return false;
+   return true;
 }
 
 void

diff --git a/src/gallium/drivers/nouveau/nouveau_fence.h b/src/gallium/drivers/nouveau/nouveau_fence.h
index 384084b..711beac 100644 (file)
--- a/src/gallium/drivers/nouveau/nouveau_fence.h
+++ b/src/gallium/drivers/nouveau/nouveau_fence.h
@@ -24,6 +24,7 @@ struct nouveau_fence {
    struct nouveau_fence *next;
    struct nouveau_screen *screen;
    struct nouveau_context *context;
+   struct nouveau_bo *bo;
    int state;
    int ref;
    uint32_t sequence;
@@ -37,7 +38,7 @@ struct nouveau_fence_list {
    uint32_t sequence;
    uint32_t sequence_ack;
    simple_mtx_t lock;
-   void (*emit)(struct pipe_context *, uint32_t *sequence);
+   void (*emit)(struct pipe_context *, uint32_t *sequence, struct nouveau_bo *wait);
    uint32_t (*update)(struct pipe_screen *);
 };
 

diff --git a/src/gallium/drivers/nouveau/nv30/nv30_screen.c b/src/gallium/drivers/nouveau/nv30/nv30_screen.c
index 78597f7..c65dc50 100644 (file)
--- a/src/gallium/drivers/nouveau/nv30/nv30_screen.c
+++ b/src/gallium/drivers/nouveau/nv30/nv30_screen.c
@@ -510,11 +510,13 @@ nv30_screen_get_compiler_options(struct pipe_screen *pscreen,
 }
 
 static void
-nv30_screen_fence_emit(struct pipe_context *pcontext, uint32_t *sequence)
+nv30_screen_fence_emit(struct pipe_context *pcontext, uint32_t *sequence,
+                       struct nouveau_bo *wait)
 {
    struct nv30_context *nv30 = nv30_context(pcontext);
    struct nv30_screen *screen = nv30->screen;
    struct nouveau_pushbuf *push = nv30->base.pushbuf;
+   struct nouveau_pushbuf_refn ref = { wait, NOUVEAU_BO_GART | NOUVEAU_BO_RDWR };
 
    *sequence = ++screen->base.fence.sequence;
 
@@ -523,6 +525,8 @@ nv30_screen_fence_emit(struct pipe_context *pcontext, uint32_t *sequence)
               (2 /* size */ << 18) | (7 /* subchan */ << 13));
    PUSH_DATA (push, 0);
    PUSH_DATA (push, *sequence);
+
+   nouveau_pushbuf_refn(push, &ref, 1);
 }
 
 static uint32_t

diff --git a/src/gallium/drivers/nouveau/nv50/nv50_screen.c b/src/gallium/drivers/nouveau/nv50/nv50_screen.c
index 86b21b1..8083f2a 100644 (file)
--- a/src/gallium/drivers/nouveau/nv50/nv50_screen.c
+++ b/src/gallium/drivers/nouveau/nv50/nv50_screen.c
@@ -512,11 +512,13 @@ nv50_screen_destroy(struct pipe_screen *pscreen)
 }
 
 static void
-nv50_screen_fence_emit(struct pipe_context *pcontext, u32 *sequence)
+nv50_screen_fence_emit(struct pipe_context *pcontext, u32 *sequence,
+                       struct nouveau_bo *wait)
 {
    struct nv50_context *nv50 = nv50_context(pcontext);
    struct nv50_screen *screen = nv50->screen;
    struct nouveau_pushbuf *push = nv50->base.pushbuf;
+   struct nouveau_pushbuf_refn ref = { wait, NOUVEAU_BO_GART | NOUVEAU_BO_RDWR };
 
    /* we need to do it after possible flush in MARK_RING */
    *sequence = ++screen->base.fence.sequence;
@@ -532,6 +534,8 @@ nv50_screen_fence_emit(struct pipe_context *pcontext, u32 *sequence)
                     NV50_3D_QUERY_GET_TYPE_QUERY |
                     NV50_3D_QUERY_GET_QUERY_SELECT_ZERO |
                     NV50_3D_QUERY_GET_SHORT);
+
+   nouveau_pushbuf_refn(push, &ref, 1);
 }
 
 static u32

diff --git a/src/gallium/drivers/nouveau/nvc0/nvc0_screen.c b/src/gallium/drivers/nouveau/nvc0/nvc0_screen.c
index 2788954..746a9df 100644 (file)
--- a/src/gallium/drivers/nouveau/nvc0/nvc0_screen.c
+++ b/src/gallium/drivers/nouveau/nvc0/nvc0_screen.c
@@ -756,11 +756,13 @@ nvc0_magic_3d_init(struct nouveau_pushbuf *push, uint16_t obj_class)
 }
 
 static void
-nvc0_screen_fence_emit(struct pipe_context *pcontext, u32 *sequence)
+nvc0_screen_fence_emit(struct pipe_context *pcontext, u32 *sequence,
+                       struct nouveau_bo *wait)
 {
    struct nvc0_context *nvc0 = nvc0_context(pcontext);
    struct nvc0_screen *screen = nvc0->screen;
    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
+   struct nouveau_pushbuf_refn ref = { wait, NOUVEAU_BO_GART | NOUVEAU_BO_RDWR };
 
    /* we need to do it after possible flush in MARK_RING */
    *sequence = ++screen->base.fence.sequence;
@@ -772,6 +774,8 @@ nvc0_screen_fence_emit(struct pipe_context *pcontext, u32 *sequence)
    PUSH_DATA (push, *sequence);
    PUSH_DATA (push, NVC0_3D_QUERY_GET_FENCE | NVC0_3D_QUERY_GET_SHORT |
               (0xf << NVC0_3D_QUERY_GET_UNIT__SHIFT));
+
+   nouveau_pushbuf_refn(push, &ref, 1);
 }
 
 static u32