freedreno: Add fd_replace_buffer_storage()

This will be used by threaded_context to avoid stalls in the
DISCARD_WHOLE_RESOURCE case (and DISCARD_RANGE cases that can
be promoted to DISCARD_WHOLE_RESOURCE).

Signed-off-by: Rob Clark <robdclark@chromium.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/9323>

diff --git a/src/gallium/drivers/freedreno/freedreno_resource.c b/src/gallium/drivers/freedreno/freedreno_resource.c
index d2115bf..649532f 100644 (file)
--- a/src/gallium/drivers/freedreno/freedreno_resource.c
+++ b/src/gallium/drivers/freedreno/freedreno_resource.c
@@ -246,6 +246,54 @@ do_blit(struct fd_context *ctx, const struct pipe_blit_info *blit, bool fallback
        }
 }
 
+/**
+ * Replace the storage of dst with src.  This is only used by TC in the
+ * DISCARD_WHOLE_RESOURCE path, and src is a freshly allocated buffer.
+ */
+void
+fd_replace_buffer_storage(struct pipe_context *pctx, struct pipe_resource *pdst,
+               struct pipe_resource *psrc)
+{
+       struct fd_context  *ctx = fd_context(pctx);
+       struct fd_resource *dst = fd_resource(pdst);
+       struct fd_resource *src = fd_resource(psrc);
+
+       DBG("pdst=%p, psrc=%p", pdst, psrc);
+
+       /* This should only be called with buffers.. which side-steps some tricker
+        * cases, like a rsc that is in a batch-cache key...
+        */
+       assert(pdst->target == PIPE_BUFFER);
+       assert(psrc->target == PIPE_BUFFER);
+       assert(dst->track->bc_batch_mask == 0);
+       assert(src->track->bc_batch_mask == 0);
+       assert(src->track->batch_mask == 0);
+       assert(src->track->write_batch == NULL);
+       assert(memcmp(&dst->layout, &src->layout, sizeof(dst->layout)) == 0);
+
+       /* get rid of any references that batch-cache might have to us (which
+        * should empty/destroy rsc->batches hashset)
+        *
+        * Note that we aren't actually destroying dst, but we are replacing
+        * it's storage so we want to go thru the same motions of decoupling
+        * it's batch connections.
+        */
+       fd_bc_invalidate_resource(dst, true);
+       rebind_resource(dst);
+
+       fd_screen_lock(ctx->screen);
+
+       fd_bo_del(dst->bo);
+       dst->bo = fd_bo_ref(src->bo);
+
+       fd_resource_tracking_reference(&dst->track, src->track);
+       src->is_replacement = true;
+
+       dst->seqno = p_atomic_inc_return(&ctx->screen->rsc_seqno);
+
+       fd_screen_unlock(ctx->screen);
+}
+
 static void
 flush_resource(struct fd_context *ctx, struct fd_resource *rsc, unsigned usage);
 
@@ -918,7 +966,9 @@ fd_resource_destroy(struct pipe_screen *pscreen,
                struct pipe_resource *prsc)
 {
        struct fd_resource *rsc = fd_resource(prsc);
-       fd_bc_invalidate_resource(rsc, true);
+
+       if (!rsc->is_replacement)
+               fd_bc_invalidate_resource(rsc, true);
        if (rsc->bo)
                fd_bo_del(rsc->bo);
        if (rsc->lrz)

diff --git a/src/gallium/drivers/freedreno/freedreno_resource.h b/src/gallium/drivers/freedreno/freedreno_resource.h
index 1e24204..0982b2f 100644 (file)
--- a/src/gallium/drivers/freedreno/freedreno_resource.h
+++ b/src/gallium/drivers/freedreno/freedreno_resource.h
@@ -142,6 +142,13 @@ struct fd_resource {
        /* Sequence # incremented each time bo changes: */
        uint16_t seqno;
 
+       /* Is this buffer a replacement created by threaded_context to avoid
+        * a stall in PIPE_MAP_DISCARD_WHOLE_RESOURCE|PIPE_MAP_WRITE case?
+        * If so, it no longer "owns" it's rsc->track, and so should not
+        * invalidate when the rsc is destroyed.
+        */
+       bool is_replacement : 1;
+
        /* Uninitialized resources with UBWC format need their UBWC flag data
         * cleared before writes, as the UBWC state is read and used during
         * writes, so undefined UBWC flag data results in undefined results.
@@ -335,6 +342,8 @@ void fd_resource_context_init(struct pipe_context *pctx);
 
 uint32_t fd_setup_slices(struct fd_resource *rsc);
 void fd_resource_resize(struct pipe_resource *prsc, uint32_t sz);
+void fd_replace_buffer_storage(struct pipe_context *ctx, struct pipe_resource *dst,
+               struct pipe_resource *src) in_dt;
 void fd_resource_uncompress(struct fd_context *ctx, struct fd_resource *rsc) assert_dt;
 void fd_resource_dump(struct fd_resource *rsc, const char *name);