historically zink has been bound to 4 gfx batches and then a separate compute batch
was added. this is not ideal for a number of reasons, the primary one being that if
an application performs 5 glFlush commands, the fifth one will force a gpu stall
this patch aims to do the following, all of which are necessarily done in the same patch
because they can't be added incrementally and still have the same function:
* rewrite batch tracking for resources/views/queries/descriptors/...
|originally this was done with a single uint32_t as a bitmask, but that becomes cumbersome
to track as batch counts increase, not to mention it becomes doubly-annoying
when factoring in separate compute batches with their own ids. zink_batch_usage gives
us separate tracking for gfx and compute batches along with a standardized api for
managing usage
* flatten batch objects to a gfx batch and a compute batch
|these are separate queues, so we can use an enum to choose between an array[2] of
all batch-related objects
* switch to monotonic batch ids with batch "states"
|with the flattened queues, we can just use monotonic uints to represent batch ids,
thus freeing us from constantly using bitfield operations here and also enabling
batch counts to scale dynamically by allocating/caching "states" that represent a batch
for a given queue
Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/9547>
#include "wsi_common.h"
+static void
+batch_usage_unset(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id)
+{
+ p_atomic_cmpxchg(&u->usage[queue], batch_id, 0);
+}
+
void
zink_batch_state_clear_resources(struct zink_screen *screen, struct zink_batch_state *bs)
{
/* unref all used resources */
set_foreach(bs->resources, entry) {
struct zink_resource_object *obj = (struct zink_resource_object *)entry->key;
+ batch_usage_unset(&obj->reads, !!bs->is_compute, bs->batch_id);
+ batch_usage_unset(&obj->writes, !!bs->is_compute, bs->batch_id);
zink_resource_object_reference(screen, &obj, NULL);
_mesa_set_remove(bs->resources, entry);
}
zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs)
{
struct zink_screen *screen = zink_screen(ctx->base.screen);
- if (bs->fence->submitted)
- zink_fence_finish(screen, &ctx->base, bs->fence, PIPE_TIMEOUT_INFINITE);
zink_batch_state_clear_resources(screen, bs);
set_foreach(bs->surfaces, entry) {
struct zink_surface *surf = (struct zink_surface *)entry->key;
- surf->batch_uses &= ~BITFIELD64_BIT(bs->batch_id);
+ batch_usage_unset(&surf->batch_uses, !!bs->is_compute, bs->batch_id);
pipe_surface_reference((struct pipe_surface**)&surf, NULL);
_mesa_set_remove(bs->surfaces, entry);
}
set_foreach(bs->bufferviews, entry) {
struct zink_buffer_view *buffer_view = (struct zink_buffer_view *)entry->key;
- buffer_view->batch_uses &= ~BITFIELD64_BIT(bs->batch_id);
+ batch_usage_unset(&buffer_view->batch_uses, !!bs->is_compute, bs->batch_id);
zink_buffer_view_reference(screen, &buffer_view, NULL);
_mesa_set_remove(bs->bufferviews, entry);
}
set_foreach(bs->desc_sets, entry) {
struct zink_descriptor_set *zds = (void*)entry->key;
- zds->batch_uses &= ~BITFIELD_BIT(bs->batch_id);
+ batch_usage_unset(&zds->batch_uses, !!bs->is_compute, bs->batch_id);
/* reset descriptor pools when no bs is using this program to avoid
* having some inactive program hogging a billion descriptors
*/
bs->flush_res = NULL;
bs->descs_used = 0;
- bs->fence->submitted = false;
+ ctx->resource_size[bs->is_compute] -= bs->resource_size;
bs->resource_size = 0;
}
void
+zink_batch_reset_all(struct zink_context *ctx, enum zink_queue queue)
+{
+ hash_table_foreach(&ctx->batch_states[queue], entry) {
+ struct zink_batch_state *bs = entry->data;
+ zink_reset_batch_state(ctx, bs);
+ _mesa_hash_table_remove(&ctx->batch_states[queue], entry);
+ util_dynarray_append(&ctx->free_batch_states[queue], struct zink_batch_state *, bs);
+ }
+}
+
+void
zink_batch_state_destroy(struct zink_screen *screen, struct zink_batch_state *bs)
{
if (!bs)
}
static struct zink_batch_state *
-create_batch_state(struct zink_context *ctx, unsigned idx)
+create_batch_state(struct zink_context *ctx, enum zink_queue queue)
{
struct zink_screen *screen = zink_screen(ctx->base.screen);
struct zink_batch_state *bs = rzalloc(NULL, struct zink_batch_state);
SET_CREATE_OR_FAIL(bs->active_queries);
util_dynarray_init(&bs->zombie_samplers, NULL);
util_dynarray_init(&bs->persistent_resources, NULL);
- bs->batch_id = idx;
if (!zink_create_fence(screen, bs))
/* this destroys the batch state on failure */
return NULL;
- bs->is_compute = idx == ZINK_COMPUTE_BATCH_ID;
+ bs->is_compute = queue == ZINK_QUEUE_COMPUTE;
return bs;
fail:
return NULL;
}
+static bool
+find_unused_state(struct hash_entry *entry)
+{
+ struct zink_fence *fence = entry->data;
+ /* we can't reset these from fence_finish because threads */
+ bool submitted = p_atomic_read(&fence->submitted);
+ return !submitted;
+}
+
static void
init_batch_state(struct zink_context *ctx, struct zink_batch *batch)
{
- struct zink_batch_state *bs = create_batch_state(ctx, batch->batch_id);
+ struct zink_batch_state *bs = NULL;
+
+ if (util_dynarray_num_elements(&ctx->free_batch_states[batch->queue], struct zink_batch_state*))
+ bs = util_dynarray_pop(&ctx->free_batch_states[batch->queue], struct zink_batch_state*);
+ if (!bs) {
+ struct hash_entry *he = _mesa_hash_table_random_entry(&ctx->batch_states[batch->queue], find_unused_state);
+ if (he) { //there may not be any entries available
+ bs = he->data;
+ _mesa_hash_table_remove(&ctx->batch_states[batch->queue], he);
+ zink_reset_batch_state(ctx, bs);
+ }
+ }
+ if (!bs) {
+ if (!batch->state) {
+ /* this is batch init, so create a few more states for later use */
+ for (int i = 0; i < 3; i++) {
+ struct zink_batch_state *state = create_batch_state(ctx, batch->queue);
+ util_dynarray_append(&ctx->free_batch_states[batch->queue], struct zink_batch_state *, state);
+ }
+ }
+ bs = create_batch_state(ctx, batch->queue);
+ }
batch->state = bs;
}
if (vkBeginCommandBuffer(batch->state->cmdbuf, &cbbi) != VK_SUCCESS)
debug_printf("vkBeginCommandBuffer failed\n");
+ batch->state->batch_id = ctx->curr_batch;
+ if (ctx->last_fence[batch->queue]) {
+ struct zink_batch_state *last_state = zink_batch_state(ctx->last_fence[batch->queue]);
+ batch->last_batch_id = last_state->batch_id;
+ }
if (!ctx->queries_disabled)
zink_resume_queries(ctx, batch);
}
si.pNext = &mem_signal;
}
- if (vkQueueSubmit(ctx->queue, 1, &si, batch->state->fence->fence) != VK_SUCCESS) {
+ if (vkQueueSubmit(ctx->queue, 1, &si, batch->state->fence.fence) != VK_SUCCESS) {
debug_printf("ZINK: vkQueueSubmit() failed\n");
ctx->is_device_lost = true;
ctx->reset.reset(ctx->reset.data, PIPE_GUILTY_CONTEXT_RESET);
}
}
- batch->state->fence->submitted = true;
+
+ ctx->last_fence[batch->queue] = &batch->state->fence;
+ _mesa_hash_table_insert_pre_hashed(&ctx->batch_states[batch->queue], batch->state->batch_id, (void*)(uintptr_t)batch->state->batch_id, batch->state);
+ ctx->resource_size[batch->queue] += batch->state->resource_size;
}
/* returns a queue based on whether a resource
enum zink_queue
zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource *res, bool write)
{
- unsigned mask = write ? ZINK_RESOURCE_ACCESS_WRITE : ZINK_RESOURCE_ACCESS_READ;
enum zink_queue batch_to_flush = 0;
/* u_transfer_helper unrefs the stencil buffer when the depth buffer is unrefed,
zink_get_depth_stencil_resources((struct pipe_resource*)res, NULL, &stencil);
- if (batch->batch_id == ZINK_COMPUTE_BATCH_ID) {
+ if (batch->queue == ZINK_QUEUE_COMPUTE) {
if ((write && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_RW, ZINK_QUEUE_GFX)) ||
(!write && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_WRITE, ZINK_QUEUE_GFX)))
batch_to_flush = ZINK_QUEUE_GFX;
}
/* if the resource already has usage of any sort set for this batch, we can skip hashing */
- if (!zink_resource_has_usage_for_id(res, batch->state->batch_id)) {
+ if (!zink_batch_usage_matches(&res->obj->reads, batch->queue, batch->state->batch_id) &&
+ !zink_batch_usage_matches(&res->obj->writes, batch->queue, batch->state->batch_id)) {
bool found = false;
_mesa_set_search_and_add(batch->state->resources, res->obj, &found);
if (!found) {
pipe_reference(NULL, &res->obj->reference);
- batch->state->resource_size += res->obj->size;
+ if (!batch->last_batch_id || !zink_batch_usage_matches(&res->obj->reads, batch->queue, batch->last_batch_id))
+ /* only add resource usage if it's "new" usage, though this only checks the most recent usage
+ * and not all pending usages
+ */
+ batch->state->resource_size += res->obj->size;
if (stencil) {
pipe_reference(NULL, &stencil->obj->reference);
- batch->state->resource_size += stencil->obj->size;
+ if (!batch->last_batch_id || !zink_batch_usage_matches(&stencil->obj->reads, batch->queue, batch->last_batch_id))
+ batch->state->resource_size += stencil->obj->size;
}
}
+ }
+ if (write) {
+ if (stencil)
+ zink_batch_usage_set(&stencil->obj->writes, batch->queue, batch->state->batch_id);
+ zink_batch_usage_set(&res->obj->writes, batch->queue, batch->state->batch_id);
+ } else {
+ if (stencil)
+ zink_batch_usage_set(&stencil->obj->reads, batch->queue, batch->state->batch_id);
+ zink_batch_usage_set(&res->obj->reads, batch->queue, batch->state->batch_id);
}
/* multiple array entries are fine */
if (res->obj->persistent_maps)
util_dynarray_append(&batch->state->persistent_resources, struct zink_resource*, res);
- /* the batch_uses value for this batch is guaranteed to not be in use now because
- * zink_reset_batch() waits on the fence and removes access before resetting
- */
- res->obj->batch_uses[batch->batch_id] |= mask;
-
- if (stencil)
- stencil->obj->batch_uses[batch->batch_id] |= mask;
batch->has_work = true;
return batch_to_flush;
}
static bool
-ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, uint32_t *u)
+ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage *u)
{
bool found = false;
- uint32_t bit = BITFIELD_BIT(batch->state->batch_id);
- if ((*u) & bit)
+ if (zink_batch_usage_matches(u, batch->queue, batch->state->batch_id))
return false;
_mesa_set_search_and_add(s, ptr, &found);
assert(!found);
- *u |= bit;
+ zink_batch_usage_set(u, batch->queue, batch->state->batch_id);
return true;
}
}
batch->has_work = true;
}
+
+void
+zink_batch_usage_set(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id)
+{
+ if (queue == ZINK_QUEUE_ANY) {
+ p_atomic_set(&u->usage[ZINK_QUEUE_GFX], batch_id);
+ p_atomic_set(&u->usage[ZINK_QUEUE_COMPUTE], batch_id);
+ } else
+ p_atomic_set(&u->usage[queue], batch_id);
+}
+
+bool
+zink_batch_usage_matches(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id)
+{
+ if (queue < ZINK_QUEUE_ANY) {
+ uint32_t usage = p_atomic_read(&u->usage[queue]);
+ return usage == batch_id;
+ }
+ for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++) {
+ uint32_t usage = p_atomic_read(&u->usage[queue]);
+ if (usage == batch_id)
+ return true;
+ }
+ return false;
+}
+
+bool
+zink_batch_usage_exists(struct zink_batch_usage *u)
+{
+ uint32_t usage = p_atomic_read(&u->usage[ZINK_QUEUE_GFX]);
+ if (usage)
+ return true;
+ usage = p_atomic_read(&u->usage[ZINK_QUEUE_COMPUTE]);
+ return !!usage;
+}
#include "util/list.h"
#include "util/u_dynarray.h"
+#include "zink_fence.h"
+
struct pipe_reference;
struct zink_context;
struct zink_descriptor_set;
-struct zink_fence;
struct zink_framebuffer;
struct zink_image_view;
struct zink_program;
struct zink_render_pass;
struct zink_resource;
struct zink_sampler_view;
-struct zink_screen;
struct zink_surface;
enum zink_queue {
ZINK_QUEUE_ANY,
};
+struct zink_batch_usage {
+ /* this has to be atomic for fence access, so we can't use a bitmask and make everything neat */
+ uint32_t usage[2]; //gfx, compute
+};
+
struct zink_batch_state {
- unsigned batch_id : 3;
+ struct zink_fence fence;
VkCommandPool cmdpool;
VkCommandBuffer cmdbuf;
struct zink_resource *flush_res;
unsigned short descs_used; //number of descriptors currently allocated
- struct zink_fence *fence;
struct set *fbs;
struct set *programs;
VkDeviceSize resource_size;
+ uint32_t batch_id;
bool is_compute;
};
struct zink_batch {
- unsigned batch_id : 3;
struct zink_batch_state *state;
enum zink_queue queue;
+ uint32_t last_batch_id;
+
bool has_work;
bool in_rp; //renderpass is currently active
};
+static inline struct zink_batch_state *
+zink_batch_state(struct zink_fence *fence)
+{
+ return (struct zink_batch_state *)fence;
+}
+
void
zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs);
void
+zink_batch_reset_all(struct zink_context *ctx, enum zink_queue queue);
+
+void
zink_batch_state_destroy(struct zink_screen *screen, struct zink_batch_state *bs);
void
bool
zink_batch_add_desc_set(struct zink_batch *batch, struct zink_descriptor_set *zds);
+void
+zink_batch_usage_set(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id);
+bool
+zink_batch_usage_matches(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id);
+bool
+zink_batch_usage_exists(struct zink_batch_usage *u);
#endif
{
struct zink_context *ctx = zink_context(pctx);
struct pipe_framebuffer_state *fb = &ctx->fb_state;
- struct zink_batch *batch = zink_curr_batch(ctx);
+ struct zink_batch *batch = zink_batch_g(ctx);
bool needs_rp = false;
if (scissor_state) {
struct pipe_screen *pscreen = pctx->screen;
struct u_rect region = zink_rect_from_box(box);
bool needs_rp = !zink_blit_region_fills(region, pres->width0, pres->height0) || ctx->render_condition_active;
- struct zink_batch *batch = zink_curr_batch(ctx);
+ struct zink_batch *batch = zink_batch_g(ctx);
struct pipe_surface *surf = NULL;
if (res->aspect & VK_IMAGE_ASPECT_COLOR_BIT) {
if (!fb_clear->enabled)
return;
if (zink_resource(pres)->aspect == VK_IMAGE_ASPECT_COLOR_BIT) {
- assert(!zink_curr_batch(ctx)->in_rp);
+ assert(!zink_batch_g(ctx)->in_rp);
if (zink_fb_clear_needs_explicit(fb_clear) || !check_3d_layers(ctx->fb_state.cbufs[i]))
/* this will automatically trigger all the clears */
zink_batch_rp(ctx);
zink_fb_clear_reset(&ctx->fb_clears[i]);
return;
} else {
- assert(!zink_curr_batch(ctx)->in_rp);
+ assert(!zink_batch_g(ctx)->in_rp);
if (zink_fb_clear_needs_explicit(fb_clear) || !check_3d_layers(ctx->fb_state.zsbuf))
/* this will automatically trigger all the clears */
zink_batch_rp(ctx);
#define XXH_INLINE_ALL
#include "util/xxhash.h"
+static void
+incr_curr_batch(struct zink_context *ctx)
+{
+ ctx->curr_batch++;
+ if (!ctx->curr_batch)
+ ctx->curr_batch = 1;
+}
+
static struct zink_resource *
get_resource_for_descriptor(struct zink_context *ctx, enum zink_descriptor_type type, enum pipe_shader_type shader, int idx)
{
struct zink_context *ctx = zink_context(pctx);
struct zink_screen *screen = zink_screen(pctx->screen);
- if (vkQueueWaitIdle(ctx->queue) != VK_SUCCESS)
+ if (ctx->queue && vkQueueWaitIdle(ctx->queue) != VK_SUCCESS)
debug_printf("vkQueueWaitIdle failed\n");
util_blitter_destroy(ctx->blitter);
for (unsigned i = 0; i < ARRAY_SIZE(ctx->null_buffers); i++)
pipe_resource_reference(&ctx->null_buffers[i], NULL);
- for (int i = 0; i < ARRAY_SIZE(ctx->batches); ++i) {
+ for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++) {
+ struct zink_fence *fence = zink_fence(&ctx->batches[i].state);
zink_reset_batch_state(ctx, ctx->batches[i].state);
- zink_batch_state_destroy(screen, ctx->batches[i].state);
+ zink_fence_reference(zink_screen(pctx->screen), &fence, NULL);
+ hash_table_foreach(&ctx->batch_states[i], entry) {
+ fence = entry->data;
+ zink_reset_batch_state(ctx, entry->data);
+ zink_fence_reference(zink_screen(pctx->screen), &fence, NULL);
+ }
+ util_dynarray_foreach(&ctx->free_batch_states[i], struct zink_batch_state*, bs) {
+ fence = zink_fence(*bs);
+ zink_reset_batch_state(ctx, *bs);
+ zink_fence_reference(zink_screen(pctx->screen), &fence, NULL);
+ }
}
- zink_reset_batch_state(ctx, ctx->compute_batch.state);
- zink_batch_state_destroy(screen, ctx->compute_batch.state);
hash_table_foreach(ctx->render_pass_cache, he)
zink_destroy_render_pass(screen, he->data);
void *sampler_state)
{
struct zink_sampler_state *sampler = sampler_state;
- struct zink_batch *batch = zink_curr_batch(zink_context(pctx));
+ struct zink_batch *batch = zink_batch_g(zink_context(pctx));
zink_descriptor_set_refs_clear(&sampler->desc_set_refs, sampler_state);
util_dynarray_append(&batch->state->zombie_samplers, VkSampler,
sampler->sampler);
void
zink_begin_render_pass(struct zink_context *ctx, struct zink_batch *batch)
{
- assert(batch == zink_curr_batch(ctx));
+ assert(batch == zink_batch_g(ctx));
setup_framebuffer(ctx);
assert(ctx->gfx_pipeline_state.render_pass);
}
static void
-flush_batch(struct zink_context *ctx)
+flush_batch(struct zink_context *ctx, enum zink_queue queue)
{
- struct zink_batch *batch = zink_curr_batch(ctx);
- zink_end_render_pass(ctx, batch);
+ struct zink_batch *batch = zink_batch_queue(ctx, queue);
+ if (queue == ZINK_QUEUE_GFX)
+ zink_end_render_pass(ctx, batch);
zink_end_batch(ctx, batch);
- ctx->curr_batch++;
- if (ctx->curr_batch == ARRAY_SIZE(ctx->batches))
- ctx->curr_batch = 0;
+ incr_curr_batch(ctx);
- zink_start_batch(ctx, zink_curr_batch(ctx));
+ zink_start_batch(ctx, batch);
}
struct zink_batch *
zink_batch_rp(struct zink_context *ctx)
{
- struct zink_batch *batch = zink_curr_batch(ctx);
+ struct zink_batch *batch = zink_batch_g(ctx);
if (!batch->in_rp) {
zink_begin_render_pass(ctx, batch);
assert(ctx->framebuffer && ctx->framebuffer->rp);
struct zink_batch *
zink_batch_no_rp(struct zink_context *ctx)
{
- struct zink_batch *batch = zink_curr_batch(ctx);
+ struct zink_batch *batch = zink_batch_g(ctx);
zink_end_render_pass(ctx, batch);
assert(!batch->in_rp);
return batch;
void
zink_flush_compute(struct zink_context *ctx)
{
- zink_end_batch(ctx, &ctx->compute_batch);
- zink_start_batch(ctx, &ctx->compute_batch);
+ flush_batch(ctx, ZINK_QUEUE_COMPUTE);
}
struct zink_batch *
zink_flush_batch(struct zink_context *ctx, struct zink_batch *batch)
{
- if (batch && batch->batch_id >= ZINK_COMPUTE_BATCH_ID) {
- zink_flush_compute(ctx);
- return &ctx->compute_batch;
- }
- flush_batch(ctx);
- return zink_curr_batch(ctx);
+ flush_batch(ctx, batch->queue);
+ return batch;
}
static void
/* only barrier if we're changing layout or doing something besides read -> read */
if (!batch) {
if (pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT)
- batch = &ctx->compute_batch;
+ batch = zink_batch_c(ctx);
else
batch = zink_batch_no_rp(ctx);
}
/* only barrier if we're changing layout or doing something besides read -> read */
if (!batch) {
if (pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT)
- batch = &ctx->compute_batch;
+ batch = zink_batch_c(ctx);
else
batch = zink_batch_no_rp(ctx);
}
{
struct zink_context *ctx = zink_context(pctx);
bool deferred = flags & PIPE_FLUSH_DEFERRED;
- struct zink_batch *batch = zink_curr_batch(ctx);
+ struct zink_batch *batch = zink_batch_g(ctx);
+ struct zink_fence *fence = &batch->state->fence;
if (deferred)
- batch->state->fence->deferred_ctx = pctx;
+ batch->state->fence.deferred_ctx = pctx;
else if (batch->has_work) {
if (flags & PIPE_FLUSH_END_OF_FRAME) {
if (ctx->fb_state.nr_cbufs)
if (zink_screen(pctx->screen)->needs_mesa_flush_wsi && ctx->fb_state.cbufs[0])
batch->state->flush_res = zink_resource(ctx->fb_state.cbufs[0]->texture);
}
- flush_batch(ctx);
+ flush_batch(ctx, ZINK_QUEUE_GFX);
if (zink_screen(pctx->screen)->info.have_EXT_transform_feedback && ctx->num_so_targets)
ctx->dirty_so_targets = true;
if (!pfence)
return;
if (deferred && !batch->has_work) {
- batch = zink_prev_batch(ctx);
+ fence = ctx->last_fence[ZINK_QUEUE_GFX];
}
zink_fence_reference(zink_screen(pctx->screen),
(struct zink_fence **)pfence,
- batch->state->fence);
+ fence);
if (flags & PIPE_FLUSH_END_OF_FRAME) {
/* if the first frame has not yet occurred, we need an explicit fence here
* in some cases in order to correctly draw the first frame, though it's
* unknown at this time why this is the case
*/
if (!ctx->first_frame_done)
- zink_fence_finish(zink_screen(pctx->screen), pctx, batch->state->fence, PIPE_TIMEOUT_INFINITE);
+ zink_fence_finish(zink_screen(pctx->screen), pctx, fence, PIPE_TIMEOUT_INFINITE);
ctx->first_frame_done = true;
}
}
void
-zink_fence_wait(struct pipe_context *pctx)
+zink_maybe_flush_or_stall(struct zink_context *ctx, enum zink_queue queue)
{
- struct pipe_fence_handle *fence = NULL;
- pctx->flush(pctx, &fence, PIPE_FLUSH_HINT_FINISH);
- if (fence) {
- pctx->screen->fence_finish(pctx->screen, NULL, fence,
- PIPE_TIMEOUT_INFINITE);
- pctx->screen->fence_reference(pctx->screen, &fence, NULL);
+ struct zink_screen *screen = zink_screen(ctx->base.screen);
+ /* flush anytime our total batch memory usage is potentially >= 1/10 of total system memory */
+ if (zink_batch_queue(ctx, queue)->state->resource_size >= screen->total_mem / 10)
+ flush_batch(ctx, queue);
+
+ if (ctx->resource_size[queue] >= screen->total_mem / 10) {
+ zink_fence_finish(zink_screen(ctx->base.screen), &ctx->base, ctx->last_fence[queue], PIPE_TIMEOUT_INFINITE);
+ zink_batch_reset_all(ctx, queue);
}
}
void
-zink_wait_on_batch(struct zink_context *ctx, int batch_id)
-{
- if (batch_id >= 0) {
- struct zink_batch *batch = batch_id == ZINK_COMPUTE_BATCH_ID ? &ctx->compute_batch : &ctx->batches[batch_id];
- if (batch != zink_curr_batch(ctx)) {
- if (!batch->state->fence->submitted) { // this is the compute batch
- zink_flush_compute(ctx);
- } else
- ctx->base.screen->fence_finish(ctx->base.screen, NULL, (struct pipe_fence_handle*)batch->state->fence,
- PIPE_TIMEOUT_INFINITE);
- return;
+zink_fence_wait(struct pipe_context *pctx)
+{
+ struct zink_context *ctx = zink_context(pctx);
+
+ if (zink_batch_g(ctx)->has_work)
+ pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH);
+ if (ctx->last_fence[ZINK_QUEUE_GFX])
+ zink_fence_finish(zink_screen(pctx->screen), pctx, ctx->last_fence[ZINK_QUEUE_GFX], PIPE_TIMEOUT_INFINITE);
+}
+
+void
+zink_wait_on_batch(struct zink_context *ctx, enum zink_queue queue, uint32_t batch_id)
+{
+ struct zink_batch_state *bs = zink_batch_queue(ctx, queue)->state;
+ assert(bs);
+ if (!batch_id || bs->batch_id == batch_id)
+ /* not submitted yet */
+ flush_batch(ctx, queue);
+
+ struct zink_fence *fence;
+
+ assert(batch_id || ctx->last_fence[queue]);
+ if (ctx->last_fence[queue] && (!batch_id || batch_id == zink_batch_state(ctx->last_fence[queue])->batch_id))
+ fence = ctx->last_fence[queue];
+ else {
+ struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&ctx->batch_states[queue], batch_id, (void*)(uintptr_t)batch_id);
+ if (!he) {
+ util_dynarray_foreach(&ctx->free_batch_states[queue], struct zink_batch_state*, bs) {
+ if ((*bs)->batch_id == batch_id)
+ return;
+ }
+ unreachable("should've found batch state");
}
+ fence = he->data;
}
- zink_fence_wait(&ctx->base);
+ assert(fence);
+ ctx->base.screen->fence_finish(ctx->base.screen, &ctx->base, (struct pipe_fence_handle*)fence, PIPE_TIMEOUT_INFINITE);
}
static void
zink_texture_barrier(struct pipe_context *pctx, unsigned flags)
{
struct zink_context *ctx = zink_context(pctx);
- if (zink_curr_batch(ctx)->has_work)
+ if (zink_batch_g(ctx)->has_work)
pctx->flush(pctx, NULL, 0);
zink_flush_compute(ctx);
}
b.srcAccessMask = sflags;
b.dstAccessMask = dflags;
- struct zink_batch *batch = zink_curr_batch(ctx);
+ struct zink_batch *batch = zink_batch_g(ctx);
if (batch->has_work) {
zink_end_render_pass(ctx, batch);
/* this should be the only call needed */
vkCmdPipelineBarrier(batch->state->cmdbuf, src, dst, 0, 0, &b, 0, NULL, 0, NULL);
- flush_batch(ctx);
+ flush_batch(ctx, ZINK_QUEUE_GFX);
}
- batch = &ctx->compute_batch;
+ batch = zink_batch_c(ctx);
if (batch->has_work) {
/* this should be the only call needed */
vkCmdPipelineBarrier(batch->state->cmdbuf, src, dst, 0, 0, &b, 0, NULL, 0, NULL);
}
}
-static bool
-init_batch(struct zink_context *ctx, struct zink_batch *batch, unsigned idx)
+static void
+init_batch(struct zink_context *ctx, enum zink_queue queue)
{
- batch->queue = idx == ZINK_COMPUTE_BATCH_ID ? ZINK_QUEUE_COMPUTE : ZINK_QUEUE_GFX;
- batch->batch_id = idx;
+ struct zink_batch *batch = zink_batch_queue(ctx, queue);
+ batch->queue = queue;
zink_start_batch(ctx, batch);
- return !!batch->state;
}
struct pipe_context *
zink_context_resource_init(&ctx->base);
zink_context_query_init(&ctx->base);
+ for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++) {
+ util_dynarray_init(&ctx->free_batch_states[i], ctx);
+ _mesa_hash_table_init(&ctx->batch_states[i], ctx, NULL, _mesa_key_pointer_equal);
+ }
+
ctx->gfx_pipeline_state.have_EXT_extended_dynamic_state = screen->info.have_EXT_extended_dynamic_state;
slab_create_child(&ctx->transfer_pool, &screen->transfer_pool);
if (!ctx->blitter)
goto fail;
- for (int i = 0; i < ARRAY_SIZE(ctx->batches); ++i) {
- if (!init_batch(ctx, &ctx->batches[i], i))
- goto fail;
- }
+ incr_curr_batch(ctx);
+ init_batch(ctx, ZINK_QUEUE_GFX);
+ if (!zink_batch_g(ctx)->state)
+ goto fail;
- if (!init_batch(ctx, &ctx->compute_batch, ZINK_COMPUTE_BATCH_ID))
+ init_batch(ctx, ZINK_QUEUE_COMPUTE);
+ if (!zink_batch_c(ctx)->state)
goto fail;
vkGetDeviceQueue(screen->dev, screen->gfx_queue, 0, &ctx->queue);
#define ZINK_CONTEXT_H
#define ZINK_SHADER_COUNT (PIPE_SHADER_TYPES - 1)
-#define ZINK_NUM_GFX_BATCHES 4
-#define ZINK_COMPUTE_BATCH_ID ZINK_NUM_GFX_BATCHES
-#define ZINK_COMPUTE_BATCH_COUNT 1
-#define ZINK_NUM_BATCHES (ZINK_NUM_GFX_BATCHES + 1)
#define ZINK_DEFAULT_MAX_DESCS 5000
#include "util/slab.h"
#include "util/list.h"
+#include "util/u_dynarray.h"
#include <vulkan/vulkan.h>
VkSampler sampler;
uint32_t hash;
struct zink_descriptor_refs desc_set_refs;
- uint32_t batch_uses;
+ struct zink_batch_usage batch_uses;
bool custom_border_color;
};
VkBufferViewCreateInfo bvci;
VkBufferView buffer_view;
uint32_t hash;
- uint32_t batch_uses;
+ struct zink_batch_usage batch_uses;
};
struct zink_sampler_view {
struct pipe_device_reset_callback reset;
- struct zink_batch batches[ZINK_NUM_GFX_BATCHES];
bool is_device_lost;
- unsigned curr_batch;
+ uint32_t curr_batch; //the current batch id
+ struct zink_batch batches[2]; //gfx, compute
+ struct zink_fence *last_fence[2]; //gfx, compute; the last command buffer submitted
VkQueue queue;
-
- struct zink_batch compute_batch;
+ struct hash_table batch_states[2]; //gfx, compute; submitted batch states
+ struct util_dynarray free_batch_states[2]; //gfx, compute; unused batch states
+ VkDeviceSize resource_size[2]; //gfx, compute; the accumulated size of resources in submitted buffers
struct pipe_constant_buffer ubos[PIPE_SHADER_TYPES][PIPE_MAX_CONSTANT_BUFFERS];
struct pipe_shader_buffer ssbos[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_BUFFERS];
}
static inline struct zink_batch *
-zink_curr_batch(struct zink_context *ctx)
+zink_batch_queue(struct zink_context *ctx, enum zink_queue queue_type)
{
- assert(ctx->curr_batch < ARRAY_SIZE(ctx->batches));
- return ctx->batches + ctx->curr_batch;
+ assert(queue_type < ARRAY_SIZE(ctx->batches));
+ return &ctx->batches[queue_type];
}
static inline struct zink_batch *
-zink_prev_batch(struct zink_context *ctx)
+zink_batch_g(struct zink_context *ctx)
{
- unsigned curr_batch = ctx->curr_batch;
- if (!curr_batch)
- curr_batch = ZINK_NUM_GFX_BATCHES - 1;
- else
- curr_batch--;
- assert(curr_batch < ARRAY_SIZE(ctx->batches));
- return ctx->batches + curr_batch;
+ return &ctx->batches[ZINK_QUEUE_GFX];
+}
+
+static inline struct zink_batch *
+zink_batch_c(struct zink_context *ctx)
+{
+ return &ctx->batches[ZINK_QUEUE_COMPUTE];
}
struct zink_batch *
zink_fence_wait(struct pipe_context *ctx);
void
-zink_wait_on_batch(struct zink_context *ctx, int batch_id);
+zink_wait_on_batch(struct zink_context *ctx, enum zink_queue queue, uint32_t batch_id);
void
zink_flush_compute(struct zink_context *ctx);
struct zink_batch *
zink_flush_batch(struct zink_context *ctx, struct zink_batch *batch);
+void
+zink_maybe_flush_or_stall(struct zink_context *ctx, enum zink_queue queue);
+
bool
zink_resource_access_is_write(VkAccessFlags flags);
pipe_reference_init(&zds->reference, 1);
zds->pool = pool;
zds->hash = 0;
- zds->batch_uses = 0;
+ zds->batch_uses.usage[0] = zds->batch_uses.usage[1] = 0;
zds->invalid = true;
zds->punted = zds->recycled = false;
if (num_resources) {
struct zink_descriptor_set *zds;
struct zink_screen *screen = zink_screen(ctx->base.screen);
struct zink_program *pg = is_compute ? (struct zink_program *)ctx->curr_compute : (struct zink_program *)ctx->curr_program;
- struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx);
+ struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx);
struct zink_descriptor_pool *pool = pg->pool[type];
unsigned descs_used = 1;
assert(type < ZINK_DESCRIPTOR_TYPES);
zds->recycled = false;
}
if (zds->invalid) {
- if (zds->batch_uses)
+ if (zink_batch_usage_exists(&zds->batch_uses))
punt_invalid_set(zds, NULL);
else
/* this set is guaranteed to be in pool->alloc_desc_sets */
bool recycled = false, punted = false;
if (he) {
zds = (void*)he->data;
- if (zds->invalid && zds->batch_uses) {
+ if (zds->invalid && zink_batch_usage_exists(&zds->batch_uses)) {
punt_invalid_set(zds, he);
zds = NULL;
punted = true;
}
if (pool->num_sets_allocated + pool->key.num_descriptors > ZINK_DEFAULT_MAX_DESCS) {
- batch = zink_flush_batch(ctx, batch);
+ zink_fence_wait(&ctx->base);
zink_batch_reference_program(batch, pg);
return zink_descriptor_set_get(ctx, type, is_compute, cache_hit, need_resource_refs);
}
bool recycled;
struct zink_descriptor_state_key key;
struct util_dynarray barriers;
- uint32_t batch_uses;
+ struct zink_batch_usage batch_uses;
#ifndef NDEBUG
/* for extra debug asserts */
unsigned num_resources;
{
struct zink_context *ctx = zink_context(pctx);
struct zink_screen *screen = zink_screen(pctx->screen);
- struct zink_batch *batch = zink_curr_batch(ctx);
+ struct zink_batch *batch = zink_batch_g(ctx);
VkBuffer buffers[PIPE_MAX_SO_OUTPUTS] = {};
VkDeviceSize buffer_offsets[PIPE_MAX_SO_OUTPUTS] = {};
VkDeviceSize buffer_sizes[PIPE_MAX_SO_OUTPUTS] = {};
bool is_compute, bool cache_hit, bool need_resource_refs)
{
bool need_flush = false;
- struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx);
+ struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx);
struct zink_screen *screen = zink_screen(ctx->base.screen);
assert(zds->desc_set);
enum zink_queue check_flush_id = is_compute ? ZINK_QUEUE_GFX : ZINK_QUEUE_COMPUTE;
desc_set_sampler_add(ctx, zds, sampler_view, sampler, num_resources++,
zink_shader_descriptor_is_buffer(shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j),
cache_hit);
- struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx);
+ struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx);
if (sampler_view)
zink_batch_reference_sampler_view(batch, sampler_view);
if (sampler)
/* this only tracks the most recent usage for now */
- sampler->batch_uses = BITFIELD_BIT(batch->state->batch_id);
+ zink_batch_usage_set(&sampler->batch_uses, batch->queue, batch->state->batch_id);
}
assert(num_wds < num_descriptors);
&num_buffer_info, &buffer_views[num_buffer_info],
NULL, imageview, bufferview, !k);
- struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx);
+ struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx);
if (res)
zink_batch_reference_image_view(batch, image_view);
}
else
zds[h] = NULL;
}
- struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx);
+ struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx);
zink_batch_reference_program(batch, pg);
uint32_t dynamic_offsets[PIPE_MAX_CONSTANT_BUFFERS];
VkDeviceSize counter_buffer_offsets[PIPE_MAX_SO_OUTPUTS] = {};
bool need_index_buffer_unref = false;
- /* flush anytime our total batch memory usage is potentially >= 1/10 of total gpu memory
- * this should also eventually trigger a stall if the app is going nuts with gpu memory
- */
- if (zink_curr_batch(ctx)->state->resource_size >= screen->total_mem / 10 / ZINK_NUM_BATCHES)
- ctx->base.flush(&ctx->base, NULL, 0);
+ /* check memory usage and flush/stall as needed to avoid oom */
+ zink_maybe_flush_or_stall(ctx, ZINK_QUEUE_GFX);
if (dinfo->primitive_restart && !restart_supported(dinfo->mode)) {
util_draw_vbo_without_prim_restart(pctx, dinfo, dindirect, &draws[0]);
{
struct zink_context *ctx = zink_context(pctx);
struct zink_screen *screen = zink_screen(pctx->screen);
- struct zink_batch *batch = &ctx->compute_batch;
+ struct zink_batch *batch = zink_batch_c(ctx);
- /* flush anytime our total batch memory usage is potentially >= 1/10 of total gpu memory
- * this should also eventually trigger a stall if the app is going nuts with gpu memory
- */
- if (batch->state->resource_size >= screen->total_mem / 10 / ZINK_NUM_BATCHES)
- zink_flush_compute(ctx);
+ /* check memory usage and flush/stall as needed to avoid oom */
+ zink_maybe_flush_or_stall(ctx, ZINK_QUEUE_COMPUTE);
struct zink_compute_program *comp_program = get_compute_program(ctx);
if (!comp_program)
{
if (fence->fence)
vkDestroyFence(screen->dev, fence->fence, NULL);
- util_dynarray_fini(&fence->resources);
- FREE(fence);
+ zink_batch_state_destroy(screen, zink_batch_state(fence));
}
bool
zink_create_fence(struct zink_screen *screen, struct zink_batch_state *bs)
{
+ struct zink_fence *fence = zink_fence(bs);
+
VkFenceCreateInfo fci = {};
fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
- struct zink_fence *ret = CALLOC_STRUCT(zink_fence);
- if (!ret) {
- debug_printf("CALLOC_STRUCT failed\n");
- return false;
- }
-
- if (vkCreateFence(screen->dev, &fci, NULL, &ret->fence) != VK_SUCCESS) {
+ if (vkCreateFence(screen->dev, &fci, NULL, &fence->fence) != VK_SUCCESS) {
debug_printf("vkCreateFence failed\n");
goto fail;
}
- ret->batch_id = bs->batch_id;
- util_dynarray_init(&ret->resources, NULL);
- pipe_reference_init(&ret->reference, 1);
- bs->fence = ret;
+ pipe_reference_init(&fence->reference, 1);
return true;
-
fail:
- destroy_fence(screen, ret);
+ destroy_fence(screen, fence);
return false;
}
void
zink_fence_init(struct zink_context *ctx, struct zink_batch *batch)
{
- struct zink_fence *fence = batch->state->fence;
- set_foreach(batch->state->resources, entry) {
- /* the fence needs its own reference to ensure it can safely access lifetime-dependent
- * resource members
- */
- struct zink_resource_object *obj = (struct zink_resource_object *)entry->key;
- pipe_reference(NULL, &obj->reference);
- util_dynarray_append(&fence->resources, struct zink_resource_object*, obj);
- }
+ struct zink_fence *fence = zink_fence(batch->state);
+
vkResetFences(zink_screen(ctx->base.screen)->dev, 1, &fence->fence);
fence->deferred_ctx = NULL;
fence->submitted = true;
zink_fence(pfence));
}
-static inline void
-fence_remove_resource_access(struct zink_fence *fence, struct zink_resource_object *obj)
-{
- p_atomic_set(&obj->batch_uses[fence->batch_id], 0);
-}
-
bool
zink_fence_finish(struct zink_screen *screen, struct pipe_context *pctx, struct zink_fence *fence,
uint64_t timeout_ns)
{
if (pctx && fence->deferred_ctx == pctx) {
- zink_curr_batch(zink_context(pctx))->has_work = true;
+ zink_batch_g(zink_context(pctx))->has_work = true;
/* this must be the current batch */
pctx->flush(pctx, NULL, 0);
}
success = vkGetFenceStatus(screen->dev, fence->fence) == VK_SUCCESS;
if (success) {
- /* unref all used resources */
- util_dynarray_foreach(&fence->resources, struct zink_resource_object*, obj) {
- fence_remove_resource_access(fence, *obj);
-
- zink_resource_object_reference(screen, obj, NULL);
- }
- util_dynarray_clear(&fence->resources);
- fence->submitted = false;
+ struct zink_batch_state *bs = zink_batch_state(fence);
+ zink_batch_state_clear_resources(screen, bs);
+ p_atomic_set(&fence->submitted, false);
}
return success;
}
return;
if (fence->deferred_ctx) {
- zink_curr_batch(zink_context(pctx))->has_work = true;
+ zink_batch_g(zink_context(pctx))->has_work = true;
/* this must be the current batch */
pctx->flush(pctx, NULL, 0);
}
#define ZINK_FENCE_H
#include "util/u_inlines.h"
-#include "util/u_dynarray.h"
#include <vulkan/vulkan.h>
struct pipe_context;
struct pipe_screen;
+struct zink_batch;
struct zink_batch_state;
struct zink_context;
struct zink_screen;
struct zink_fence {
struct pipe_reference reference;
- unsigned batch_id : 3;
VkFence fence;
- struct util_dynarray resources;
struct pipe_context *deferred_ctx;
bool submitted;
};
static inline struct zink_fence *
-zink_fence(struct pipe_fence_handle *pfence)
+zink_fence(void *pfence)
{
return (struct zink_fence *)pfence;
}
bool have_gs[NUM_QUERIES]; /* geometry shaders use GEOMETRY_SHADER_PRIMITIVES_BIT */
bool have_xfb[NUM_QUERIES]; /* xfb was active during this query */
- unsigned batch_id : 3; //batch that the query was started in
+ struct zink_batch_usage batch_id; //batch that the query was started in
union pipe_query_result accumulated_result;
};
get_batch_for_query(struct zink_context *ctx, struct zink_query *query, bool no_rp)
{
if (query && is_cs_query(query))
- return &ctx->compute_batch;
+ return zink_batch_c(ctx);
if (no_rp)
return zink_batch_no_rp(ctx);
- return zink_curr_batch(ctx);
+ return zink_batch_g(ctx);
}
static struct pipe_query *
unsigned result_size = result_type <= PIPE_QUERY_TYPE_U32 ? sizeof(uint32_t) : sizeof(uint64_t);
struct zink_query *query = (struct zink_query*)pquery;
union pipe_query_result result;
- if (zink_curr_batch(ctx)->state->batch_id == query->batch_id)
+ if (zink_batch_usage_matches(&query->batch_id, ZINK_QUEUE_GFX, zink_batch_g(ctx)->state->batch_id))
pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH);
else if (is_cs_query(query))
zink_flush_compute(ctx);
if (needs_stats_list(q))
list_addtail(&q->stats_list, &ctx->primitives_generated_queries);
p_atomic_inc(&q->fences);
- q->batch_id = batch->state->batch_id;
+ zink_batch_usage_set(&q->batch_id, batch->queue, batch->state->batch_id);
_mesa_set_add(batch->state->active_queries, q);
}
if (is_time_query(q)) {
vkCmdWriteTimestamp(batch->state->cmdbuf, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
q->query_pool, q->curr_query);
- q->batch_id = batch->state->batch_id;
+ zink_batch_usage_set(&q->batch_id, batch->queue, batch->state->batch_id);
} else if (q->type == PIPE_QUERY_PRIMITIVES_EMITTED ||
q->type == PIPE_QUERY_PRIMITIVES_GENERATED ||
q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE)
struct zink_query *query = (void*)q;
struct zink_context *ctx = zink_context(pctx);
if (is_cs_query(query)) {
- if (wait)
- zink_wait_on_batch(ctx, ZINK_COMPUTE_BATCH_ID);
- else {
+ if (wait) {
+ uint32_t batch_id = p_atomic_read(&query->batch_id.usage[ZINK_QUEUE_COMPUTE]);
+ zink_wait_on_batch(ctx, ZINK_QUEUE_COMPUTE, batch_id);
+ } else {
zink_flush_compute(ctx);
}
} else {
struct zink_context *ctx = zink_context(pctx);
ctx->queries_disabled = !enable;
- struct zink_batch *batch = zink_curr_batch(ctx);
+ struct zink_batch *batch = zink_batch_g(ctx);
if (ctx->queries_disabled)
zink_suspend_queries(ctx, batch);
else
if (query->type != PIPE_QUERY_PRIMITIVES_GENERATED &&
!is_so_overflow_query(query)) {
copy_results_to_buffer(ctx, query, res, 0, num_results, flags);
- batch = zink_curr_batch(ctx);
+ batch = zink_batch_g(ctx);
} else {
/* these need special handling */
force_cpu_read(ctx, pquery, true, PIPE_QUERY_TYPE_U32, pres, 0);
- batch = zink_curr_batch(ctx);
+ batch = zink_batch_g(ctx);
zink_batch_reference_resource_rw(batch, res, false);
}
}
static uint32_t
-get_resource_usage(struct zink_resource *res)
+get_resource_usage(struct zink_resource *res, enum zink_queue queue)
{
+ assert(queue < 2);
+ uint32_t reads = p_atomic_read(&res->obj->reads.usage[queue]);
+ uint32_t writes = p_atomic_read(&res->obj->writes.usage[queue]);
uint32_t batch_uses = 0;
- for (unsigned i = 0; i < ARRAY_SIZE(res->obj->batch_uses); i++)
- batch_uses |= p_atomic_read(&res->obj->batch_uses[i]) << i;
+ if (reads)
+ batch_uses |= ZINK_RESOURCE_ACCESS_READ << queue;
+ if (writes)
+ batch_uses |= ZINK_RESOURCE_ACCESS_WRITE << queue;
+ return batch_uses;
+}
+
+static uint32_t
+get_all_resource_usage(struct zink_resource *res)
+{
+ uint32_t batch_uses = 0;
+ for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++)
+ batch_uses |= get_resource_usage(res, i);
return batch_uses;
}
static void
-resource_sync_writes_from_batch_usage(struct zink_context *ctx, struct zink_resource *res)
+resource_sync_reads_from_compute(struct zink_context *ctx, struct zink_resource *res)
{
- uint32_t batch_uses = get_resource_usage(res);
- batch_uses &= ~(ZINK_RESOURCE_ACCESS_READ << ZINK_COMPUTE_BATCH_ID);
-
- uint32_t write_mask = 0;
- for (int i = 0; i < ZINK_NUM_GFX_BATCHES + ZINK_COMPUTE_BATCH_COUNT; i++)
- write_mask |= ZINK_RESOURCE_ACCESS_WRITE << i;
- while (batch_uses & write_mask) {
- int batch_id = zink_get_resource_latest_batch_usage(ctx, batch_uses);
- if (batch_id == -1)
- break;
- zink_wait_on_batch(ctx, batch_id);
- batch_uses &= ~((ZINK_RESOURCE_ACCESS_RW) << batch_id);
- }
+ uint32_t reads = p_atomic_read(&res->obj->reads.usage[ZINK_QUEUE_COMPUTE]);
+ assert(reads);
+ zink_wait_on_batch(ctx, ZINK_QUEUE_COMPUTE, reads);
}
-int
-zink_get_resource_latest_batch_usage(struct zink_context *ctx, uint32_t batch_uses)
+static void
+resource_sync_writes_from_batch_usage(struct zink_context *ctx, struct zink_resource *res)
{
- unsigned cur_batch = zink_curr_batch(ctx)->batch_id;
-
- if (batch_uses & ZINK_RESOURCE_ACCESS_WRITE << ZINK_COMPUTE_BATCH_ID)
- return ZINK_COMPUTE_BATCH_ID;
- batch_uses &= ~(ZINK_RESOURCE_ACCESS_WRITE << ZINK_COMPUTE_BATCH_ID);
- if (!batch_uses)
- return -1;
- for (unsigned i = 0; i < ZINK_NUM_BATCHES + 1; i++) {
- /* loop backwards and sync with highest batch id that has writes */
- if (batch_uses & (ZINK_RESOURCE_ACCESS_WRITE << cur_batch)) {
- return cur_batch;
- }
- cur_batch--;
- if (cur_batch > ZINK_COMPUTE_BATCH_ID - 1) // underflowed past max batch id
- cur_batch = ZINK_COMPUTE_BATCH_ID - 1;
- }
- return -1;
+ uint32_t writes[2];
+ for (int i = 0; i < ZINK_QUEUE_ANY; i++)
+ writes[i] = p_atomic_read(&res->obj->writes.usage[i]);
+
+ enum zink_queue queue = writes[0] < writes[1] ? ZINK_QUEUE_COMPUTE : ZINK_QUEUE_GFX;
+ /* sync lower id first */
+ if (writes[!queue])
+ zink_wait_on_batch(ctx, !queue, writes[!queue]);
+ zink_wait_on_batch(ctx, queue, writes[queue]);
}
static uint32_t
res->bind_history &= ~ZINK_RESOURCE_USAGE_STREAMOUT;
util_range_set_empty(&res->valid_buffer_range);
- if (!get_resource_usage(res))
+ if (!get_all_resource_usage(res))
return;
struct zink_resource_object *old_obj = res->obj;
box.y, box.z, trans->base.level, &box, trans->base.usage);
}
-#define ALL_GFX_USAGE(batch_uses, usage) (batch_uses & ((usage << ZINK_NUM_GFX_BATCHES) - ((usage & ZINK_RESOURCE_ACCESS_RW))))
-#define ALL_COMPUTE_USAGE(batch_uses, usage) (batch_uses & (usage << ZINK_COMPUTE_BATCH_ID))
-
bool
zink_resource_has_usage(struct zink_resource *res, enum zink_resource_access usage, enum zink_queue queue)
{
- uint32_t batch_uses = get_resource_usage(res);
+ uint32_t batch_uses = get_all_resource_usage(res);
switch (queue) {
case ZINK_QUEUE_COMPUTE:
- return ALL_COMPUTE_USAGE(batch_uses, usage);
+ return batch_uses & (usage << ZINK_QUEUE_COMPUTE);
case ZINK_QUEUE_GFX:
- return ALL_GFX_USAGE(batch_uses, usage);
+ return batch_uses & (usage << ZINK_QUEUE_GFX);
case ZINK_QUEUE_ANY:
- return ALL_GFX_USAGE(batch_uses, usage) || ALL_COMPUTE_USAGE(batch_uses, usage);
+ return batch_uses & ((usage << ZINK_QUEUE_GFX) | (usage << ZINK_QUEUE_COMPUTE));
default:
break;
}
return false;
}
-bool
-zink_resource_has_usage_for_id(struct zink_resource *res, uint32_t id)
-{
- uint32_t batch_uses = get_resource_usage(res);
- return batch_uses & (ZINK_RESOURCE_ACCESS_RW) << id;
-}
-
static void *
zink_transfer_map(struct pipe_context *pctx,
struct pipe_resource *pres,
if (util_ranges_intersect(&res->valid_buffer_range, box->x, box->x + box->width)) {
/* special case compute reads since they aren't handled by zink_fence_wait() */
if (usage & PIPE_MAP_WRITE && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_READ, ZINK_QUEUE_COMPUTE))
- zink_wait_on_batch(ctx, ZINK_COMPUTE_BATCH_ID);
+ resource_sync_reads_from_compute(ctx, res);
if (usage & PIPE_MAP_READ && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_WRITE, ZINK_QUEUE_ANY))
resource_sync_writes_from_batch_usage(ctx, res);
else if (usage & PIPE_MAP_WRITE && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_RW, ZINK_QUEUE_ANY)) {
assert(!res->optimal_tiling);
/* special case compute reads since they aren't handled by zink_fence_wait() */
if (zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_READ, ZINK_QUEUE_COMPUTE))
- zink_wait_on_batch(ctx, ZINK_COMPUTE_BATCH_ID);
+ resource_sync_reads_from_compute(ctx, res);
if (zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_RW, ZINK_QUEUE_ANY)) {
if (usage & PIPE_MAP_READ)
resource_sync_writes_from_batch_usage(ctx, res);
#include "util/u_range.h"
#include "util/u_dynarray.h"
+#include "zink_batch.h"
#include "zink_descriptors.h"
#include <vulkan/vulkan.h>
unsigned persistent_maps; //if nonzero, requires vkFlushMappedMemoryRanges during batch use
struct zink_descriptor_refs desc_set_refs;
- /* this has to be atomic for fence access, so we can't use a bitmask and make everything neat */
- uint8_t batch_uses[5]; //ZINK_NUM_BATCHES
+ struct zink_batch_usage reads;
+ struct zink_batch_usage writes;
bool is_buffer;
bool host_visible;
};
void
zink_resource_setup_transfer_layouts(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst);
-int
-zink_get_resource_latest_batch_usage(struct zink_context *ctx, uint32_t batch_uses);
-
bool
zink_resource_has_usage(struct zink_resource *res, enum zink_resource_access usage, enum zink_queue queue);
#define ZINK_SURFACE_H
#include "pipe/p_state.h"
-
+#include "zink_batch.h"
#include <vulkan/vulkan.h>
struct pipe_context;
VkImageViewCreateInfo ivci;
VkImageView image_view;
uint32_t hash;
- uint32_t batch_uses;
+ struct zink_batch_usage batch_uses;
};
static inline struct zink_surface *
projects / platform / upstream / mesa.git / commitdiff